code_execution_with_mcp/synthetic_test_dataset.json at main · madhurprash/code_execution_with_mcp · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
{
  "test_dataset": {
    "name": "GitHub MCP Tools Test Dataset - Filtered Analysis",
    "description": "Questions designed to test efficient data filtering and summarization from madhurprash repositories",
    "total_questions": 10,
    "questions": [
      {
        "id": 1,
        "category": "repository_analysis",
        "question": "Analyze ALL repositories owned by madhurprash and identify the top 5 most starred repositories. For each, provide the name, star count, primary language, and description.",
        "difficulty": "medium",
        "expected_tools": [
          "search_users",
          "get_repos"
        ],
        "expected_behavior": "Fetch all repos, sort by stars, return only top 5 with specific fields"
      },
      {
        "id": 2,
        "category": "repository_analysis",
        "question": "Examine ALL repositories owned by madhurprash and categorize them by primary programming language. Return a summary showing each language and how many repos use it, sorted by count descending.",
        "difficulty": "medium",
        "expected_tools": [
          "search_users",
          "get_repos"
        ],
        "expected_behavior": "Fetch all repos, group by language, return language distribution summary"
      },
      {
        "id": 3,
        "category": "repository_analysis",
        "question": "From ALL madhurprash repositories, identify those specifically related to 'agents' or 'multi-agent systems' by analyzing repository names, descriptions, and topics. Return only the top 10 most relevant with their names, descriptions, and star counts.",
        "difficulty": "medium",
        "expected_tools": [
          "search_users",
          "get_repos"
        ],
        "expected_behavior": "Fetch all repos, filter by keywords in name/description/topics, return top 10 matches"
      },
      {
        "id": 4,
        "category": "repository_analysis",
        "question": "Analyze ALL madhurprash repositories and identify which ones are forked vs original. Return summary statistics: total repos, number of forks, number of original repos, and list the top 5 most forked repositories.",
        "difficulty": "medium",
        "expected_tools": [
          "search_users",
          "get_repos"
        ],
        "expected_behavior": "Fetch all repos, check fork status, return aggregated statistics + top 5"
      },
      {
        "id": 5,
        "category": "repository_analysis",
        "question": "Scan ALL madhurprash repositories and find those that contain 'AWS' or 'Bedrock' in their name, description, or topics. Return a filtered list with repository names, descriptions, and AWS-related topics.",
        "difficulty": "medium",
        "expected_tools": [
          "search_users",
          "get_repos"
        ],
        "expected_behavior": "Fetch all repos, filter by AWS/Bedrock keywords, return matches with specific fields"
      },
      {
        "id": 6,
        "category": "repository_analysis",
        "question": "Analyze ALL madhurprash repositories to find the most active ones based on recent updates. Return the top 10 repositories by last update date, showing name, last updated date, and description.",
        "difficulty": "medium",
        "expected_tools": [
          "search_users",
          "get_repos"
        ],
        "expected_behavior": "Fetch all repos, sort by updated_at, return top 10 with dates"
      },
      {
        "id": 7,
        "category": "repository_analysis",
        "question": "From ALL madhurprash repositories, identify those with open source licenses. Summarize by license type showing count per license, and list the top 5 most starred licensed repositories.",
        "difficulty": "medium",
        "expected_tools": [
          "search_users",
          "get_repos"
        ],
        "expected_behavior": "Fetch all repos, filter by license, group by license type, return summary + top 5"
      },
      {
        "id": 8,
        "category": "repository_analysis",
        "question": "Examine ALL madhurprash repositories and calculate aggregate statistics: total repositories, total stars across all repos, total forks across all repos, and average stars per repository. Also identify the language with most total stars.",
        "difficulty": "hard",
        "expected_tools": [
          "search_users",
          "get_repos"
        ],
        "expected_behavior": "Fetch all repos, compute aggregates, return statistical summary"
      },
      {
        "id": 9,
        "category": "repository_analysis",
        "question": "Analyze ALL madhurprash repositories and identify those related to machine learning or AI by checking names, descriptions, and topics for keywords like 'ML', 'AI', 'model', 'LLM', 'agent'. Return top 10 matches sorted by stars.",
        "difficulty": "medium",
        "expected_tools": [
          "search_users",
          "get_repos"
        ],
        "expected_behavior": "Fetch all repos, filter by ML/AI keywords, sort by stars, return top 10"
      },
      {
        "id": 10,
        "category": "repository_analysis",
        "question": "From ALL madhurprash repositories, find those that have associated topics/tags. Analyze the topics and return the 15 most frequently used topics across all repositories with their occurrence counts.",
        "difficulty": "hard",
        "expected_tools": [
          "search_users",
          "get_repos"
        ],
        "expected_behavior": "Fetch all repos, extract all topics, count frequencies, return top 15"
      }
    ]
  }
}