Add mbpp

promptsource/templates.py

@@ -27,7 +27,20 @@
 # These are users whose datasets should be included in the results returned by
 # filter_english_datasets (regardless of their metadata)
 
-INCLUDED_USERS = {"Zaid", "craffel", "GEM", "aps", "khalidalt", "shanya", "rbawden", "BigScienceBiasEval", "gsarti"}
+INCLUDED_USERS = {
+    "Zaid",
+    "craffel",
+    "GEM",
+    "aps",
+    "khalidalt",
+    "shanya",
+    "rbawden",
+    "BigScienceBiasEval",
+    "gsarti",
+    "Helsinki-NLP",
+    "Muennighoff",
+    "facebook",
+}
 
 # These are the metrics with which templates can be tagged
 METRICS = {

promptsource/templates/Muennighoff/mbpp/sanitized/templates.yaml

@@ -0,0 +1,32 @@
+dataset: Muennighoff/mbpp
+subset: sanitized
+templates:
+  4b108b1c-7514-488f-99ed-3ca5da70e103: !Template
+    answer_choices: null
+    id: 4b108b1c-7514-488f-99ed-3ca5da70e103
+    jinja: '{{ prompt }}
+      Here is a solution in Python:
+      |||
+      {{ code }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - en
+      metrics:
+      - Other
+      original_task: true
+    name: function solution
+    reference: ''
+  9d85c898-70fe-4a51-be37-5111be357762: !Template
+    answer_choices: null
+    id: 9d85c898-70fe-4a51-be37-5111be357762
+    jinja: "{{ prompt }} This can be solved in Python with the following code: |||{{ code }}"
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - en
+      metrics:
+      - Other
+      original_task: false
+    name: function solved
+    reference: ''

promptsource/templates/super_glue/copa/templates.yaml

@@ -22,21 +22,6 @@ templates:
       original_task: true
     name: exercise
     reference: ''
-  150789fe-e309-47a1-82c9-0a4dc2c6b12b: !Template
-    answer_choices: '{{choice1}} ||| {{choice2}}'
-    id: 150789fe-e309-47a1-82c9-0a4dc2c6b12b
-    jinja: "{% if question == \"effect\" %} \n{{ premise }} What could happen next,\
-      \ \"{{ answer_choices[0] }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label\
-      \ != -1 %}{{ answer_choices[label] }}{%endif%}\n{% endif %}"
-    metadata: !TemplateMetadata
-      choices_in_prompt: true
-      languages:
-      - en
-      metrics:
-      - Accuracy
-      original_task: true
-    name: "\u2026What could happen next, C1 or C2?"
-    reference: ''
   4d879cbe-2fd7-424a-9d78-3f5200313fba: !Template
     answer_choices: '{{choice1}} ||| {{choice2}}'
     id: 4d879cbe-2fd7-424a-9d78-3f5200313fba
@@ -88,21 +73,6 @@ templates:
       original_task: true
     name: "C1 or C2? premise, so/because\u2026"
     reference: "Adapted from Perez et al. 2021 and Schick & Sch\xFCtz 2021."
-  84da62c2-9440-4cfc-bdd4-d70c65e33a82: !Template
-    answer_choices: '{{choice1}} ||| {{choice2}}'
-    id: 84da62c2-9440-4cfc-bdd4-d70c65e33a82
-    jinja: "{% if question == \"effect\" %} \n{{ premise }} As a result, \"{{ answer_choices[0]\
-      \ }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label != -1 %}{{ answer_choices[label]\
-      \ }}{%endif%}\n{% endif %}"
-    metadata: !TemplateMetadata
-      choices_in_prompt: true
-      languages:
-      - en
-      metrics:
-      - Accuracy
-      original_task: true
-    name: "\u2026As a result, C1 or C2?"
-    reference: ''
   8ce80f8a-239e-4393-892c-f63dbb0d9929: !Template
     answer_choices: '{{choice1}} ||| {{choice2}}'
     id: 8ce80f8a-239e-4393-892c-f63dbb0d9929
@@ -118,21 +88,6 @@ templates:
       original_task: true
     name: best_option
     reference: ''
-  8cf2ba73-aee5-4651-b5d4-b1b88afe4abb: !Template
-    answer_choices: '{{choice1}} ||| {{choice2}}'
-    id: 8cf2ba73-aee5-4651-b5d4-b1b88afe4abb
-    jinja: "{% if question == \"cause\" %} \n{{ premise }} Which may be caused by\
-      \ \"{{ answer_choices[0] }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label\
-      \ != -1 %}{{ answer_choices[label] }}{%endif%}\n{% endif %}"
-    metadata: !TemplateMetadata
-      choices_in_prompt: true
-      languages:
-      - en
-      metrics:
-      - Accuracy
-      original_task: true
-    name: "\u2026which may be caused by"
-    reference: ''
   a1f9951e-2b6b-4530-9636-9cdf4c1658c5: !Template
     answer_choices: '{{choice1}} ||| {{choice2}}'
     id: a1f9951e-2b6b-4530-9636-9cdf4c1658c5
@@ -174,21 +129,6 @@ templates:
       original_task: true
     name: cause_effect
     reference: ''
-  a8bf11c3-bea2-45ba-a533-957d8bee5e2e: !Template
-    answer_choices: '{{choice1}} ||| {{choice2}}'
-    id: a8bf11c3-bea2-45ba-a533-957d8bee5e2e
-    jinja: "{% if question == \"cause\" %} \n{{ premise }} Why? \"{{ answer_choices[0]\
-      \ }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label != -1 %}{{ answer_choices[label]\
-      \ }}{%endif%}\n{% endif %}"
-    metadata: !TemplateMetadata
-      choices_in_prompt: true
-      languages:
-      - en
-      metrics:
-      - Accuracy
-      original_task: true
-    name: "\u2026why? C1 or C2"
-    reference: ''
   f32348cd-d3cb-4619-87b9-e24f99c78567: !Template
     answer_choices: '{{choice1}} ||| {{choice2}}'
     id: f32348cd-d3cb-4619-87b9-e24f99c78567