open-problem-lab/schemas/agent-submission.schema.json at main · Open-Problem-Lab/open-problem-lab · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://open-problem-lab.local/schemas/agent-submission.schema.json",
  "title": "Agent Submission",
  "description": "A structured submission from an AI agent or human contributor. One submission covers one task, one role, one claim. Submissions that mix roles or make multiple independent claims will be returned for splitting.",
  "type": "object",
  "additionalProperties": false,
  "required": [
    "task_id",
    "agent_role",
    "claim",
    "evidence",
    "method",
    "reproducibility_steps",
    "assumptions",
    "failure_modes",
    "kill_condition",
    "confidence",
    "suggested_next_issue"
  ],
  "properties": {
    "task_id": {
      "type": "string",
      "minLength": 3,
      "description": "The exact id field from tasks.json for the task this submission addresses. Example: source-inventory"
    },
    "agent_role": {
      "type": "string",
      "description": "The role guide used for this submission. Pick one and stay in it. See agents/ directory for each role's requirements and failure modes.",
      "enum": [
        "literature-scout",
        "data-cleaner",
        "red-team-reviewer",
        "field-reality-reviewer",
        "implementation-planner"
      ]
    },
    "claim": {
      "type": "string",
      "minLength": 20,
      "description": "One specific, falsifiable statement. State what can be proven, at what grain, with what evidence. A claim that cannot be shown false is not a claim. Do not include what the source cannot prove — put that in failure_modes or limitations."
    },
    "evidence": {
      "type": "array",
      "description": "List of evidence record IDs from evidence.json that support this claim. Every evidence ID here must exist in the relevant pack's evidence.json before this submission is merged.",
      "items": {
        "type": "string",
        "minLength": 3,
        "description": "An evidence record ID from evidence.json. Example: who-malaria-fact-sheet-2024"
      },
      "minItems": 1
    },
    "method": {
      "type": "string",
      "minLength": 10,
      "description": "How you derived the claim from the evidence. A reviewer should be able to repeat your reasoning and arrive at the same conclusion — or name the step where they disagree."
    },
    "reproducibility_steps": {
      "type": "array",
      "description": "Ordered steps a reviewer or replicator can follow to verify this submission independently. Each step should be specific enough to execute without asking the submitter for clarification.",
      "items": {
        "type": "string",
        "minLength": 5
      },
      "minItems": 1
    },
    "assumptions": {
      "type": "array",
      "description": "Conditions that must be true for the claim to hold. State them explicitly. A hidden assumption is a failure mode waiting to be found by a reviewer.",
      "items": {
        "type": "string",
        "minLength": 5
      },
      "minItems": 1
    },
    "failure_modes": {
      "type": "array",
      "description": "Specific ways this claim could be wrong, misleading, or harmful if applied. Include grain mismatches, causal claims that exceed the evidence, and plausible misuse scenarios. 'Results may vary' is not a failure mode.",
      "items": {
        "type": "string",
        "minLength": 5
      },
      "minItems": 1
    },
    "kill_condition": {
      "type": "string",
      "minLength": 10,
      "description": "The specific observation or experiment that would make this claim false. If no observation could falsify the claim, it is not a claim. This is not a disclaimer — it is the falsification criterion. SKILL.md pattern #3 requires this; the schema now enforces it."
    },
    "confidence": {
      "type": "string",
      "description": "Your confidence that the claim accurately represents what the evidence proves at the stated grain. Not confidence that the claim is true in the world. high = strong evidence, clear method, low risk of error. medium = adequate evidence with stated limitations. low = preliminary, needs more evidence or verification. unknown = cannot assess.",
      "enum": ["high", "medium", "low", "unknown"]
    },
    "suggested_next_issue": {
      "type": "string",
      "minLength": 5,
      "description": "The logical next step after this submission. Required field — state what should happen next even if no issue will be opened. Do not open a new GitHub issue unless the task explicitly asks for it. Example: 'Obtain district-level dengue surveillance data from NCDC to begin baseline construction.'"
    }
  }
}