feat: validate agent issue generator and canonicalize labels (#37)

- .github/labels.yml: add type:agent-task, role:*, status:open-claim,
  good-first-agent-task as canonical labels (single source of truth)
- validate-repo.mjs: validateAgentIssueGenerator() asserts every task
  produces a well-formed title, >=4 labels all defined in labels.yml,
  and a body containing the done condition + problem link
- generate-agent-issues.mjs: export builders; guard CLI for safe import

Closes the board/labels drift gap noted in the prior PR.

Author: cschanhniem

.github/labels.yml

@@ -22,6 +22,30 @@
 - name: "type:safety"
   color: "B60205"
   description: "Safety flag or high-risk misuse concern."
+- name: "type:agent-task"
+  color: "1D76DB"
+  description: "Scoped task published as an issue for an AI agent to pick up."
+- name: "good-first-agent-task"
+  color: "7057FF"
+  description: "Low-risk entry point for a new AI agent contributor."
+- name: "role:literature-scout"
+  color: "5319E7"
+  description: "Source inventory and evidence classification."
+- name: "role:data-cleaner"
+  color: "5319E7"
+  description: "Dataset provenance and cleaning."
+- name: "role:implementation-planner"
+  color: "5319E7"
+  description: "Task decomposition and back-test specs."
+- name: "role:red-team-reviewer"
+  color: "B60205"
+  description: "Failure mode and misuse analysis."
+- name: "role:field-reality-reviewer"
+  color: "0052CC"
+  description: "Operational relevance check."
+- name: "status:open-claim"
+  color: "C5DEF5"
+  description: "Unclaimed agent task, available to pick up."
 - name: "status:needs-triage"
   color: "C5DEF5"
   description: "Needs maintainer triage before work starts."

scripts/generate-agent-issues.mjs

@@ -19,7 +19,7 @@ import { dirname, join } from "node:path";
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const ROOT = join(__dirname, "..");
 
-function loadTasks() {
+export function loadTasks() {
   const raw = readFileSync(join(ROOT, "tasks-available.json"), "utf8");
   return JSON.parse(raw).tasks;
 }
@@ -34,11 +34,11 @@ const ROLE_GUIDE = {
 
 const REPO = "https://github.com/Open-Problem-Lab/open-problem-lab/blob/main";
 
-function issueTitle(task) {
+export function issueTitle(task) {
   return `[AGENT-TASK] ${task.pack_title}: ${task.title}`;
 }
 
-function issueLabels(task) {
+export function issueLabels(task) {
   const labels = [
     "type:agent-task",
     "status:open-claim",
@@ -49,7 +49,7 @@ function issueLabels(task) {
   return labels;
 }
 
-function issueBody(task) {
+export function issueBody(task) {
   const roleGuide = ROLE_GUIDE[task.owner_role] || "AGENTS.md";
   const domains = task.domain.join(", ");
   const regions = (task.region || []).join(", ") || "global";
@@ -95,7 +95,7 @@ This is one scoped step inside a neglected global problem. The repo's scarce res
 🤖 Generated from \`tasks-available.json\` by \`scripts/generate-agent-issues.mjs\`. Source of truth is the pack's \`tasks.json\`.`;
 }
 
-function diversePicks(tasks, n) {
+export function diversePicks(tasks, n) {
   const perDomain = Math.max(1, Math.ceil(n / 11));
   const cap = {};
   const picks = [];
@@ -115,25 +115,30 @@ function diversePicks(tasks, n) {
   return picks;
 }
 
-const args = process.argv.slice(2);
-const tasks = loadTasks();
-
-if (args[0] === "--list") {
-  for (const t of tasks) console.log(`${t.pack_id}\t${t.task_id}`);
-} else if (args[0] === "--body") {
-  const t = tasks.find((x) => x.pack_id === args[1]);
-  if (!t) {
-    console.error(`No task for pack_id ${args[1]}`);
+// Only run the CLI when executed directly, not when imported (e.g. by the validator).
+if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
+  const args = process.argv.slice(2);
+  const tasks = loadTasks();
+
+  if (args[0] === "--list") {
+    for (const t of tasks) console.log(`${t.pack_id}\t${t.task_id}`);
+  } else if (args[0] === "--body") {
+    const t = tasks.find((x) => x.pack_id === args[1]);
+    if (!t) {
+      console.error(`No task for pack_id ${args[1]}`);
+      process.exit(1);
+    }
+    console.log(`TITLE\t${issueTitle(t)}`);
+    console.log(`LABELS\t${issueLabels(t).join(",")}`);
+    console.log("BODY");
+    console.log(issueBody(t));
+  } else if (args[0] === "--plan") {
+    const n = Number(args[1]) || 24;
+    for (const t of diversePicks(tasks, n)) console.log(t.pack_id);
+  } else {
+    console.error(
+      "Usage: generate-agent-issues.mjs --list | --body <pack_id> | --plan [n]"
+    );
     process.exit(1);
   }
-  console.log(`TITLE\t${issueTitle(t)}`);
-  console.log(`LABELS\t${issueLabels(t).join(",")}`);
-  console.log("BODY");
-  console.log(issueBody(t));
-} else if (args[0] === "--plan") {
-  const n = Number(args[1]) || 24;
-  for (const t of diversePicks(tasks, n)) console.log(t.pack_id);
-} else {
-  console.error("Usage: generate-agent-issues.mjs --list | --body <pack_id> | --plan [n]");
-  process.exit(1);
 }

scripts/validate-repo.mjs

@@ -31,6 +31,14 @@ const STATIC_LABELS = [
   "type:playbook",
   "type:review",
   "type:safety",
+  "type:agent-task",
+  "good-first-agent-task",
+  "role:literature-scout",
+  "role:data-cleaner",
+  "role:implementation-planner",
+  "role:red-team-reviewer",
+  "role:field-reality-reviewer",
+  "status:open-claim",
   "status:needs-triage",
   "status:scoped",
   "status:agent-working",
@@ -195,6 +203,38 @@ const validateLabels = async () => {
   }
 };
 
+const validateAgentIssueGenerator = async () => {
+  const { loadTasks, issueTitle, issueLabels, issueBody } =
+    await import("./generate-agent-issues.mjs");
+  const labelsPath = path.join(root, ".github", "labels.yml");
+  const labelsContent = await fs.readFile(labelsPath, "utf8");
+  const knownLabels = new Set(
+    [...labelsContent.matchAll(/^\s*-\s+name:\s*"?([^"\n]+)"?\s*$/gm)].map((m) => m[1])
+  );
+  const tasks = loadTasks();
+  assert(tasks.length > 0, "Agent issue generator found no tasks in tasks-available.json.");
+  for (const task of tasks) {
+    const title = issueTitle(task);
+    assert(
+      title.startsWith("[AGENT-TASK] ") && title.length > 14,
+      `Agent issue title is empty or malformed for ${task.pack_id}.`
+    );
+    const labels = issueLabels(task);
+    assert(labels.length >= 4, `Agent issue for ${task.pack_id} has too few labels.`);
+    for (const label of labels) {
+      assert(
+        knownLabels.has(label),
+        `Agent issue for ${task.pack_id} uses label "${label}" not defined in .github/labels.yml.`
+      );
+    }
+    const body = issueBody(task);
+    assert(
+      body.includes(task.done_condition) && body.includes(task.problem_file),
+      `Agent issue body for ${task.pack_id} is missing its done condition or problem link.`
+    );
+  }
+};
+
 const validateMarkdown = async () => {
   const markdownFiles = await walkFiles(root, (file) => file.endsWith(".md"));
   assert(markdownFiles.length > 0, "Markdown documentation is required.");
@@ -285,6 +325,7 @@ const main = async () => {
   await validateExamples(schemas);
   await validateIssueTemplates();
   await validateLabels();
+  await validateAgentIssueGenerator();
   await validateMarkdown();
   await validateWikiFreshness();
   await validateTaskIndexFreshness();