Enrich dependency alerts with Copilot compatibility assessments

.github/actionlint.yaml

@@ -0,0 +1,7 @@
+# actionlint v1.7.12 predates GitHub's Copilot Requests token permission.
+# Keep every other workflow check active, and remove this exception when
+# actionlint adds the permission to its schema.
+paths:
+  .github/workflows/critical-dependencies.yml:
+    ignore:
+      - '^unknown permission scope "copilot-requests"\.'

.github/copilot-instructions.md

@@ -67,6 +67,10 @@ they disagree with an old comment, issue, or generated summary.
 - Keep bot-generated suggestions concrete: name affected SkillView code paths,
   expected behavior, and a test that would detect a regression. Do not invent
   breaking changes from a version number alone.
+- For critical-dependency assessments, distinguish published release facts,
+  source-code inference, and observed test results. A release-note keyword or
+  passing help/flag contract test cannot establish full compatibility. Cite
+  upstream advisories and name any untested interactive or TUI paths.
 - If GitHub's automated Copilot review is enabled, apply these instructions to
   its comments too. A human maintainer retains the decision to merge releases
   and dependency updates.

.github/scripts/critical-dependencies.js

@@ -1,4 +1,5 @@
 const fs = require('node:fs');
+const { isOwnedAssessmentComment, extractAssessment } = require('./critical-dependency-assessment.js');
 
 const LABEL = 'critical-dependency';
 const TERMINAL_GUI_REPO = 'Terminal.Gui';
@@ -48,6 +49,41 @@ function suggestedChecks(notes, kind) {
     : '- Compare upstream changes with SkillView adapters and add focused tests for any affected behavior.';
 }
 
+function releaseHighlights(notes, kind) {
+  const paragraphs = (notes || '').split(/\n\s*\n/)
+    // Release notes often put several bullets in one Markdown paragraph.
+    // Keep each bullet separate so an unrelated neighbor is not called a
+    // SkillView-relevant change.
+    .flatMap(item => item.split(/(?=^\s*[-*]\s+)/m))
+    .map(item => item.trim()).filter(Boolean);
+  const pattern = kind === 'gh'
+    ? /\bgh skills?\b|\bskill (?:search|install|update|list|preview)\b/i
+    : /terminal\.gui|keyboard|input|layout|scroll|render|thread|cancel|aot|trim/i;
+  const matches = [];
+  for (let index = 0; index < paragraphs.length && matches.length < 5; index++) {
+    if (!pattern.test(paragraphs[index])) continue;
+    let detail = paragraphs[index];
+    if (/^See https:\/\/github\.com\//.test(paragraphs[index + 1] || '')) {
+      detail += `\n${paragraphs[++index]}`;
+    }
+    matches.push(detail.slice(0, 1000));
+  }
+  return matches.length
+    ? matches.map(item => `> ${item.replace(/\n/g, '\n> ')}`).join('\n\n')
+    : 'No directly relevant entry was found in the published release notes; inspect the upstream diff.';
+}
+
+function releaseOverview(notes) {
+  const security = /## Security\b/i.test(notes || '')
+    ? '- Security section present; review all advisories in the linked upstream notes.'
+    : '- No security section identified in the published notes.';
+  const changed = (notes || '').split('\n')
+    .filter(line => /^\s*[-*]\s+/.test(line) && !/chore\(deps\)|@dependabot/i.test(line))
+    .slice(0, 8)
+    .map(line => line.trim().replace(/@(?=[A-Za-z0-9-]+)/g, ''));
+  return [security, ...changed].join('\n');
+}
+
 async function stableNugetVersion(packageName, fetchImpl) {
   const response = await fetchImpl(`https://api.nuget.org/v3-flatcontainer/${packageName.toLowerCase()}/index.json`, {
     signal: AbortSignal.timeout(15000),
@@ -76,17 +112,27 @@ async function createOnce(github, core, owner, repo, title, body) {
   const issues = await github.paginate(github.rest.issues.listForRepo, {
     owner, repo, labels: LABEL, state: 'all', per_page: 100,
   });
-  if (issues.some(issue => !issue.pull_request && issue.title === title)) {
+  const existing = issues.find(issue => !issue.pull_request && issue.title === title);
+  if (existing) {
     core.info(`Already tracked: ${title}`);
-    return;
+    const comments = await github.paginate(github.rest.issues.listComments, {
+      owner, repo, issue_number: existing.number, per_page: 100,
+    });
+    // A prior monitor run can create an issue, then fail during a later
+    // dependency check before the assessment matrix is emitted. Retry that
+    // issue until a validated assessment comment is actually published.
+    return comments.some(comment => isOwnedAssessmentComment(comment) &&
+      extractAssessment(comment.body) !== null)
+      ? null : existing.number;
   }
   const { data: issue } = await github.rest.issues.create({
     owner, repo, title, body, labels: [LABEL], assignees: [owner],
   });
   core.info(`Created ${issue.html_url}`);
+  return issue.number;
 }
 
-async function checkTerminalGui(github, core, owner, repo, project, fetchImpl) {
+async function checkTerminalGui(github, core, owner, repo, project, fetchImpl, newIssues) {
   const packages = [
     ['Terminal.Gui', propertyVersion(project, 'TerminalGuiVersion')],
     ['Terminal.Gui.Editor', propertyVersion(project, 'TerminalGuiEditorVersion')],
@@ -102,7 +148,7 @@ async function checkTerminalGui(github, core, owner, repo, project, fetchImpl) {
     }
     const release = releases.find(item => !item.draft && item.tag_name.toLowerCase() === `v${latest}`);
     const notes = release?.body || '';
-    await createOnce(github, core, owner, repo,
+    const number = await createOnce(github, core, owner, repo,
       `${packageName} ${latest} compatibility review`, `
 SkillView pins **${packageName} ${current}**; NuGet now has **${latest}**.
 
@@ -111,33 +157,70 @@ SkillView pins **${packageName} ${current}**; NuGet now has **${latest}**.
 - Find the Dependabot PR and review both Terminal.Gui packages together when appropriate.
 - Run locked tests and all four Native AOT publishes; check the extension's remaining trim suppressions.
 - Exercise keyboard selection, scrolling, resizing, install dialogs, and shutdown in a real terminal.
-- Ask Copilot to review the update PR using \`.github/copilot-instructions.md\` and propose focused compatibility tests.
+
+### Release overview
+${releaseOverview(notes)}
+
+### Potentially relevant release notes
+${releaseHighlights(notes, 'terminal-gui')}
+
+### Compatibility status
+**Needs verification.** This alert does not claim the new version is compatible or breaking. Copilot will add a separate evidence-based assessment; validate it with tests and human review.
 
 Suggested checks from release notes:
 ${suggestedChecks(notes, 'terminal-gui')}
 `.trim());
+    if (number) newIssues.push({ number, kind: 'terminal-gui', version: latest });
   }
 }
 
-async function checkGitHubCli(github, core, owner, repo, locator) {
+async function checkGitHubCli(github, core, owner, repo, locator, newIssues) {
   const minimum = minimumGhVersion(locator);
   const { data: release } = await github.rest.repos.getLatestRelease({ owner: 'cli', repo: 'cli' });
   if (release.draft || release.prerelease || !versionParts(release.tag_name)) {
     throw new Error(`Unexpected GitHub CLI latest release: ${release.tag_name}`);
   }
-  await createOnce(github, core, owner, repo,
+  const number = await createOnce(github, core, owner, repo,
     `GitHub CLI ${release.tag_name} compatibility review`, `
 [GitHub CLI ${release.tag_name}](${release.html_url}) is available. SkillView currently requires **gh ${minimum}+**.
 
+- [Upstream release notes](${release.html_url})
 - Run the required contract tests against gh ${minimum} and ${release.tag_name.slice(1)}.
 - Compare \`gh skill --help\`, search/preview/install/update/list flags, and JSON output with SkillView's adapters.
 - Diff \`gh skill install --help\` agent selectors against \`InstallAgentCatalog\` and its tests.
 - Check extension launch, \`GH_PATH\`, authentication, install defaults, and any release-note changes to \`gh skill\`.
-- Update the minimum only when a needed behavior requires it. Ask Copilot to propose focused tests or fixes; keep changes under human review.
+- Update the minimum only when a needed behavior requires it.
+
+### Release overview
+${releaseOverview(release.body)}
+
+### Skill-related release notes
+${releaseHighlights(release.body, 'gh')}
+
+### Compatibility status
+**Needs verification.** The scheduled contract tests cover command shape and key flags, not every interactive search/install path. Copilot will add a separate evidence-based assessment; validate it with tests and human review.
 
 Suggested checks from release notes:
 ${suggestedChecks(release.body, 'gh')}
 `.trim());
+  if (number) newIssues.push({ number, kind: 'gh', version: release.tag_name });
+}
+
+async function requestedReassessment(github, context, owner, repo) {
+  const raw = context.eventName === 'workflow_dispatch' && context.payload?.inputs?.issue_number;
+  if (!raw) return null;
+  if (!/^[1-9]\d*$/.test(raw)) throw new Error('issue_number must be a positive issue number');
+  const number = Number(raw);
+  if (!Number.isSafeInteger(number)) throw new Error('issue_number is too large');
+  const { data: issue } = await github.rest.issues.get({ owner, repo, issue_number: number });
+  if (issue.pull_request || !issue.labels?.some(label => label.name === LABEL)) {
+    throw new Error(`Issue #${number} is not a critical-dependency issue`);
+  }
+  const gh = /^GitHub CLI (v\d+\.\d+\.\d+) compatibility review$/.exec(issue.title);
+  if (gh) return { number, kind: 'gh', version: gh[1] };
+  const gui = /^Terminal\.Gui(?:\.Editor)? (\d+\.\d+\.\d+) compatibility review$/.exec(issue.title);
+  if (gui) return { number, kind: 'terminal-gui', version: gui[1] };
+  throw new Error(`Issue #${number} has an unexpected critical-dependency title`);
 }
 
 module.exports = async ({
@@ -146,12 +229,21 @@ module.exports = async ({
   locator = fs.readFileSync('src/SkillView.Core/Gh/GhBinaryLocator.cs', 'utf8'),
 }) => {
   const { owner, repo } = context.repo;
+  const newIssues = [];
   await ensureLabel(github, owner, repo);
-  await checkTerminalGui(github, core, owner, repo, project, fetchImpl);
-  await checkGitHubCli(github, core, owner, repo, locator);
+  await checkTerminalGui(github, core, owner, repo, project, fetchImpl, newIssues);
+  await checkGitHubCli(github, core, owner, repo, locator, newIssues);
+  const reassessment = await requestedReassessment(github, context, owner, repo);
+  if (reassessment && !newIssues.some(issue => issue.number === reassessment.number)) {
+    newIssues.push(reassessment);
+  }
+  core.setOutput?.('new-issues', JSON.stringify(newIssues));
 };
 
 module.exports.compareVersions = compareVersions;
 module.exports.propertyVersion = propertyVersion;
 module.exports.minimumGhVersion = minimumGhVersion;
 module.exports.suggestedChecks = suggestedChecks;
+module.exports.releaseHighlights = releaseHighlights;
+module.exports.releaseOverview = releaseOverview;
+module.exports.requestedReassessment = requestedReassessment;

.github/scripts/critical-dependencies.test.js

@@ -18,18 +18,31 @@ test('reads package properties and the enforced GitHub CLI minimum', () => {
 
 test('creates assigned, deduplicated issues with useful checks for new releases', async () => {
   const issues = [];
+  const comments = new Map();
+  const outputs = [];
+  const completeAssessment = `<!-- skillview-copilot-assessment-v1 -->
+### What changed
+The upstream release has a published security fix affecting interactive skill search.
+### SkillView impact
+SkillView's installer adapter passes repository selectors to the GitHub CLI process.
+### Compatibility assessment
+**Likely compatible.** Contract tests are still needed to verify the new release.
+### Focused follow-up
+Run the contract suite and inspect the installer argument boundary before closing.`;
   let labelExists = false;
   const github = {
-    paginate: async () => issues,
+    paginate: async (method, args) => (await method(args)).data,
     rest: {
       issues: {
         getLabel: async () => {
           if (!labelExists) throw Object.assign(new Error('missing label'), { status: 404 });
         },
         createLabel: async () => { labelExists = true; },
         listForRepo: async () => ({ data: issues }),
+        listComments: async ({ issue_number }) => ({ data: comments.get(issue_number) || [] }),
         create: async ({ title, body, labels, assignees }) => {
-          const issue = { title, body, labels, assignees, html_url: `https://example.invalid/${issues.length + 1}` };
+          const number = issues.length + 1;
+          const issue = { number, title, body, labels, assignees, html_url: `https://example.invalid/${number}` };
           issues.push(issue);
           return { data: issue };
         },
@@ -41,7 +54,7 @@ test('creates assigned, deduplicated issues with useful checks for new releases'
         }] }),
         getLatestRelease: async () => ({ data: {
           tag_name: 'v2.102.0', html_url: 'https://example.invalid/gh',
-          body: 'Improve gh skill JSON output.', draft: false, prerelease: false,
+          body: '## Security\n\nInteractive `gh skill search` fixed option injection.\n\nSee https://github.com/cli/cli/security/advisories/GHSA-qcwj-mr2r-2cx7\n\n## What\'s Changed\n\n* Fix auth handling', draft: false, prerelease: false,
         } }),
       },
     },
@@ -54,7 +67,7 @@ test('creates assigned, deduplicated issues with useful checks for new releases'
   });
   const args = {
     github, context: { repo: { owner: 'harder', repo: 'gh-skillview' } },
-    core: { info: () => {} }, project, locator, fetchImpl,
+    core: { info: () => {}, setOutput: (key, value) => outputs.push([key, JSON.parse(value)]) }, project, locator, fetchImpl,
   };
 
   await monitor(args);
@@ -65,9 +78,59 @@ test('creates assigned, deduplicated issues with useful checks for new releases'
   ]);
   assert.ok(issues.every(issue => issue.assignees[0] === 'harder' && issue.labels[0] === 'critical-dependency'));
   assert.match(issues[0].body, /keyboard shortcuts/);
-  assert.match(issues[1].body, /JSON inventory\/search output/);
+  assert.match(issues[1].body, /Interactive `gh skill search` fixed option injection/);
+  assert.match(issues[1].body, /GHSA-qcwj-mr2r-2cx7/);
+  assert.match(issues[1].body, /Needs verification/);
+  assert.deepEqual(outputs[0], ['new-issues', [
+    { number: 1, kind: 'terminal-gui', version: '2.5.1' },
+    { number: 2, kind: 'gh', version: 'v2.102.0' },
+  ]]);
   await monitor(args);
   assert.equal(issues.length, 2);
+  assert.deepEqual(outputs[1], ['new-issues', outputs[0][1]]);
+  comments.set(1, [{ user: { login: 'github-actions[bot]' }, body: '<!-- skillview-copilot-assessment-v1 -->\nMalformed assessment' }]);
+  comments.set(2, [{ user: { login: 'another-user' }, body: '<!-- skillview-copilot-assessment-v1 -->\nSpoofed marker' }]);
+  await monitor(args);
+  assert.deepEqual(outputs[2], ['new-issues', [
+    { number: 1, kind: 'terminal-gui', version: '2.5.1' },
+    { number: 2, kind: 'gh', version: 'v2.102.0' },
+  ]]);
+  comments.set(1, [{ user: { login: 'github-actions[bot]' }, body: completeAssessment }]);
+  comments.set(2, [{ user: { login: 'github-actions[bot]' }, body: completeAssessment }]);
+  await monitor(args);
+  assert.equal(issues.length, 2);
+  assert.deepEqual(outputs[3], ['new-issues', []]);
+});
+
+test('release summaries distinguish direct gh skill notes from unrelated skill content', () => {
+  const notes = '## Security\n\nInteractive `gh skill search` changed.\n\nSee https://example.invalid/advisory\n\n* Add a skill for recordings';
+  assert.match(monitor.releaseHighlights(notes, 'gh'), /gh skill search/);
+  assert.doesNotMatch(monitor.releaseHighlights(notes, 'gh'), /recordings/);
+  assert.match(monitor.releaseOverview(notes), /Security section present/);
+  assert.match(monitor.releaseHighlights('No CLI changes.', 'gh'), /No directly relevant entry/);
+});
+
+test('release highlights keep adjacent unrelated bullets out of skill excerpts', () => {
+  const notes = '## Changes\n\n* New repository skill content\n* Fix `gh skill search` option injection\n\nSee https://github.com/cli/cli/security/advisories/GHSA-qcwj-mr2r-2cx7\n\n* Update auth flow';
+  const highlight = monitor.releaseHighlights(notes, 'gh');
+  assert.match(highlight, /gh skill search/);
+  assert.match(highlight, /GHSA-qcwj-mr2r-2cx7/);
+  assert.doesNotMatch(highlight, /repository skill content|Update auth flow/);
+});
+
+test('manual reassessment accepts only labeled dependency issues with known titles', async () => {
+  const github = { rest: { issues: { get: async () => ({ data: {
+    title: 'GitHub CLI v2.102.0 compatibility review',
+    labels: [{ name: 'critical-dependency' }],
+  } }) } } };
+  const context = { eventName: 'workflow_dispatch', payload: { inputs: { issue_number: '31' } } };
+  assert.deepEqual(await monitor.requestedReassessment(github, context, 'harder', 'gh-skillview'),
+    { number: 31, kind: 'gh', version: 'v2.102.0' });
+  context.payload.inputs.issue_number = '31; echo unsafe';
+  await assert.rejects(() => monitor.requestedReassessment(github, context, 'harder', 'gh-skillview'), /positive issue number/);
+  context.payload.inputs.issue_number = '31';
+  github.rest.issues.get = async () => ({ data: { title: 'Other issue', labels: [] } });
+  await assert.rejects(() => monitor.requestedReassessment(github, context, 'harder', 'gh-skillview'), /not a critical-dependency issue/);
 });
 
 test('fails visibly when a critical version cannot be determined', async () => {

.github/scripts/critical-dependency-assessment.js

@@ -0,0 +1,53 @@
+const fs = require('node:fs');
+const ASSESSMENT_MARKER = '<!-- skillview-copilot-assessment-v1 -->';
+
+function isOwnedAssessmentComment(comment) {
+  return comment?.user?.login === 'github-actions[bot]' &&
+    comment.body?.includes(ASSESSMENT_MARKER) === true;
+}
+
+const headings = [
+  '### What changed',
+  '### SkillView impact',
+  '### Compatibility assessment',
+  '### Focused follow-up',
+];
+
+function extractAssessment(raw) {
+  if (typeof raw !== 'string') return null;
+  // Copilot's silent text mode can include a planning message before its
+  // final answer. Publish only the requested, complete assessment.
+  const start = raw.lastIndexOf(headings[0]);
+  if (start < 0) return null;
+  const assessment = raw.slice(start).trim();
+  if (assessment.length < 200 || assessment.length > 10000) return null;
+  if (assessment.split(/\s+/).length > 500) return null;
+  let previous = -1;
+  for (const heading of headings) {
+    const index = assessment.indexOf(heading);
+    if (index <= previous) return null;
+    previous = index;
+  }
+  const sections = headings.map((heading, index) => {
+    const from = assessment.indexOf(heading) + heading.length;
+    const to = index + 1 < headings.length
+      ? assessment.indexOf(headings[index + 1]) : assessment.length;
+    return assessment.slice(from, to).trim();
+  });
+  if (sections.some(section => section.length < 15)) return null;
+  const statuses = [...assessment.matchAll(/\*\*(Likely compatible|Potential break|Unknown)\.?\*\*/g)];
+  if (statuses.length !== 1 || !/^\*\*(Likely compatible|Potential break|Unknown)\.?\*\*/.test(sections[2])) {
+    return null;
+  }
+  return assessment;
+}
+
+if (require.main === module) {
+  const [input, output] = process.argv.slice(2);
+  if (!input || !output) throw new Error('Expected input and output file paths');
+  const assessment = extractAssessment(fs.readFileSync(input, 'utf8'));
+  if (!assessment) throw new Error('Copilot did not produce a complete assessment');
+  fs.writeFileSync(output, `${assessment}\n`);
+}
+
+module.exports = { ASSESSMENT_MARKER, isOwnedAssessmentComment, extractAssessment };