trunk-merge/pr-108934/12827012-b562-462f-9ef9-75cf79b1f4ac

.github/scripts/check-snapshot-baseline-removals.mjs

@@ -0,0 +1,135 @@
+import { appendFileSync, readFileSync } from 'node:fs'
+import { posix } from 'node:path'
+import { pathToFileURL } from 'node:url'
+import { parseArgs } from 'node:util'
+
+const BASELINE_PATH = 'frontend/snapshots.yml'
+
+// Returns each snapshot id with its lines as written, so a failure can print them for restoring.
+// The YAML writer emits a key over 128 characters in the explicit `? key` form.
+export function readSnapshotEntries(yamlText) {
+    const entries = new Map()
+    let inSnapshots = false
+    let current = null
+    for (const line of yamlText.split('\n')) {
+        if (/^\s*(#|$)/.test(line)) {
+            continue
+        }
+        if (/^\S/.test(line)) {
+            inSnapshots = /^snapshots:\s*(#.*)?$/.test(line)
+            current = null
+            continue
+        }
+        if (!inSnapshots) {
+            continue
+        }
+        const key = /^ {4}(?:\? +(['"]?)([^\s'":?]+)\1|(['"]?)([^\s'":?]+)\3:)(?:\s+#.*)?\s*$/.exec(line)
+        if (key) {
+            current = [line]
+            entries.set(key[2] ?? key[4], current)
+        } else if (current) {
+            current.push(line)
+        }
+    }
+    return entries
+}
+
+// Storybook sanitizes a story id to exactly one `--` (title--export), and the test runner appends
+// `--<viewport width>`, `--<theme>` and `--<browser>` to it, so the first two segments name the story.
+export function storyIdOf(snapshotId) {
+    return snapshotId.split('--').slice(0, 2).join('--')
+}
+
+export function readSnapshottedStories(indexJsonText, storybookRoot) {
+    const entries = JSON.parse(indexJsonText).entries ?? {}
+    const stories = new Map()
+    for (const entry of Object.values(entries)) {
+        if (entry?.type !== 'story' || (entry.tags ?? []).includes('test-skip')) {
+            continue
+        }
+        stories.set(entry.id, entry.importPath ? posix.normalize(posix.join(storybookRoot, entry.importPath)) : null)
+    }
+    return stories
+}
+
+// A story the PR's Visual Review run renders shows a wrong removal there for review. Only stories
+// that run skips can lose every entry silently.
+export function findLiveRemovals({ baseIds, headIds, stories, renderedFiles }) {
+    const storiesWithHeadEntries = new Set([...headIds].map(storyIdOf))
+    const removals = []
+    for (const id of baseIds) {
+        if (headIds.has(id)) {
+            continue
+        }
+        const storyId = storyIdOf(id)
+        if (!stories.has(storyId) || storiesWithHeadEntries.has(storyId) || renderedFiles.has(stories.get(storyId))) {
+            continue
+        }
+        removals.push(id)
+    }
+    return removals.sort()
+}
+
+export function formatFailure(removals, baseEntries) {
+    const storyCount = new Set(removals.map(storyIdOf)).size
+    const stories = storyCount === 1 ? 'story that still exists' : 'stories that still exist'
+    return [
+        `This PR removes every ${BASELINE_PATH} entry of ${storyCount} ${stories}.`,
+        "This PR's Visual Review run does not render these stories, so it cannot show the removal.",
+        'The merge queue renders every story and rejects the batch that holds this PR.',
+        '',
+        `To fix it, add these lines back to ${BASELINE_PATH} under snapshots, in alphabetical order:`,
+        '',
+        ...removals.flatMap((id) => baseEntries.get(id)),
+        '',
+        "If a story should no longer take snapshots, tag it 'test-skip' in its story file and this check passes.",
+    ].join('\n')
+}
+
+function main() {
+    const { values } = parseArgs({
+        options: {
+            'base-snapshots': { type: 'string' },
+            'head-snapshots': { type: 'string' },
+            'storybook-index': { type: 'string' },
+            'storybook-root': { type: 'string', default: 'common/storybook' },
+            'rendered-files': { type: 'string' },
+        },
+    })
+    for (const required of ['base-snapshots', 'head-snapshots', 'storybook-index', 'rendered-files']) {
+        if (!values[required]) {
+            console.error(`Missing --${required}`)
+            process.exit(2)
+        }
+    }
+
+    const baseEntries = readSnapshotEntries(readFileSync(values['base-snapshots'], 'utf8'))
+    const removals = findLiveRemovals({
+        baseIds: new Set(baseEntries.keys()),
+        headIds: new Set(readSnapshotEntries(readFileSync(values['head-snapshots'], 'utf8')).keys()),
+        stories: readSnapshottedStories(readFileSync(values['storybook-index'], 'utf8'), values['storybook-root']),
+        renderedFiles: new Set(JSON.parse(readFileSync(values['rendered-files'], 'utf8'))),
+    })
+    if (removals.length === 0) {
+        console.log(`No ${BASELINE_PATH} entry was removed for a story that still takes snapshots.`)
+        return
+    }
+
+    const message = formatFailure(removals, baseEntries)
+    console.log(message)
+    if (process.env.GITHUB_STEP_SUMMARY) {
+        appendFileSync(
+            process.env.GITHUB_STEP_SUMMARY,
+            `### Snapshot baselines removed\n\n\`\`\`\n${message}\n\`\`\`\n`
+        )
+    }
+    if (process.env.GITHUB_ACTIONS) {
+        const annotation = message.replaceAll('%', '%25').replaceAll('\r', '%0D').replaceAll('\n', '%0A')
+        console.log(`::error title=Snapshot baselines removed for existing stories::${annotation}`)
+    }
+    process.exit(1)
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+    main()
+}

.github/scripts/check-snapshot-baseline-removals.test.mjs

@@ -0,0 +1,130 @@
+import assert from 'node:assert/strict'
+import test from 'node:test'
+
+import {
+    findLiveRemovals,
+    formatFailure,
+    readSnapshotEntries,
+    readSnapshottedStories,
+} from './check-snapshot-baseline-removals.mjs'
+
+const LONG_KEY_LENGTH = 60
+
+const entryLines = (id) => {
+    const comment = id.endsWith('--light') ? ' # approved' : ''
+    return id.length > LONG_KEY_LENGTH
+        ? [`    ? ${id}${comment}`, `    :   hash: v1.k1.${id.length}.fake`]
+        : [`    ${id}:${comment}`, `        hash: v1.k1.${id.length}.fake`]
+}
+
+const baselineYaml = (ids) =>
+    [
+        'version: 1',
+        'config:',
+        '    api: https://us.example.com',
+        "    team: '2'",
+        'snapshots: # approved baselines',
+        ...ids.flatMap(entryLines),
+        '# end of baselines',
+        '',
+    ].join('\n')
+
+const indexJson = (stories) =>
+    JSON.stringify({
+        v: 5,
+        entries: Object.fromEntries([
+            [
+                'settings-form--docs',
+                { type: 'docs', id: 'settings-form--docs', importPath: '../../frontend/src/SettingsForm.mdx' },
+            ],
+            ...stories.map(({ id, file, tags = ['dev', 'test'] }) => [
+                id,
+                { type: 'story', id, importPath: `../../${file}`, tags },
+            ]),
+        ]),
+    })
+
+const LONG_STORY = 'products-dashboards-widget-types-error-tracking-settings--before-ingestion'
+
+const BASE = [
+    `${LONG_STORY}--dark`,
+    `${LONG_STORY}--light`,
+    'settings-form--edited--dark',
+    'settings-form--edited--light',
+    'settings-form--empty--dark',
+    'settings-form--empty--light',
+    'widget-grid--default--narrow--dark',
+    'widget-grid--default--narrow--light',
+]
+
+const STORIES = [
+    { id: LONG_STORY, file: 'products/dashboards/frontend/ErrorTrackingSettings.stories.tsx' },
+    { id: 'settings-form--edited', file: 'frontend/src/SettingsForm.stories.tsx' },
+    { id: 'settings-form--empty', file: 'frontend/src/SettingsForm.stories.tsx' },
+    { id: 'widget-grid--default', file: 'products/widgets/frontend/WidgetGrid.stories.tsx' },
+]
+
+const cases = [
+    {
+        name: 'flags every entry of an existing story the PR run does not render',
+        head: BASE.filter((id) => !id.startsWith('settings-form--edited')),
+        stories: STORIES,
+        renderedFiles: [],
+        expected: ['settings-form--edited--dark', 'settings-form--edited--light'],
+    },
+    {
+        name: 'flags entries written with explicit keys',
+        head: BASE.filter((id) => !id.startsWith(LONG_STORY)),
+        stories: STORIES,
+        renderedFiles: [],
+        expected: [`${LONG_STORY}--dark`, `${LONG_STORY}--light`],
+    },
+    {
+        name: 'passes when the story was deleted or renamed',
+        head: BASE.filter((id) => !id.startsWith('widget-grid')),
+        stories: STORIES.filter((story) => story.id !== 'widget-grid--default'),
+        renderedFiles: [],
+        expected: [],
+    },
+    {
+        name: 'passes when the story keeps another variant',
+        head: BASE.filter((id) => id !== 'widget-grid--default--narrow--dark'),
+        stories: STORIES,
+        renderedFiles: [],
+        expected: [],
+    },
+    {
+        name: 'passes when the story is tagged test-skip',
+        head: BASE.filter((id) => !id.startsWith('widget-grid')),
+        stories: STORIES.map((story) =>
+            story.id === 'widget-grid--default' ? { ...story, tags: ['dev', 'test', 'test-skip'] } : story
+        ),
+        renderedFiles: [],
+        expected: [],
+    },
+    {
+        name: 'passes when the PR run renders the story',
+        head: BASE.filter((id) => !id.startsWith('widget-grid')),
+        stories: STORIES,
+        renderedFiles: ['products/widgets/frontend/WidgetGrid.stories.tsx'],
+        expected: [],
+    },
+]
+
+for (const { name, head, stories, renderedFiles, expected } of cases) {
+    test(name, () => {
+        const removals = findLiveRemovals({
+            baseIds: new Set(readSnapshotEntries(baselineYaml(BASE)).keys()),
+            headIds: new Set(readSnapshotEntries(baselineYaml(head)).keys()),
+            stories: readSnapshottedStories(indexJson(stories), 'common/storybook'),
+            renderedFiles: new Set(renderedFiles),
+        })
+        assert.deepEqual(removals, expected)
+    })
+}
+
+test('the failure prints the removed entries exactly as the base file wrote them', () => {
+    const removed = [`${LONG_STORY}--dark`, 'settings-form--edited--dark']
+    const message = formatFailure(removed, readSnapshotEntries(baselineYaml(BASE)))
+    assert.ok(message.includes(removed.flatMap(entryLines).join('\n')))
+})

.github/scripts/post-ci-sections.test.mjs

@@ -3,6 +3,7 @@ import { describe, it } from 'node:test'
 
 import { buildDocsPreviewSection } from './post-docs-preview-section.mjs'
 import { buildHobbySection } from './post-hobby-section.mjs'
+import { buildHogboxPreviewSection } from './post-hogbox-preview-section.mjs'
 import { buildTrunkLaneSection, postTrunkLaneSection } from './post-trunk-lane-section.mjs'
 
 const commonHobby = {
@@ -123,6 +124,36 @@ describe('CI report section builders', () => {
         assert.match(section.body, /actions\/runs\/42/)
     })
 
+    for (const testCase of [
+        {
+            name: 'claims the PR frontend only when it was swapped in',
+            frontendSwapped: true,
+            expected: /\*\*and\*\* frontend/,
+        },
+        {
+            name: 'says the frontend is unchanged when it was not swapped in',
+            frontendSwapped: false,
+            expected: /frontend unchanged by this PR/,
+        },
+    ]) {
+        it(testCase.name, () => {
+            const section = buildHogboxPreviewSection({
+                state: 'ready',
+                sha: '1234567890abcdef',
+                runUrl: 'https://github.com/PostHog/posthog/actions/runs/42',
+                url: 'https://preview.example.com',
+                boxId: 'box-1',
+                penId: 'None',
+                consoleHost: 'console.example.com',
+                frontendSwapped: testCase.frontendSwapped,
+            })
+            assert.equal(section.status, 'ok')
+            assert.match(section.summary, /https:\/\/preview\.example\.com/)
+            assert.match(section.body, testCase.expected)
+            assert.doesNotMatch(section.body, /console\/fleet\/pens/)
+        })
+    }
+
     it('moves a hobby preview through setup, ready, and failed states', () => {
         const initial = buildHobbySection({ state: 'initial', ...commonHobby })
         const ready = buildHobbySection({