Add local eslint rule to validate markdown codeblocks with React Compiler

In facebook/react#34462 for example, we found an issue where the compiler was incorrectly validating an example straight from the docs.

In order to find more issues like this + also provide more feedback to doc authors on valid/invalid patterns, this PR adds a new local eslint rule which validates all markdown codeblocks containing components/hooks with React Compiler. An autofixer is also provided.

To express that a codeblock has an expected error, we can use the following metadata:

```ts
// pseudo type def
type MarkdownCodeBlockMetadata = {
    expectedErrors?: {
      'react-compiler'?: number[];
    };
  };
```

and can be used like so:

````
```js {expectedErrors: {'react-compiler': [4]}}
// ❌ setState directly in render
function Component({value}) {
  const [count, setCount] = useState(0);
  setCount(value); // error on L4
  return <div>{count}</div>;
}
```
````

Because this is defined as a local rule, we don't have the same granular reporting that `eslint-plugin-react-hooks` yet. I can look into that later but for now this first PR just sets us up with something basic.

Author: poteto

.eslintrc

@@ -2,18 +2,38 @@
   "root": true,
   "extends": "next/core-web-vitals",
   "parser": "@typescript-eslint/parser",
-  "plugins": ["@typescript-eslint", "eslint-plugin-react-compiler"],
+  "plugins": ["@typescript-eslint", "eslint-plugin-react-compiler", "local-rules"],
   "rules": {
     "no-unused-vars": "off",
     "@typescript-eslint/no-unused-vars": ["error", {"varsIgnorePattern": "^_"}],
     "react-hooks/exhaustive-deps": "error",
     "react/no-unknown-property": ["error", {"ignore": ["meta"]}],
-    "react-compiler/react-compiler": "error"
+    "react-compiler/react-compiler": "error",
+    "local-rules/lint-markdown-code-blocks": "error"
   },
   "env": {
     "node": true,
     "commonjs": true,
     "browser": true,
     "es6": true
-  }
+  },
+  "overrides": [
+    {
+      "files": ["src/content/**/*.md"],
+      "processor": "local-rules/markdown",
+      "parser": "espree",
+      "parserOptions": {
+        "ecmaVersion": "latest",
+        "sourceType": "script"
+      },
+      "rules": {
+        "no-unused-vars": "off",
+        "@typescript-eslint/no-unused-vars": "off",
+        "react-hooks/exhaustive-deps": "off",
+        "react/no-unknown-property": "off",
+        "react-compiler/react-compiler": "off",
+        "local-rules/lint-markdown-code-blocks": "error"
+      }
+    }
+  ]
 }

eslint-local-rules/__tests__/fixtures/src/content/basic-error.md

@@ -0,0 +1,8 @@
+```jsx
+import {useState} from 'react';
+function Counter() {
+  const [count, setCount] = useState(0);
+  setCount(count + 1);
+  return <div>{count}</div>;
+}
+```

eslint-local-rules/__tests__/fixtures/src/content/duplicate-metadata.md

@@ -0,0 +1,8 @@
+```jsx title="Counter" {expectedErrors: {'react-compiler': [99]}} {expectedErrors: {'react-compiler': [2]}}
+import {useState} from 'react';
+function Counter() {
+  const [count, setCount] = useState(0);
+  setCount(count + 1);
+  return <div>{count}</div>;
+}
+```

eslint-local-rules/__tests__/fixtures/src/content/malformed-metadata.md

@@ -0,0 +1,8 @@
+```jsx {expectedErrors: {'react-compiler': 'invalid'}}
+import {useState} from 'react';
+function Counter() {
+  const [count, setCount] = useState(0);
+  setCount(count + 1);
+  return <div>{count}</div>;
+}
+```

eslint-local-rules/__tests__/fixtures/src/content/mixed-language.md

@@ -0,0 +1,7 @@
+```bash
+setCount()
+```
+
+```txt
+import {useState} from 'react';
+```

eslint-local-rules/__tests__/fixtures/src/content/stale-expected-error.md

@@ -0,0 +1,5 @@
+```jsx {expectedErrors: {'react-compiler': [3]}}
+function Hello() {
+  return <h1>Hello</h1>;
+}
+```

eslint-local-rules/__tests__/fixtures/src/content/suppressed-error.md

@@ -0,0 +1,8 @@
+```jsx {expectedErrors: {'react-compiler': [4]}}
+import {useState} from 'react';
+function Counter() {
+  const [count, setCount] = useState(0);
+  setCount(count + 1);
+  return <div>{count}</div>;
+}
+```

eslint-local-rules/__tests__/lint-markdown-code-blocks.test.js

@@ -0,0 +1,124 @@
+const assert = require('assert');
+const fs = require('fs');
+const path = require('path');
+const {ESLint} = require('eslint');
+const plugin = require('..');
+
+const FIXTURES_DIR = path.join(
+  __dirname,
+  'fixtures',
+  'src',
+  'content'
+);
+
+function createESLint({fix = false} = {}) {
+  return new ESLint({
+    useEslintrc: false,
+    fix,
+    plugins: {
+      'local-rules': plugin,
+    },
+    overrideConfig: {
+      processor: 'local-rules/markdown',
+      plugins: ['local-rules'],
+      rules: {
+        'local-rules/lint-markdown-code-blocks': 'error',
+      },
+      parserOptions: {
+        ecmaVersion: 2020,
+        sourceType: 'module',
+      },
+    },
+  });
+}
+
+function readFixture(name) {
+  return fs.readFileSync(path.join(FIXTURES_DIR, name), 'utf8');
+}
+
+async function lintFixture(name, {fix = false} = {}) {
+  const eslint = createESLint({fix});
+  const filePath = path.join(FIXTURES_DIR, name);
+  const markdown = readFixture(name);
+  const [result] = await eslint.lintText(markdown, {filePath});
+  return result;
+}
+
+async function run() {
+  const basicResult = await lintFixture('basic-error.md');
+  assert.strictEqual(
+    basicResult.messages.length,
+    1,
+    'expected one diagnostic'
+  );
+  assert(
+    basicResult.messages[0].message.includes('Calling setState during render'),
+    'expected message to mention setState during render'
+  );
+
+  const suppressedResult = await lintFixture('suppressed-error.md');
+  assert.strictEqual(
+    suppressedResult.messages.length,
+    0,
+    'expected suppression metadata to silence diagnostic'
+  );
+
+  const staleResult = await lintFixture('stale-expected-error.md');
+  assert.strictEqual(
+    staleResult.messages.length,
+    1,
+    'expected stale metadata error'
+  );
+  assert.strictEqual(
+    staleResult.messages[0].message,
+    'React Compiler expected error on line 3 was not triggered'
+  );
+
+  const duplicateResult = await lintFixture('duplicate-metadata.md');
+  assert.strictEqual(
+    duplicateResult.messages.length,
+    2,
+    'expected duplicate metadata to surface compiler diagnostic and stale metadata notice'
+  );
+  const duplicateFixed = await lintFixture('duplicate-metadata.md', {
+    fix: true,
+  });
+  assert(
+    duplicateFixed.output.includes(
+      "{expectedErrors: {'react-compiler': [4]}}"
+    ),
+    'expected duplicates to be rewritten to a single canonical block'
+  );
+  assert(
+    !duplicateFixed.output.includes('[99]'),
+    'expected stale line numbers to be removed from metadata'
+  );
+
+  const mixedLanguageResult = await lintFixture('mixed-language.md');
+  assert.strictEqual(
+    mixedLanguageResult.messages.length,
+    0,
+    'expected non-js code fences to be ignored'
+  );
+
+  const malformedResult = await lintFixture('malformed-metadata.md');
+  assert.strictEqual(
+    malformedResult.messages.length,
+    1,
+    'expected malformed metadata to fall back to compiler diagnostics'
+  );
+  const malformedFixed = await lintFixture('malformed-metadata.md', {
+    fix: true,
+  });
+  assert(
+    malformedFixed.output.includes(
+      "{expectedErrors: {'react-compiler': [4]}}"
+    ),
+    'expected malformed metadata to be replaced with canonical form'
+  );
+}
+
+run().catch(error => {
+  console.error(error);
+  process.exitCode = 1;
+});

eslint-local-rules/index.js

@@ -0,0 +1,47 @@
+const lintMarkdownCodeBlocks = require('./rules/lint-markdown-code-blocks');
+
+module.exports = {
+  rules: {
+    'lint-markdown-code-blocks': lintMarkdownCodeBlocks,
+  },
+  processors: {
+    markdown: {
+      preprocess(text) {
+        // Hack: wrap the markdown file in a dummy JS comment so ESLint feeds it
+        // through our custom rule without needing a real markdown parser here.
+        return [
+          `// Markdown content for processing\n/* ${text.replace(
+            /\*\//g,
+            '*\\/'
+          )} */`,
+        ];
+      },
+      postprocess(messages) {
+        const processedMessages = messages.flat().map((message) => {
+          if (message.suggestions) {
+            const processedSuggestions = message.suggestions.map(
+              (suggestion) => {
+                if (suggestion.fix) {
+                  // Unescape */ in fix text
+                  const fixedText = suggestion.fix.text.replace(
+                    /\*\\\//g,
+                    '*/'
+                  );
+                  return {
+                    ...suggestion,
+                    fix: {...suggestion.fix, text: fixedText},
+                  };
+                }
+                return suggestion;
+              }
+            );
+            return {...message, suggestions: processedSuggestions};
+          }
+          return message;
+        });
+        return processedMessages;
+      },
+      supportsAutofix: true,
+    },
+  },
+};

eslint-local-rules/package.json

@@ -0,0 +1,9 @@
+{
+  "name": "eslint-plugin-local-rules",
+  "version": "0.0.0",
+  "main": "index.js",
+  "private": "true",
+  "scripts": {
+    "test": "node __tests__/lint-markdown-code-blocks.test.js"
+  }
+}