Add local eslint rule to validate markdown codeblocks with React Compiler

In facebook/react#34462 for example, we found an issue where the compiler was incorrectly validating an example straight from the docs.

In order to find more issues like this + also provide more feedback to doc authors on valid/invalid patterns, this PR adds a new local eslint rule which validates all markdown codeblocks containing components/hooks with React Compiler. An autofixer is also provided.

To express that a codeblock has an expected error, we can use the following metadata:

```ts
// pseudo type def
type MarkdownCodeBlockMetadata = {
    expectedErrors?: {
      'react-compiler'?: number[];
    };
  };
```

and can be used like so:

````
```js {expectedErrors: {'react-compiler': [4]}}
// ❌ setState directly in render
function Component({value}) {
  const [count, setCount] = useState(0);
  setCount(value); // error on L4
  return <div>{count}</div>;
}
```
````

Because this is defined as a local rule, we don't have the same granular reporting that `eslint-plugin-react-hooks` yet. I can look into that later but for now this first PR just sets us up with something basic.

Author: poteto

.eslintrc

@@ -2,18 +2,36 @@
   "root": true,
   "extends": "next/core-web-vitals",
   "parser": "@typescript-eslint/parser",
-  "plugins": ["@typescript-eslint", "eslint-plugin-react-compiler"],
+  "plugins": ["@typescript-eslint", "eslint-plugin-react-compiler", "local-rules"],
   "rules": {
     "no-unused-vars": "off",
     "@typescript-eslint/no-unused-vars": ["error", {"varsIgnorePattern": "^_"}],
     "react-hooks/exhaustive-deps": "error",
     "react/no-unknown-property": ["error", {"ignore": ["meta"]}],
-    "react-compiler/react-compiler": "error"
+    "react-compiler/react-compiler": "error",
+    "local-rules/lint-markdown-code-blocks": "error"
   },
   "env": {
     "node": true,
     "commonjs": true,
     "browser": true,
     "es6": true
-  }
+  },
+  "overrides": [
+    {
+      "files": ["src/content/**/*.md"],
+      "parser": "./eslint-local-rules/parser",
+      "parserOptions": {
+        "sourceType": "module"
+      },
+      "rules": {
+        "no-unused-vars": "off",
+        "@typescript-eslint/no-unused-vars": "off",
+        "react-hooks/exhaustive-deps": "off",
+        "react/no-unknown-property": "off",
+        "react-compiler/react-compiler": "off",
+        "local-rules/lint-markdown-code-blocks": "error"
+      }
+    }
+  ]
 }

.github/workflows/site_lint.yml

@@ -32,6 +32,9 @@ jobs:
 
       - name: Install deps
         run: yarn install --frozen-lockfile
+      - name: Install deps (eslint-local-rules)
+        run: yarn install --frozen-lockfile
+        working-directory: eslint-local-rules
 
       - name: Lint codebase
         run: yarn ci-check

.gitignore

@@ -1,7 +1,7 @@
 # See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
 
 # dependencies
-/node_modules
+node_modules
 /.pnp
 .pnp.js
 

eslint-local-rules/__tests__/fixtures/src/content/basic-error.md

@@ -0,0 +1,8 @@
+```jsx
+import {useState} from 'react';
+function Counter() {
+  const [count, setCount] = useState(0);
+  setCount(count + 1);
+  return <div>{count}</div>;
+}
+```

eslint-local-rules/__tests__/fixtures/src/content/duplicate-metadata.md

@@ -0,0 +1,8 @@
+```jsx title="Counter" {expectedErrors: {'react-compiler': [99]}} {expectedErrors: {'react-compiler': [2]}}
+import {useState} from 'react';
+function Counter() {
+  const [count, setCount] = useState(0);
+  setCount(count + 1);
+  return <div>{count}</div>;
+}
+```

eslint-local-rules/__tests__/fixtures/src/content/malformed-metadata.md

@@ -0,0 +1,8 @@
+```jsx {expectedErrors: {'react-compiler': 'invalid'}}
+import {useState} from 'react';
+function Counter() {
+  const [count, setCount] = useState(0);
+  setCount(count + 1);
+  return <div>{count}</div>;
+}
+```

eslint-local-rules/__tests__/fixtures/src/content/mixed-language.md

@@ -0,0 +1,7 @@
+```bash
+setCount()
+```
+
+```txt
+import {useState} from 'react';
+```

eslint-local-rules/__tests__/fixtures/src/content/stale-expected-error.md

@@ -0,0 +1,5 @@
+```jsx {expectedErrors: {'react-compiler': [3]}}
+function Hello() {
+  return <h1>Hello</h1>;
+}
+```

eslint-local-rules/__tests__/fixtures/src/content/suppressed-error.md

@@ -0,0 +1,8 @@
+```jsx {expectedErrors: {'react-compiler': [4]}}
+import {useState} from 'react';
+function Counter() {
+  const [count, setCount] = useState(0);
+  setCount(count + 1);
+  return <div>{count}</div>;
+}
+```

eslint-local-rules/__tests__/lint-markdown-code-blocks.test.js

@@ -0,0 +1,131 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+const assert = require('assert');
+const fs = require('fs');
+const path = require('path');
+const {ESLint} = require('eslint');
+const plugin = require('..');
+
+const FIXTURES_DIR = path.join(
+  __dirname,
+  'fixtures',
+  'src',
+  'content'
+);
+const PARSER_PATH = path.join(__dirname, '..', 'parser.js');
+
+function createESLint({fix = false} = {}) {
+  return new ESLint({
+    useEslintrc: false,
+    fix,
+    plugins: {
+      'local-rules': plugin,
+    },
+    overrideConfig: {
+      parser: PARSER_PATH,
+      plugins: ['local-rules'],
+      rules: {
+        'local-rules/lint-markdown-code-blocks': 'error',
+      },
+      parserOptions: {
+        sourceType: 'module',
+      },
+    },
+  });
+}
+
+function readFixture(name) {
+  return fs.readFileSync(path.join(FIXTURES_DIR, name), 'utf8');
+}
+
+async function lintFixture(name, {fix = false} = {}) {
+  const eslint = createESLint({fix});
+  const filePath = path.join(FIXTURES_DIR, name);
+  const markdown = readFixture(name);
+  const [result] = await eslint.lintText(markdown, {filePath});
+  return result;
+}
+
+async function run() {
+  const basicResult = await lintFixture('basic-error.md');
+  assert.strictEqual(
+    basicResult.messages.length,
+    1,
+    'expected one diagnostic'
+  );
+  assert(
+    basicResult.messages[0].message.includes('Calling setState during render'),
+    'expected message to mention setState during render'
+  );
+
+  const suppressedResult = await lintFixture('suppressed-error.md');
+  assert.strictEqual(
+    suppressedResult.messages.length,
+    0,
+    'expected suppression metadata to silence diagnostic'
+  );
+
+  const staleResult = await lintFixture('stale-expected-error.md');
+  assert.strictEqual(
+    staleResult.messages.length,
+    1,
+    'expected stale metadata error'
+  );
+  assert.strictEqual(
+    staleResult.messages[0].message,
+    'React Compiler expected error on line 3 was not triggered'
+  );
+
+  const duplicateResult = await lintFixture('duplicate-metadata.md');
+  assert.strictEqual(
+    duplicateResult.messages.length,
+    2,
+    'expected duplicate metadata to surface compiler diagnostic and stale metadata notice'
+  );
+  const duplicateFixed = await lintFixture('duplicate-metadata.md', {
+    fix: true,
+  });
+  assert(
+    duplicateFixed.output.includes(
+      "{expectedErrors: {'react-compiler': [4]}}"
+    ),
+    'expected duplicates to be rewritten to a single canonical block'
+  );
+  assert(
+    !duplicateFixed.output.includes('[99]'),
+    'expected stale line numbers to be removed from metadata'
+  );
+
+  const mixedLanguageResult = await lintFixture('mixed-language.md');
+  assert.strictEqual(
+    mixedLanguageResult.messages.length,
+    0,
+    'expected non-js code fences to be ignored'
+  );
+
+  const malformedResult = await lintFixture('malformed-metadata.md');
+  assert.strictEqual(
+    malformedResult.messages.length,
+    1,
+    'expected malformed metadata to fall back to compiler diagnostics'
+  );
+  const malformedFixed = await lintFixture('malformed-metadata.md', {
+    fix: true,
+  });
+  assert(
+    malformedFixed.output.includes(
+      "{expectedErrors: {'react-compiler': [4]}}"
+    ),
+    'expected malformed metadata to be replaced with canonical form'
+  );
+}
+
+run().catch(error => {
+  console.error(error);
+  process.exitCode = 1;
+});