Adds integration tests

.circleci/config.yml

@@ -19,4 +19,6 @@ jobs:
                 command: yarn test
             - run:
                 name: Integration test
-                command: yarn test:integration
+                command: yarn test:integration
+                environment:
+                  CI: 'null'

.eslintrc.js

@@ -30,7 +30,7 @@ module.exports = {
     },
     overrides: [
         {
-            files: ['config.ts', 'test/integration/*.ts'],
+            files: ['config.ts', 'test/**/*.ts'],
             rules: {
                 '@typescript-eslint/no-var-requires': 'off',
             },

jest.config.integration.js

@@ -1,11 +1,12 @@
 const sharedConfig = require('./jest.config');
 
 module.exports = {
-    ...sharedConfig,
-    roots: ['test/integration'],
-    testPathIgnorePatterns: [/* Reset ignores set by shared config */],
-    setupFilesAfterEnv: [
-        ...sharedConfig.setupFilesAfterEnv,
-        './test/integration/jest.setup.integration.ts'
-    ],
+  ...sharedConfig,
+  roots: ['test/integration'],
+  testPathIgnorePatterns: [/* Reset ignores set by shared config */],
+  setupFiles: ['./test/integration/jest.env.setup.integration.ts'],
+  setupFilesAfterEnv: [
+    ...sharedConfig.setupFilesAfterEnv,
+    './test/integration/jest.setup.integration.ts',
+  ],
 };

lib/config/validator.ts

@@ -58,7 +58,10 @@ export default function constructAndValidateConfiguration(
         errors.push(`CI (${CI}) should be a boolean.`);
     } else {
         // Resolve CI from environment variables, if found. Fallback to configuration file.
-        config.CI = process.env.CI == null ? CI : process.env.CI === 'true';
+        config.CI =
+            process.env.CI == null || process.env.CI == 'null'
+                ? CI
+                : process.env.CI === 'true';
     }
 
     if (resultParser && !RESULT_PARSERS.includes(resultParser)) {

lib/file-client/index.ts

@@ -1,3 +1,8 @@
 export { getFiles } from './file-client';
-export { writeResults, getResults, clearResults } from './results-writer';
+export {
+    writeResults,
+    getResults,
+    clearResults,
+    RESULTS_LOCATION,
+} from './results-writer';
 export { CACHE_LOCATION } from './repository-client';

lib/file-client/results-writer.ts

@@ -10,7 +10,7 @@ import config from '@config';
 import { LintMessage } from '@engine/types';
 
 const RESULTS: string[] = [];
-const RESULTS_LOCATION = './eslint-remote-tester-results';
+export const RESULTS_LOCATION = './eslint-remote-tester-results';
 const RESULT_TEMPLATE = RESULT_PARSER_TO_TEMPLATE[config.resultParser];
 const RESULT_EXTENSION = RESULT_PARSER_TO_EXTENSION[config.resultParser];
 

package.json

@@ -21,7 +21,7 @@
         "start:ci": "CI=true yarn start",
         "start:memory-limit-crash": "NODE_OPTIONS=--max_old_space_size=10 node dist",
         "test": "jest",
-        "test:integration": "jest --config jest.config.integration.js",
+        "test:integration": "jest --config jest.config.integration.js --runInBand",
         "lint:react": "node dist --config eslint-remote-tester.react.config.js"
     },
     "repository": {

test/integration/integration.ci.test.ts

@@ -0,0 +1,120 @@
+import fs from 'fs';
+import {
+    getConsoleLogCalls,
+    runProductionBuild,
+    setConfig,
+    CI_CONFIGURATION_LOCATION,
+    INTEGRATION_REPO_OWNER,
+    INTEGRATION_REPO_NAME,
+} from '../utils';
+import { CACHE_LOCATION } from '@file-client';
+
+describe('CI mode', () => {
+    beforeEach(async () => {
+        setConfig(CI_CONFIGURATION_LOCATION);
+        await runProductionBuild();
+    });
+
+    test("validates repository's files", () => {
+        const finalLogCall = getConsoleLogCalls().pop();
+
+        expect(finalLogCall).toMatchInlineSnapshot(`
+            "Results:
+            Repository: AriPerkkio/eslint-remote-tester-integration-test-target
+            Rule: no-undef
+            Message: 'bar' is not defined.
+            Path: AriPerkkio/eslint-remote-tester-integration-test-target/index.js
+            Link: https://github.com/AriPerkkio/eslint-remote-tester-integration-test-target/blob/HEAD/index.js#L1-L1
+            var foo = bar;
+
+            if (foo) {
+            }
+
+            var p = {
+
+            Rule: no-empty
+            Message: Empty block statement.
+            Path: AriPerkkio/eslint-remote-tester-integration-test-target/index.js
+            Link: https://github.com/AriPerkkio/eslint-remote-tester-integration-test-target/blob/HEAD/index.js#L3-L4
+            var foo = bar;
+
+            if (foo) {
+            }
+
+            var p = {
+                get name(){
+                    // no returns.
+                }
+
+            Rule: getter-return
+            Message: Expected to return a value in getter 'name'.
+            Path: AriPerkkio/eslint-remote-tester-integration-test-target/index.js
+            Link: https://github.com/AriPerkkio/eslint-remote-tester-integration-test-target/blob/HEAD/index.js#L7-L7
+            if (foo) {
+            }
+
+            var p = {
+                get name(){
+                    // no returns.
+                }
+            };
+            p.getName();
+
+
+            Rule: no-compare-neg-zero
+            Message: Do not use the '===' operator to compare against -0.
+            Path: AriPerkkio/eslint-remote-tester-integration-test-target/index.js
+            Link: https://github.com/AriPerkkio/eslint-remote-tester-integration-test-target/blob/HEAD/index.js#L14-L14
+            };
+            p.getName();
+
+
+            if (foo === -0) {
+              // prevent no-empty
+            }
+            "
+        `);
+    });
+
+    test('excludes files matching exclude pattern', () => {
+        const finalLog = getConsoleLogCalls().pop();
+
+        expect(finalLog).not.toMatch('expected-to-be-excluded');
+    });
+
+    test('progress is logged', () => {
+        const repository = `${INTEGRATION_REPO_OWNER}/${INTEGRATION_REPO_NAME}`;
+
+        const [
+            startMessage,
+            cloneMessage,
+            readMessage,
+            lintStartMessage,
+            lintDoneMessage,
+            scanDoneMessage,
+        ] = getConsoleLogCalls();
+
+        expect(startMessage).toMatch(`[STARTING] ${repository}`);
+        expect(cloneMessage).toMatch(`[CLONING] ${repository}`);
+        expect(readMessage).toMatch(`[READING] ${repository}`);
+        expect(lintStartMessage).toMatch(`[LINTING] ${repository} - 1 files`);
+        expect(lintDoneMessage).toMatch(`[DONE] ${repository} 4 errors`);
+        expect(scanDoneMessage).toMatch(
+            `[DONE] Finished scan of 1 repositories`
+        );
+    });
+
+    test('repositories are cached', async () => {
+        const cachedRepository = `${CACHE_LOCATION}/${INTEGRATION_REPO_OWNER}/${INTEGRATION_REPO_NAME}`;
+
+        const [, cloneMessage] = getConsoleLogCalls();
+        expect(cloneMessage).toMatch('[CLONING]');
+        expect(fs.existsSync(cachedRepository)).toBe(true);
+
+        await runProductionBuild();
+
+        const [, pullMessage] = getConsoleLogCalls();
+        expect(pullMessage).toMatch('[PULLING]');
+        expect(fs.existsSync(cachedRepository)).toBe(true);
+    });
+});

test/integration/integration.cli.test.ts

@@ -0,0 +1,130 @@
+import fs from 'fs';
+import {
+    INTEGRATION_REPO_NAME,
+    runProductionBuild,
+    INTEGRATION_REPO_OWNER,
+    getStdoutWriteCalls,
+    getConsoleLogCalls,
+} from '../utils';
+import { RESULTS_LOCATION, CACHE_LOCATION } from '@file-client';
+
+describe('CLI', () => {
+    beforeEach(async () => {
+        await runProductionBuild();
+    });
+
+    test('creates results file', async () => {
+        const files = fs.readdirSync(RESULTS_LOCATION);
+        expect(files).toHaveLength(1);
+        expect(files[0]).toMatch(`${INTEGRATION_REPO_NAME}.md`);
+    });
+
+    test("validates repository's files", () => {
+        const results = fs.readFileSync(
+            `${RESULTS_LOCATION}/${INTEGRATION_REPO_NAME}.md`,
+            'utf8'
+        );
+
+        expect(results).toMatchInlineSnapshot(`
+                "## Rule: no-undef
+                - Message: \`'bar' is not defined.\`
+                - Path: \`AriPerkkio/eslint-remote-tester-integration-test-target/index.js\`
+                - [Link](https://github.com/AriPerkkio/eslint-remote-tester-integration-test-target/blob/HEAD/index.js#L1-L1)
+                \`\`\`js
+                var foo = bar;
+
+                if (foo) {
+                }
+
+                var p = {
+                \`\`\`
+
+                ## Rule: no-empty
+                - Message: \`Empty block statement.\`
+                - Path: \`AriPerkkio/eslint-remote-tester-integration-test-target/index.js\`
+                - [Link](https://github.com/AriPerkkio/eslint-remote-tester-integration-test-target/blob/HEAD/index.js#L3-L4)
+                \`\`\`js
+                var foo = bar;
+
+                if (foo) {
+                }
+
+                var p = {
+                    get name(){
+                        // no returns.
+                    }
+                \`\`\`
+
+                ## Rule: getter-return
+                - Message: \`Expected to return a value in getter 'name'.\`
+                - Path: \`AriPerkkio/eslint-remote-tester-integration-test-target/index.js\`
+                - [Link](https://github.com/AriPerkkio/eslint-remote-tester-integration-test-target/blob/HEAD/index.js#L7-L7)
+                \`\`\`js
+                if (foo) {
+                }
+
+                var p = {
+                    get name(){
+                        // no returns.
+                    }
+                };
+                p.getName();
+
+                \`\`\`
+
+                ## Rule: no-compare-neg-zero
+                - Message: \`Do not use the '===' operator to compare against -0.\`
+                - Path: \`AriPerkkio/eslint-remote-tester-integration-test-target/index.js\`
+                - [Link](https://github.com/AriPerkkio/eslint-remote-tester-integration-test-target/blob/HEAD/index.js#L14-L14)
+                \`\`\`js
+                };
+                p.getName();
+
+
+                if (foo === -0) {
+                  // prevent no-empty
+                }
+                \`\`\`
+                "
+            `);
+    });
+
+    test('excludes files matching exclude pattern', () => {
+        const results = fs.readFileSync(
+            `${RESULTS_LOCATION}/${INTEGRATION_REPO_NAME}.md`,
+            'utf8'
+        );
+
+        expect(results).not.toMatch('expected-to-be-excluded');
+    });
+
+    // TODO testing this is really tricky with current setup
+    // - process.stdout.write is called on intervals -> How to replace interval with direct calls
+    // - process.stdout.write is called with partial updates (log-diff.ts) -> How to validate partial rows
+    test.todo('progress is displayed on terminal');
+
+    test('final log is logged on terminal', () => {
+        const finalLog = getConsoleLogCalls().pop();
+
+        expect(finalLog).toMatchInlineSnapshot(`
+            "Full log:
+            [DONE] AriPerkkio/eslint-remote-tester-integration-test-target 4 errors
+            [DONE] Finished scan of 1 repositories"
+        `);
+    });
+
+    test('repositories are cached', async () => {
+        const cachedRepository = `${CACHE_LOCATION}/${INTEGRATION_REPO_OWNER}/${INTEGRATION_REPO_NAME}`;
+
+        const writes = getStdoutWriteCalls();
+        expect(writes.some(call => /CLONING/.test(call))).toBe(true);
+        expect(fs.existsSync(cachedRepository)).toBe(true);
+
+        await runProductionBuild();
+
+        const secondRunWrites = getStdoutWriteCalls();
+        expect(secondRunWrites.some(call => /CLONING/.test(call))).toBe(false);
+        expect(secondRunWrites.some(call => /PULLING/.test(call))).toBe(true);
+        expect(fs.existsSync(cachedRepository)).toBe(true);
+    });
+});

test/integration/jest.env.setup.integration.ts

@@ -0,0 +1,3 @@
+import { setConfig } from '../utils';
+
+setConfig();