diff --git a/.github/workflows/ci-linux.yml b/.github/workflows/ci-linux.yml index 05185562b..eaaa07515 100644 --- a/.github/workflows/ci-linux.yml +++ b/.github/workflows/ci-linux.yml @@ -35,7 +35,7 @@ jobs: with: name: artifacts-${{ matrix.os }} path: | - test-resources/screenshots/*.png + /tmp/test-resources/screenshots/ retention-days: 2 - name: Upload screenshots uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 #v4 @@ -103,7 +103,7 @@ jobs: with: name: artifacts-${{ matrix.os }} path: | - test-resources/screenshots/*.png + /tmp/test-resources/screenshots/*.png retention-days: 2 - name: Upload screenshots uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 #v4 diff --git a/package.json b/package.json index 92310210d..d5936466e 100644 --- a/package.json +++ b/package.json @@ -934,19 +934,20 @@ }, "scripts": { "vscode:prepublish": "npm run compile", - "compile": "tsc -p ./ && npm run copytestproject && npm run copymedia", + "compile": "tsc -p ./ && npm run copytestproject && npm run copye2etestproject && npm run copymedia", "lint": "eslint . --ext .ts --no-fix --max-warnings 0", "lint:fix": "npm run lint -- --fix", - "watch": "tsc -w -p ./ && npm run copytestproject", + "watch": "tsc -w -p ./ && npm run copytestproject && npm run copye2etestproject", "copytestproject": "copyfiles -u 2 \"src/resources/**/*\" out/test/ -E", + "copye2etestproject": "copyfiles -u 2 \"src/resources/**/*\" out/e2e/ -E", "copymedia": "copyfiles \"media/icons/*\" out/ -E", "configure-husky": "npx husky install && npx husky add .husky/pre-commit \"npx --no-install lint-staged\"", "test": "export TEST=true && npm run compile && extest setup-and-run './out/test/**/*test.js' -c 1.87.2 -i -r .", "win-test": "set TEST=true&& npm run compile && extest setup-and-run './out/test/**/*test.js' -c 1.87.2 -i -r .", "unit-test": "mocha --require ts-node/register './src/unit/**/*.test.ts'", "unit-coverage": "npx nyc npm run unit-test", - "test:ui-end-to-end": "export TEST=uiEndToEnd && npm run compile && extest setup-and-run './out/e2e/getScan.test.js' -c 1.87.2 -i -r .", - "win-test:ui-end-to-end": "set TEST=uiEndToEnd&& npm run compile && extest setup-and-run './out/e2e/getScan.test.js' -c 1.87.2 -i -r ." + "test:ui-end-to-end": "export TEST=uiEndToEnd && npm run compile && extest setup-and-run './out/e2e/*.test.js' -c 1.87.2 -i -r .", + "win-test:ui-end-to-end": "set TEST=uiEndToEnd&& npm run compile && extest setup-and-run './out/e2e/*.test.js' -c 1.87.2 -i -r ." }, "devDependencies": { "@istanbuljs/nyc-config-typescript": "^1.0.2", diff --git a/src/e2e/getScan.test.ts b/src/e2e/getScan.test.ts index e8d920475..fa5c9d7cb 100644 --- a/src/e2e/getScan.test.ts +++ b/src/e2e/getScan.test.ts @@ -5,7 +5,12 @@ import { VSBrowser, WebDriver, By, - WebView + WebView, + InputBox, + TextEditor, + BottomBarPanel, + MarkerType, + SettingsEditor } from "vscode-extension-tester"; import { expect } from "chai"; import { initialize } from "../test/utils/utils"; @@ -28,6 +33,10 @@ import { selectItem, } from "./utils/utils"; +import { constants } from "../utils/common/constants"; +import * as path from "path"; +import * as fsp from "fs/promises"; + // Load environment variables dotenv.config(); const CX_AUTHENTICATION_COMMAND = "ast-results.showAuth"; @@ -40,7 +49,7 @@ describe("Checkmarx VS Code Extension Tests", () => { let bench: Workbench; let driver: WebDriver; - it("Authentication: should authenticate using API key and verify button state", async function() { + it("Authentication: should authenticate using API key and verify button state", async function () { this.timeout(120000); console.log("Starting API key authentication test..."); bench = new Workbench(); @@ -64,7 +73,7 @@ describe("Checkmarx VS Code Extension Tests", () => { // Find and select the API key radio button option const radioButtons = await webview.findWebElements(By.css("input[type='radio']")); console.log(`Found ${radioButtons.length} radio buttons`); - + if (radioButtons.length >= 2) { const apiKeyRadio = radioButtons[1]; await apiKeyRadio.click(); @@ -87,11 +96,11 @@ describe("Checkmarx VS Code Extension Tests", () => { const state = await authButton.getAttribute("disabled"); return state !== "true"; }, 5000, "Auth button did not become enabled"); - + // Verify that the auth button is now enabled disabledAttr = await authButton.getAttribute("disabled"); expect(disabledAttr).to.not.equal("true", "Auth button should be enabled after API key entry"); - + // Click the auth button await authButton.click(); console.log("Clicked 'Sign in' button"); @@ -162,5 +171,63 @@ describe("Checkmarx VS Code Extension Tests", () => { expect(branch).is.not.undefined; }); }); - + + // describe("OSS Scanner E2E Integration", () => { + // before(async function () { + // this.timeout(60000); + // // Enable OSS realtime scanner in settings + // console.log("Enabling OSS scanner for E2E tests..."); + // const settingsEditor = await bench.openSettings(); + // const ossCheckbox = await settingsEditor.findSetting( + // constants.activateOssRealtimeScanner, + // constants.ossRealtimeScanner + // ); + // await ossCheckbox.setValue(true); + // console.log("OSS scanner enabled"); + + // // Close settings + // await new EditorView().closeAllEditors(); + // }); + + // it("should scan package.json and detect security issues", async function () { + // this.timeout(220000); + + // const packageJsonPath = path.join(__dirname, "menifastFiles", "package.json"); + + // await bench.executeCommand("workbench.view.explorer"); + // await sleep(2000); + + // const folderPath = path.join(__dirname, "menifastFiles"); + + // await bench.executeCommand("workbench.action.files.openFolder"); + // const folderInput = await InputBox.create(); + // await folderInput.setText(folderPath); + // await folderInput.confirm(); + // await sleep(3000); + + // await bench.executeCommand("workbench.action.files.openFile"); + // const input = await InputBox.create(); + // await input.setText(packageJsonPath); + // await input.confirm(); + + // await sleep(5000); + + // const editorView = new EditorView(); + // const editor = await editorView.openEditor("package.json") as TextEditor; + // expect(editor).to.not.be.undefined; + + // await sleep(15000); + // const bottomBar = new BottomBarPanel(); + // await bottomBar.toggle(true); + + // const problemsView = await bottomBar.openProblemsView(); + + // await sleep(25000); + + // const markers = await problemsView.getAllMarkers(MarkerType.Error); + // expect(markers.length).to.be.greaterThan(0, "Expected OSS scanner to find security issues"); + + // }); + // }); + }); diff --git a/src/e2e/ossRealtimeScanner.test.ts b/src/e2e/ossRealtimeScanner.test.ts new file mode 100644 index 000000000..67e127187 --- /dev/null +++ b/src/e2e/ossRealtimeScanner.test.ts @@ -0,0 +1,217 @@ +import dotenv from "dotenv"; +import { + InputBox, + VSBrowser, + WebDriver, + Workbench, + EditorView, + TextEditor, + BottomBarPanel, + MarkerType, + SettingsEditor +} from "vscode-extension-tester"; +import { expect } from "chai"; +import { initialize } from "../test/utils/utils"; +import { + CX_CLEAR, +} from "../test/utils/constants"; +import { + waitForElementToAppear, + waitForInputBoxToOpen, + selectItem, +} from "./utils/utils"; +import { constants } from "../utils/common/constants"; +import * as path from "path"; +import * as fsp from "fs/promises"; + +// Load environment variables +dotenv.config(); + +function sleep(ms: number) { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +describe("OSS Scanner E2E Tests", () => { + let bench: Workbench; + let driver: WebDriver; + let editorView: EditorView; + + before(async function () { + this.timeout(120000); + console.log("Starting OSS Scanner E2E tests setup..."); + bench = new Workbench(); + driver = VSBrowser.instance.driver; + editorView = new EditorView(); + + // Enable OSS realtime scanner in settings + console.log("Opening settings to enable OSS scanner..."); + const settingsEditor = await bench.openSettings(); + const ossCheckbox = await settingsEditor.findSetting( + constants.activateOssRealtimeScanner, + constants.ossRealtimeScanner + ); + await ossCheckbox.setValue(true); + console.log("OSS scanner enabled in settings"); + + // Close settings by closing all editors + await editorView.closeAllEditors(); + + await bench.executeCommand("workbench.view.explorer"); + await sleep(2000); + + const folderPath = path.join(__dirname, "menifastFiles"); + + await bench.executeCommand("workbench.action.files.openFolder"); + const folderInput = await InputBox.create(); + await folderInput.setText(folderPath); + await folderInput.confirm(); + await sleep(3000); + + + await initialize(); + console.log("OSS Scanner E2E tests setup completed"); + }); + + // after(async () => { + // console.log("Cleaning up OSS Scanner E2E tests..."); + // await bench.executeCommand(CX_CLEAR); + // await editorView.closeAllEditors(); + // }); + + describe("Real-time OSS Scanning E2E", () => { + it("should scan package.json file on open and show security diagnostics", async function () { + this.timeout(220000); + + const packageJsonPath = path.join(__dirname, "menifastFiles", "package.json"); + + await bench.executeCommand("workbench.action.files.openFile"); + const input = await InputBox.create(); + await input.setText(packageJsonPath); + await input.confirm(); + + await sleep(5000); + + const editorView = new EditorView(); + const editor = await editorView.openEditor("package.json") as TextEditor; + expect(editor).to.not.be.undefined; + + await sleep(15000); + const bottomBar = new BottomBarPanel(); + await bottomBar.toggle(true); + + const problemsView = await bottomBar.openProblemsView(); + + await sleep(25000); + + const markers = await problemsView.getAllMarkers(MarkerType.Error); + expect(markers.length).to.be.greaterThan(0, "Expected OSS scanner to find security issues"); + + const maliciousMarkers = ( + await Promise.all(markers.map(async (marker) => { + const text = await marker.getText(); + return text.includes("Malicious package detected") ? marker : null; + })) + ).filter(Boolean); + + const scaCriticalVulnerabilityMarkers = ( + await Promise.all(markers.map(async (marker) => { + const text = await marker.getText(); + return text.includes("Critical-risk package") + })) + ).filter(Boolean); + + const scaHighVulnerabilityMarkers = ( + await Promise.all(markers.map(async (marker) => { + const text = await marker.getText(); + return text.includes("High-risk package") + })) + ).filter(Boolean); + const scaMediumVulnerabilityMarkers = ( + await Promise.all(markers.map(async (marker) => { + const text = await marker.getText(); + return text.includes("Medium-risk package") + })) + ).filter(Boolean); + + expect(maliciousMarkers.length, "Expected to find malicious package markers").to.be.greaterThan(0); + expect(scaCriticalVulnerabilityMarkers.length, "Expected to find critical-risk package markers").to.be.greaterThan(0); + expect(scaHighVulnerabilityMarkers.length, "Expected to find high-risk package markers").to.be.greaterThan(0); + expect(scaMediumVulnerabilityMarkers.length, "Expected to find medium-risk package markers").to.be.greaterThan(0); + + }); + + + it("should scan file on content change and generate problems", async function () { + this.timeout(120000); + console.log("Starting dynamic content change scan test..."); + + const packageJsonPath = path.join(__dirname, "menifastFiles", "package.json"); + const originalContent = await fsp.readFile(packageJsonPath, "utf8"); + console.log("Original package.json content loaded"); + + await bench.executeCommand("workbench.action.files.openFile"); + const input = await InputBox.create(); + await input.setText(packageJsonPath); + await input.confirm(); + + const editor = await editorView.openEditor("package.json") as TextEditor; + expect(editor).to.not.be.undefined; + + const bottomBar = new BottomBarPanel(); + await bottomBar.toggle(true); + const problemsView = await bottomBar.openProblemsView(); + + try { + // Clear content to remove all issues + console.log("Clearing package.json content..."); + await editor.setText(`{}`); + await sleep(3000); + + let markers = await problemsView.getAllMarkers(MarkerType.Error); + console.log(`Markers after clearing content: ${markers.length}`); + expect(markers.length).to.equal(0, "Expected no markers with empty package.json"); + + // Restore original content with vulnerabilities + // console.log("Restoring original content with vulnerabilities..."); + // await editor.setText(originalContent); + // await sleep(8000); // Give more time for scanner to process + + // markers = await problemsView.getAllMarkers(MarkerType.Error); + // console.log(`Markers after restoring content: ${markers.length}`); + + // // Debug: Log marker texts + // const markerTexts = await Promise.all(markers.map(async (marker) => { + // return await marker.getText(); + // })); + // console.log("Marker texts after restore:", markerTexts); + + // expect(markers.length).to.be.greaterThan(0, "Expected markers to appear after restoring vulnerable content"); + + // console.log("Dynamic content change scan test completed successfully"); + } finally { + // Ensure we restore the original content + // await editor.setText(originalContent); + await sleep(1000); + } + }); + }); + + describe("OSS Scanner Settings Verification", () => { + // it("should verify OSS scanner is enabled in settings", async function () { + // this.timeout(60000); + // console.log("Verifying OSS scanner settings..."); + + // const settingsEditor = await bench.openSettings(); + // const ossCheckbox = await settingsEditor.findSetting( + // constants.activateOssRealtimeScanner, + // constants.ossRealtimeScanner + // ); + + // const isEnabled = await ossCheckbox.getValue(); + // expect(isEnabled).to.be.true; + // console.log("OSS scanner is properly enabled in settings"); + + // await editorView.closeAllEditors(); + // }); + }); +}); diff --git a/src/test/13.ossRealtimScanner.test.ts b/src/test/13.ossRealtimScanner.test.ts new file mode 100644 index 000000000..b2f70df2f --- /dev/null +++ b/src/test/13.ossRealtimScanner.test.ts @@ -0,0 +1,132 @@ +import { + InputBox, + VSBrowser, + WebDriver, + Workbench, + EditorView, + TextEditor, + BottomBarPanel, + MarkerType +} from "vscode-extension-tester"; +import { + CX_CLEAR, + VS_OPEN_FOLDER, +} from "./utils/constants"; +import { retryTest } from "./utils/utils"; +import { expect } from "chai"; +import * as path from "path"; +import * as fsp from "fs/promises"; +import { constants } from "../utils/common/constants"; + +describe("OSS Scanner Tests", () => { + let bench: Workbench; + let driver: WebDriver; + let editorView: EditorView; + + before(async function () { + this.timeout(100000); + bench = new Workbench(); + driver = VSBrowser.instance.driver; + editorView = new EditorView(); + + // Enable OSS realtime scanner in settings + const settingsEditor = await bench.openSettings(); + const ossCheckbox = await settingsEditor.findSetting( + constants.activateOssRealtimeScanner, + constants.ossRealtimeScanner + ); + await ossCheckbox.setValue(true); + console.log(ossCheckbox); + // Close settings by closing all editors + await editorView.closeAllEditors(); + + await bench.executeCommand(VS_OPEN_FOLDER); + this.timeout(30000); + }); + + after(async () => { + await bench.executeCommand(CX_CLEAR); + await editorView.closeAllEditors(); + }); + + describe("Real-time OSS Scanning", () => { + it("should scan package.json file on open and show malicious package diagnostics", retryTest(async function () { + + const packageJsonPath = path.join(__dirname, "menifastFiles", "package.json"); + await bench.executeCommand("workbench.action.files.openFile"); + const input = await InputBox.create(); + await input.setText(packageJsonPath); + await input.confirm(); + + await driver.sleep(3000); + + const editor = await editorView.openEditor("package.json") as TextEditor; + expect(editor).to.not.be.undefined; + + const bottomBar = new BottomBarPanel(); + await bottomBar.toggle(true); + const problemsView = await bottomBar.openProblemsView(); + + await driver.sleep(2000); + + const markers = await problemsView.getAllMarkers(MarkerType.Error); + expect(markers.length).to.be.greaterThan(0); + const maliciousMarkers = ( + await Promise.all(markers.map(async (marker) => { + const text = await marker.getText(); + return text.includes("Malicious package detected") ? marker : null; + })) + ).filter(Boolean); + + const scaVulnerabilityMarkers = ( + await Promise.all(markers.map(async (marker) => { + const text = await marker.getText(); + return text.includes("High-risk package") || text.includes("vulnerability detected") ? marker : null; + })) + ).filter(Boolean); + + expect(maliciousMarkers.length).to.be.greaterThan(0); + expect(scaVulnerabilityMarkers.length).to.be.greaterThan(0); + + })); + + it.skip("should scan file on content change and generate problems", retryTest(async function () { + const packageJsonPath = path.join(__dirname, "menifastFiles", "package.json"); + + const originalContent = await fsp.readFile(packageJsonPath, "utf8"); + + await bench.executeCommand("workbench.action.files.openFile"); + const input = await InputBox.create(); + await input.setText(packageJsonPath); + await input.confirm(); + + const editor = await editorView.openEditor("package.json") as TextEditor; + + await bench.executeCommand("workbench.actions.view.problems"); + + try { + await editor.setText(`{}`); + await driver.sleep(2000); + + const bottomBar = new BottomBarPanel(); + await bottomBar.toggle(true); + const problemsView = await bottomBar.openProblemsView(); + + await driver.sleep(2000); + + let markers = await problemsView.getAllMarkers(MarkerType.Error); + expect(markers.length).to.equal(0); + + await editor.setText(originalContent); + await driver.sleep(5000); + + markers = await problemsView.getAllMarkers(MarkerType.Error); + expect(markers.length).to.be.greaterThan(0); + + await driver.sleep(1000); + } finally { + await editor.setText(originalContent); + } + })); + }); +})