From 3a64643a7a074bb1e6e8259aa31941a86fd3953f Mon Sep 17 00:00:00 2001 From: Jamie Danielson Date: Tue, 30 Jan 2024 14:22:56 -0500 Subject: [PATCH] maint: build out smoke tests and connect to CI (#55) ## Which problem is this PR solving? - Closes #17 ## Short description of the changes - Add smoke tests using BATS, similar to our other distros. This primarily allows for re-using smoke test logic independent of language, as it inspects and asserts against telemetry collected by the Collector and exported to a file. - Update CircleCI config to run smoke tests in every push, similar to lint/build/test - Add Makefile for easier smoke testing. This doesn't appear to be necessary for local smoke testing, as similar scripts in the `package.json` also work locally. CI didn't seem to work as well, hence the decision to keep the Makefile. - Update `DEVELOPING.md` with some details around smoke tests - Update example to send to collector at localhost, without API key. This may be the most "controversial" and may warrant having a different example app - open to ideas. If we keep an API key we have end up with CORS errors for a "With-Credentials" header. ## How to verify that this has the expected result See the [results in Circle](https://app.circleci.com/pipelines/github/honeycombio/honeycomb-opentelemetry-web/113/workflows/546edcba-4d22-4dd9-8fc3-114170373f0f/jobs/435/artifacts), including actual assertions like `Agent includes browser attributes` as well as telemetry output stored from the collector file export. --- .circleci/config.yml | 31 +++++++-- .gitignore | 1 + DEVELOPING.md | 18 +++-- Makefile | 36 ++++++++++ cypress.config.ts | 1 + cypress/e2e/spec.cy.ts | 40 ++++------- docker-compose.yml | 18 +++++ examples/hello-world-web/index.js | 4 +- package.json | 10 +-- smoke-tests/collector/data-results/.gitkeep | 0 .../collector/otel-collector-config.yaml | 24 +++++++ smoke-tests/smoke-e2e.bats | 61 ++++++++++++++++ smoke-tests/test_helpers/utilities.bash | 69 +++++++++++++++++++ 13 files changed, 272 insertions(+), 41 deletions(-) create mode 100644 Makefile create mode 100644 docker-compose.yml create mode 100644 smoke-tests/collector/data-results/.gitkeep create mode 100644 smoke-tests/collector/otel-collector-config.yaml create mode 100644 smoke-tests/smoke-e2e.bats create mode 100644 smoke-tests/test_helpers/utilities.bash diff --git a/.circleci/config.yml b/.circleci/config.yml index 98e983d9..5eb0af88 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,6 +12,10 @@ filters_publish: &filters_publish branches: ignore: /.*/ +orbs: + bats: circleci/bats@1.0.0 + cypress: cypress-io/cypress@3 + executors: node: docker: @@ -66,18 +70,33 @@ jobs: smoke_test: machine: - image: ubuntu-2004:2023.04.1 + image: ubuntu-2004:2024.01.1 steps: - checkout - attach_workspace: at: ./ + - bats/install - run: - name: Spin up example app - command: docker build -t smoke-tests . && docker run -dp 127.0.0.1:3000:3000 --name smoke-tests smoke-tests + name: What's the BATS? + command: | + which bats + bats --version + - cypress/install - run: - name: Spin down example app - command: docker stop smoke-tests && docker rm smoke-tests - + name: What's the Cypress? + command: npx cypress --version + - run: + name: Spin up and run e2e smoke tests + command: make smoke + - store_test_results: + path: ./smoke-tests/ + - store_artifacts: + path: ./smoke-tests/report.xml + - store_artifacts: + path: ./smoke-tests/collector/data-results + - run: + name: Tear down e2e smoke tests + command: make unsmoke publish_github: executor: github diff --git a/.gitignore b/.gitignore index ea831834..d01cec4f 100644 --- a/.gitignore +++ b/.gitignore @@ -135,6 +135,7 @@ junit.xml smoke-tests/collector/data.json smoke-tests/collector/data-results/*.json smoke-tests/report.xml +cypress/screenshots/* # example build outputs build diff --git a/DEVELOPING.md b/DEVELOPING.md index f131b61c..297d417e 100644 --- a/DEVELOPING.md +++ b/DEVELOPING.md @@ -12,7 +12,7 @@ - ESLint (dbaeumer.vscode-eslint) - Prettier (esbenp.prettier-vscode) - Prettier ESLint (rvest.vs-code-prettier-eslint) -- Docker - Required for running smoke-tests. +- Docker & Docker Compose - Required for running smoke-tests. - [Docker Desktop](https://www.docker.com/products/docker-desktop/) is a reliable choice if you don't have your own preference. ## Main Commands @@ -36,17 +36,27 @@ npm run test ## Smoke Tests -Smoke tests currently use Cypress and Docker, and rely on console output. +Smoke tests use Cypress and Docker with `docker-compose`, exporting telemetry to a local collector. +They can be run with either `npm` scripts or with `make` targets (the latter works better in CI). ```sh # run smoke tests with cypress and docker npm run test:smoke + +# alternative using make: run smoke tests with cypress and docker (used in CI) +make smoke ``` -If it doesn't clean up properly afterward, manually tear it down: +The results of both the tests themselves and the telemetry collected by the collector are in a file `data.json` in the `smoke-tests` directory. +These artifacts are also uploaded to Circle when run in CI. + +After smoke tests are done, tear down docker containers: ```sh -npm run clean:smoke-test-example +npm run test:unsmoke + +# alternative using make +make unsmoke ``` ## Example Application diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..9ec69eed --- /dev/null +++ b/Makefile @@ -0,0 +1,36 @@ +#: cleans up smoke test output +clean-smoke-tests: + rm -rf ./smoke-tests/collector/data.json + rm -rf ./smoke-tests/collector/data-results/*.json + rm -rf ./smoke-tests/report.* + +smoke-tests/collector/data.json: + @echo "" + @echo "+++ Zhuzhing smoke test's Collector data.json" + @touch $@ && chmod o+w $@ + +smoke-docker: smoke-tests/collector/data.json + @echo "" + @echo "+++ Spinning up the smokers." + @echo "" + docker-compose up --build --detach + +smoke-cypress: smoke-tests/collector/data.json + @echo "" + @echo "+++ Running Cypress in chrome browser." + @echo "" + npx cypress run --headed --browser chrome + +smoke-bats: smoke-tests/collector/data.json + @echo "" + @echo "+++ Running bats smoke tests." + @echo "" + cd smoke-tests && bats ./smoke-e2e.bats --report-formatter junit --output ./ + +smoke: smoke-docker smoke-cypress smoke-bats + +unsmoke: + @echo "" + @echo "+++ Spinning down the smokers." + @echo "" + cd smoke-tests && docker-compose down --volumes diff --git a/cypress.config.ts b/cypress.config.ts index 4ebba57d..aaddd779 100644 --- a/cypress.config.ts +++ b/cypress.config.ts @@ -6,5 +6,6 @@ export default defineConfig({ setupNodeEvents() { // implement node event listeners here }, + baseUrl: 'http://localhost:3000', }, }); diff --git a/cypress/e2e/spec.cy.ts b/cypress/e2e/spec.cy.ts index e12a88f0..e174eb73 100644 --- a/cypress/e2e/spec.cy.ts +++ b/cypress/e2e/spec.cy.ts @@ -1,11 +1,12 @@ describe('Smoke Tests', () => { - it('initializes the OpenTelemetry API', () => { - cy.visit('http://localhost:3000', { + beforeEach(() => { + cy.visit('index.html', { onBeforeLoad(win) { cy.stub(win.console, 'debug').as('consoleDebug'); }, }); - + }); + it('initializes the OpenTelemetry API', () => { cy.get('@consoleDebug').should( 'be.calledWithMatch', '@opentelemetry/api: Registered a global for diag', @@ -21,33 +22,20 @@ describe('Smoke Tests', () => { }); it('logs honeycomb config with debug enabled', () => { - cy.visit('http://localhost:3000', { - onBeforeLoad(win) { - cy.stub(win.console, 'debug').as('consoleDebug'); - }, - }); - cy.get('@consoleDebug').should( 'be.calledWithMatch', 'Honeycomb Web SDK Debug Mode Enabled', ); + cy.get('@consoleDebug').should( + 'be.calledWithMatch', + '@honeycombio/opentelemetry-web', + ); }); - - it('logs document load traces', () => { - cy.visit('http://localhost:3000', { - onBeforeLoad(win) { - cy.stub(win.console, 'dir').as('consoleDir'); - }, - }); - - cy.get('@consoleDir').should('be.calledWithMatch', { - name: 'documentLoad', - }); - cy.get('@consoleDir').should('be.calledWithMatch', { - name: 'resourceFetch', - }); - cy.get('@consoleDir').should('be.calledWithMatch', { - name: 'documentFetch', - }); + it('logs expected output with debug enabled', () => { + cy.get('@consoleDebug').should( + 'be.calledWithMatch', + 'BrowserDetector found resource.', + ); + cy.get('@consoleDebug').should('be.calledWithMatch', 'items to be sent'); }); }); diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..d9ea5f4e --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,18 @@ +version: '3.0' + +services: + collector: + image: otel/opentelemetry-collector:0.92.0 + command: ['--config=/etc/otel-collector-config.yaml'] + volumes: + - './smoke-tests/collector/otel-collector-config.yaml:/etc/otel-collector-config.yaml' + - './smoke-tests/collector:/var/lib' + ports: + - '4318:4318' + + app-hello-world-web: + build: + context: . + image: hny/hello-world-web:local + ports: + - '3000:3000' diff --git a/examples/hello-world-web/index.js b/examples/hello-world-web/index.js index cf06e21d..d7f61604 100644 --- a/examples/hello-world-web/index.js +++ b/examples/hello-world-web/index.js @@ -4,7 +4,9 @@ import { getWebAutoInstrumentations } from '@opentelemetry/auto-instrumentations const main = () => { // Initialize base OTel WebSDK const sdk = new HoneycombWebSDK({ - apiKey: 'api-key-goes-here', + // To send direct to Honeycomb, set API Key and comment out endpoint + // apiKey: 'api-key', + endpoint: 'http://localhost:4318/v1/traces', // send to local collector serviceName: 'web-distro', debug: true, instrumentations: [getWebAutoInstrumentations()], // add auto-instrumentation diff --git a/package.json b/package.json index d9629c26..ed2dde72 100644 --- a/package.json +++ b/package.json @@ -12,15 +12,17 @@ "build": "tsc --build", "check-format": "prettier --list-different \"./**/*.{ts,mts,cts,js,cjs,mjs,tsx,jsx}\"", "clean": "tsc --build --clean", - "clean:smoke-test-example": "docker stop smoke-tests && docker rm smoke-tests", "lint": "eslint .", "lint:ci": "npm run lint:ci-prep && npm run lint", "lint:ci-prep": "cd ./examples/hello-world-web && npm ci", "lint:fix": "eslint . --fix", - "start:smoke-test-example": "docker build -t smoke-tests . && docker run -dp 127.0.0.1:3000:3000 --name smoke-tests smoke-tests", + "test:smoke-docker": "docker-compose up --build --detach", + "test:smoke-cypress": "npx cypress run --headed --browser chrome", + "test:smoke-bats": "cd smoke-tests && bats ./smoke-e2e.bats --report-formatter junit --output ./", + "test:smoke": "npm run test:smoke-docker && npm run test:smoke-cypress && npm run test:smoke-bats", + "test:unsmoke": "docker-compose down", "test": "jest --config ./jest.config.js --no-cache -u --silent", - "test:ci": "jest --config ./jest.config.js --ci --runInBand --reporters=default --reporters=jest-junit --no-cache -u --silent", - "test:smoke": "npm run start:smoke-test-example && cypress run && npm run clean:smoke-test-example" + "test:ci": "jest --config ./jest.config.js --ci --runInBand --reporters=default --reporters=jest-junit --no-cache -u --silent" }, "author": "Honeycomb (https://www.honeycomb.io/)", "license": "Apache-2.0", diff --git a/smoke-tests/collector/data-results/.gitkeep b/smoke-tests/collector/data-results/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/smoke-tests/collector/otel-collector-config.yaml b/smoke-tests/collector/otel-collector-config.yaml new file mode 100644 index 00000000..287c584a --- /dev/null +++ b/smoke-tests/collector/otel-collector-config.yaml @@ -0,0 +1,24 @@ +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + cors: + allowed_origins: + - 'http://localhost:3000' + +processors: + batch: + +exporters: + file: + path: /var/lib/data.json + logging: + verbosity: detailed + +service: + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [file, logging] diff --git a/smoke-tests/smoke-e2e.bats b/smoke-tests/smoke-e2e.bats new file mode 100644 index 00000000..a9d2a200 --- /dev/null +++ b/smoke-tests/smoke-e2e.bats @@ -0,0 +1,61 @@ +#!/usr/bin/env bats + +load test_helpers/utilities + +CONTAINER_NAME="app-hello-world-web" +DOCUMENT_LOAD_SCOPE="@opentelemetry/instrumentation-document-load" + +setup_file() { + echo "# 🚧" >&3 +} +teardown_file() { + cp collector/data.json collector/data-results/data-${CONTAINER_NAME}.json +} + +# TESTS + +@test "Agent includes service.name in resource attributes" { + result=$(resource_attributes_received | jq "select(.key == \"service.name\").value.stringValue") + assert_equal "$result" '"web-distro"' +} + +@test "Agent includes Honeycomb distro version" { + version=$(resource_attributes_received | jq "select(.key == \"honeycomb.distro.version\").value.stringValue") + assert_not_empty "$version" + runtime_version=$(resource_attributes_received | jq "select(.key == \"honeycomb.distro.runtime_version\").value.stringValue") + assert_equal "$runtime_version" '"browser"' +} + +@test "Agent includes browser attributes" { + platform=$(resource_attributes_received | jq "select(.key == \"browser.platform\").value.stringValue") + assert_not_empty "$platform" + + mobile=$(resource_attributes_received | jq "select(.key == \"browser.mobile\").value.boolValue") + assert_equal "$mobile" 'false' + + language=$(resource_attributes_received | jq "select(.key == \"browser.language\").value.stringValue") + assert_equal "$language" '"en-US"' +} + +@test "Agent includes entry_page attributes" { + url=$(resource_attributes_received | jq "select(.key == \"entry_page.url\").value.stringValue") + assert_not_empty "$url" + + path=$(resource_attributes_received | jq "select(.key == \"entry_page.path\").value.boolValue") + assert_not_empty "$path" + + hostname=$(resource_attributes_received | jq "select(.key == \"entry_page.hostname\").value.stringValue") + assert_not_empty "$hostname" +} + +@test "Auto instrumentation produces 4 document load spans" { + result=$(span_names_for ${DOCUMENT_LOAD_SCOPE}) + assert_equal "$result" '"documentFetch" +"resourceFetch" +"documentLoad"' +} + +@test "Auto instrumentation adds session.id attribute" { + result=$(span_attributes_for ${DOCUMENT_LOAD_SCOPE} | jq "select(.key == \"session.id\").value.stringValue") + assert_not_empty "$result" +} diff --git a/smoke-tests/test_helpers/utilities.bash b/smoke-tests/test_helpers/utilities.bash new file mode 100644 index 00000000..9fbc0fa4 --- /dev/null +++ b/smoke-tests/test_helpers/utilities.bash @@ -0,0 +1,69 @@ +# UTILITY FUNCS + +# Span names for a given scope +# Arguments: $1 - scope name +span_names_for() { + spans_from_scope_named $1 | jq '.name' +} + +# Attributes for a given scope +# Arguments: $1 - scope name +span_attributes_for() { + spans_from_scope_named $1 | \ + jq ".attributes[]" +} + +# All resource attributes +resource_attributes_received() { + spans_received | jq ".resource.attributes[]?" +} + +# Spans for a given scope +# Arguments: $1 - scope name +spans_from_scope_named() { + spans_received | jq ".scopeSpans[] | select(.scope.name == \"$1\").spans[]" +} + +# All spans received +spans_received() { + jq ".resourceSpans[]?" ./collector/data.json +} + +# ASSERTION HELPERS + +# Fail and display details if the expected and actual values do not +# equal. Details include both values. +# +# Inspired by bats-assert * bats-support, but dramatically simplified +# Arguments: +# $1 - actual result +# $2 - expected result +assert_equal() { + if [[ $1 != "$2" ]]; then + { + echo + echo "-- 💥 values are not equal 💥 --" + echo "expected : $2" + echo "actual : $1" + echo "--" + echo + } >&2 # output error to STDERR + return 1 + fi +} + +# Fail and display details if the actual value is empty. +# Arguments: $1 - actual result +assert_not_empty() { + EMPTY=(\"\") + if [[ "$1" == "${EMPTY}" ]]; then + { + echo + echo "-- 💥 value is empty 💥 --" + echo "value : $1" + echo "--" + echo + } >&2 # output error to STDERR + return 1 + fi +}