maint: build out smoke tests and connect to CI (#55)

## Which problem is this PR solving?

- Closes #17 

## Short description of the changes

- Add smoke tests using BATS, similar to our other distros. This
primarily allows for re-using smoke test logic independent of language,
as it inspects and asserts against telemetry collected by the Collector
and exported to a file.
- Update CircleCI config to run smoke tests in every push, similar to
lint/build/test
- Add Makefile for easier smoke testing. This doesn't appear to be
necessary for local smoke testing, as similar scripts in the
`package.json` also work locally. CI didn't seem to work as well, hence
the decision to keep the Makefile.
- Update `DEVELOPING.md` with some details around smoke tests
- Update example to send to collector at localhost, without API key.
This may be the most "controversial" and may warrant having a different
example app - open to ideas. If we keep an API key we have end up with
CORS errors for a "With-Credentials" header.

## How to verify that this has the expected result

See the [results in
Circle](https://app.circleci.com/pipelines/github/honeycombio/honeycomb-opentelemetry-web/113/workflows/546edcba-4d22-4dd9-8fc3-114170373f0f/jobs/435/artifacts),
including actual assertions like `Agent includes browser attributes` as
well as telemetry output stored from the collector file export.

.circleci/config.yml

@@ -12,6 +12,10 @@ filters_publish: &filters_publish
     branches:
       ignore: /.*/
 
+orbs:
+  bats: circleci/[email protected]
+  cypress: cypress-io/cypress@3
+
 executors:
   node:
     docker:
@@ -66,18 +70,33 @@ jobs:
 
   smoke_test:
     machine:
-      image: ubuntu-2004:2023.04.1
+      image: ubuntu-2004:2024.01.1
     steps:
       - checkout
       - attach_workspace:
           at: ./
+      - bats/install
       - run:
-          name: Spin up example app
-          command: docker build -t smoke-tests . && docker run -dp 127.0.0.1:3000:3000 --name smoke-tests smoke-tests
+          name: What's the BATS?
+          command: |
+            which bats
+            bats --version
+      - cypress/install
       - run:
-          name: Spin down example app
-          command: docker stop smoke-tests && docker rm smoke-tests
-
+          name: What's the Cypress?
+          command: npx cypress --version
+      - run:
+          name: Spin up and run e2e smoke tests
+          command: make smoke
+      - store_test_results:
+          path: ./smoke-tests/
+      - store_artifacts:
+          path: ./smoke-tests/report.xml
+      - store_artifacts:
+          path: ./smoke-tests/collector/data-results
+      - run:
+          name: Tear down e2e smoke tests
+          command: make unsmoke
 
   publish_github:
     executor: github

.gitignore

@@ -135,6 +135,7 @@ junit.xml
 smoke-tests/collector/data.json
 smoke-tests/collector/data-results/*.json
 smoke-tests/report.xml
+cypress/screenshots/*
 
 # example build outputs
 build

DEVELOPING.md

@@ -12,7 +12,7 @@
   - ESLint (dbaeumer.vscode-eslint)
   - Prettier (esbenp.prettier-vscode)
   - Prettier ESLint (rvest.vs-code-prettier-eslint)
-- Docker - Required for running smoke-tests.
+- Docker & Docker Compose - Required for running smoke-tests.
   - [Docker Desktop](https://www.docker.com/products/docker-desktop/) is a reliable choice if you don't have your own preference.
 
 ## Main Commands
@@ -36,17 +36,27 @@ npm run test
 
 ## Smoke Tests
 
-Smoke tests currently use Cypress and Docker, and rely on console output.
+Smoke tests use Cypress and Docker with `docker-compose`, exporting telemetry to a local collector.
+They can be run with either `npm` scripts or with `make` targets (the latter works better in CI).
 
 ```sh
 # run smoke tests with cypress and docker
 npm run test:smoke
+
+# alternative using make: run smoke tests with cypress and docker (used in CI)
+make smoke
 ```
 
-If it doesn't clean up properly afterward, manually tear it down:
+The results of both the tests themselves and the telemetry collected by the collector are in a file `data.json` in the `smoke-tests` directory.
+These artifacts are also uploaded to Circle when run in CI.
+
+After smoke tests are done, tear down docker containers:
 
 ```sh
-npm run clean:smoke-test-example
+npm run test:unsmoke
+
+# alternative using make
+make unsmoke
 ```
 
 ## Example Application

Makefile

@@ -0,0 +1,36 @@
+#: cleans up smoke test output
+clean-smoke-tests:
+	rm -rf ./smoke-tests/collector/data.json
+	rm -rf ./smoke-tests/collector/data-results/*.json
+	rm -rf ./smoke-tests/report.*
+
+smoke-tests/collector/data.json:
+	@echo ""
+	@echo "+++ Zhuzhing smoke test's Collector data.json"
+	@touch $@ && chmod o+w $@
+
+smoke-docker: smoke-tests/collector/data.json
+	@echo ""
+	@echo "+++ Spinning up the smokers."
+	@echo ""
+	docker-compose up --build --detach
+
+smoke-cypress: smoke-tests/collector/data.json
+	@echo ""
+	@echo "+++ Running Cypress in chrome browser."
+	@echo ""
+	npx cypress run --headed --browser chrome
+
+smoke-bats: smoke-tests/collector/data.json
+	@echo ""
+	@echo "+++ Running bats smoke tests."
+	@echo ""
+	cd smoke-tests && bats ./smoke-e2e.bats --report-formatter junit --output ./
+
+smoke: smoke-docker smoke-cypress smoke-bats
+
+unsmoke:
+	@echo ""
+	@echo "+++ Spinning down the smokers."
+	@echo ""
+	cd smoke-tests && docker-compose down --volumes

cypress.config.ts

@@ -6,5 +6,6 @@ export default defineConfig({
     setupNodeEvents() {
       // implement node event listeners here
     },
+    baseUrl: 'http://localhost:3000',
   },
 });

cypress/e2e/spec.cy.ts

@@ -1,11 +1,12 @@
 describe('Smoke Tests', () => {
-  it('initializes the OpenTelemetry API', () => {
-    cy.visit('http://localhost:3000', {
+  beforeEach(() => {
+    cy.visit('index.html', {
       onBeforeLoad(win) {
         cy.stub(win.console, 'debug').as('consoleDebug');
       },
     });
-
+  });
+  it('initializes the OpenTelemetry API', () => {
     cy.get('@consoleDebug').should(
       'be.calledWithMatch',
       '@opentelemetry/api: Registered a global for diag',
@@ -21,33 +22,20 @@ describe('Smoke Tests', () => {
   });
 
   it('logs honeycomb config with debug enabled', () => {
-    cy.visit('http://localhost:3000', {
-      onBeforeLoad(win) {
-        cy.stub(win.console, 'debug').as('consoleDebug');
-      },
-    });
-
     cy.get('@consoleDebug').should(
       'be.calledWithMatch',
       'Honeycomb Web SDK Debug Mode Enabled',
     );
+    cy.get('@consoleDebug').should(
+      'be.calledWithMatch',
+      '@honeycombio/opentelemetry-web',
+    );
   });
-
-  it('logs document load traces', () => {
-    cy.visit('http://localhost:3000', {
-      onBeforeLoad(win) {
-        cy.stub(win.console, 'dir').as('consoleDir');
-      },
-    });
-
-    cy.get('@consoleDir').should('be.calledWithMatch', {
-      name: 'documentLoad',
-    });
-    cy.get('@consoleDir').should('be.calledWithMatch', {
-      name: 'resourceFetch',
-    });
-    cy.get('@consoleDir').should('be.calledWithMatch', {
-      name: 'documentFetch',
-    });
+  it('logs expected output with debug enabled', () => {
+    cy.get('@consoleDebug').should(
+      'be.calledWithMatch',
+      'BrowserDetector found resource.',
+    );
+    cy.get('@consoleDebug').should('be.calledWithMatch', 'items to be sent');
   });
 });

docker-compose.yml

@@ -0,0 +1,18 @@
+version: '3.0'
+
+services:
+  collector:
+    image: otel/opentelemetry-collector:0.92.0
+    command: ['--config=/etc/otel-collector-config.yaml']
+    volumes:
+      - './smoke-tests/collector/otel-collector-config.yaml:/etc/otel-collector-config.yaml'
+      - './smoke-tests/collector:/var/lib'
+    ports:
+      - '4318:4318'
+
+  app-hello-world-web:
+    build:
+      context: .
+    image: hny/hello-world-web:local
+    ports:
+      - '3000:3000'

examples/hello-world-web/index.js

@@ -4,7 +4,9 @@ import { getWebAutoInstrumentations } from '@opentelemetry/auto-instrumentations
 const main = () => {
   // Initialize base OTel WebSDK
   const sdk = new HoneycombWebSDK({
-    apiKey: 'api-key-goes-here',
+    // To send direct to Honeycomb, set API Key and comment out endpoint
+    // apiKey: 'api-key',
+    endpoint: 'http://localhost:4318/v1/traces', // send to local collector
     serviceName: 'web-distro',
     debug: true,
     instrumentations: [getWebAutoInstrumentations()], // add auto-instrumentation

package.json

@@ -12,15 +12,17 @@
     "build": "tsc --build",
     "check-format": "prettier --list-different \"./**/*.{ts,mts,cts,js,cjs,mjs,tsx,jsx}\"",
     "clean": "tsc --build --clean",
-    "clean:smoke-test-example": "docker stop smoke-tests && docker rm smoke-tests",
     "lint": "eslint .",
     "lint:ci": "npm run lint:ci-prep && npm run lint",
     "lint:ci-prep": "cd ./examples/hello-world-web && npm ci",
     "lint:fix": "eslint . --fix",
-    "start:smoke-test-example": "docker build -t smoke-tests . && docker run -dp 127.0.0.1:3000:3000 --name smoke-tests smoke-tests",
+    "test:smoke-docker": "docker-compose up --build --detach",
+    "test:smoke-cypress": "npx cypress run --headed --browser chrome",
+    "test:smoke-bats": "cd smoke-tests && bats ./smoke-e2e.bats --report-formatter junit --output ./",
+    "test:smoke": "npm run test:smoke-docker && npm run test:smoke-cypress && npm run test:smoke-bats",
+    "test:unsmoke": "docker-compose down",
     "test": "jest --config ./jest.config.js --no-cache -u --silent",
-    "test:ci": "jest --config ./jest.config.js --ci --runInBand --reporters=default --reporters=jest-junit --no-cache -u --silent",
-    "test:smoke": "npm run start:smoke-test-example && cypress run && npm run clean:smoke-test-example"
+    "test:ci": "jest --config ./jest.config.js --ci --runInBand --reporters=default --reporters=jest-junit --no-cache -u --silent"
   },
   "author": "Honeycomb <[email protected]> (https://www.honeycomb.io/)",
   "license": "Apache-2.0",

smoke-tests/collector/otel-collector-config.yaml

@@ -0,0 +1,24 @@
+receivers:
+  otlp:
+    protocols:
+      http:
+        endpoint: 0.0.0.0:4318
+        cors:
+          allowed_origins:
+            - 'http://localhost:3000'
+
+processors:
+  batch:
+
+exporters:
+  file:
+    path: /var/lib/data.json
+  logging:
+    verbosity: detailed
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [file, logging]

smoke-tests/smoke-e2e.bats

@@ -0,0 +1,61 @@
+#!/usr/bin/env bats
+
+load test_helpers/utilities
+
+CONTAINER_NAME="app-hello-world-web"
+DOCUMENT_LOAD_SCOPE="@opentelemetry/instrumentation-document-load"
+
+setup_file() {
+  echo "# 🚧" >&3
+}
+teardown_file() {
+  cp collector/data.json collector/data-results/data-${CONTAINER_NAME}.json
+}
+
+# TESTS
+
+@test "Agent includes service.name in resource attributes" {
+  result=$(resource_attributes_received | jq "select(.key == \"service.name\").value.stringValue")
+  assert_equal "$result" '"web-distro"'
+}
+
+@test "Agent includes Honeycomb distro version" {
+  version=$(resource_attributes_received | jq "select(.key == \"honeycomb.distro.version\").value.stringValue")
+  assert_not_empty "$version"
+  runtime_version=$(resource_attributes_received | jq "select(.key == \"honeycomb.distro.runtime_version\").value.stringValue")
+  assert_equal "$runtime_version" '"browser"'
+}
+
+@test "Agent includes browser attributes" {
+  platform=$(resource_attributes_received | jq "select(.key == \"browser.platform\").value.stringValue")
+  assert_not_empty "$platform"
+
+  mobile=$(resource_attributes_received | jq "select(.key == \"browser.mobile\").value.boolValue")
+  assert_equal "$mobile" 'false'
+
+  language=$(resource_attributes_received | jq "select(.key == \"browser.language\").value.stringValue")
+  assert_equal "$language" '"en-US"'
+}
+
+@test "Agent includes entry_page attributes" {
+  url=$(resource_attributes_received | jq "select(.key == \"entry_page.url\").value.stringValue")
+  assert_not_empty "$url"
+
+  path=$(resource_attributes_received | jq "select(.key == \"entry_page.path\").value.boolValue")
+  assert_not_empty "$path"
+
+  hostname=$(resource_attributes_received | jq "select(.key == \"entry_page.hostname\").value.stringValue")
+  assert_not_empty "$hostname"
+}
+
+@test "Auto instrumentation produces 4 document load spans" {
+  result=$(span_names_for ${DOCUMENT_LOAD_SCOPE})
+  assert_equal "$result" '"documentFetch"
+"resourceFetch"
+"documentLoad"'
+}
+
+@test "Auto instrumentation adds session.id attribute" {
+	result=$(span_attributes_for ${DOCUMENT_LOAD_SCOPE} | jq "select(.key == \"session.id\").value.stringValue")
+	assert_not_empty "$result"
+}