test: expand local integration harness to old-suite coverage parity

.github/workflows/integration_tests_local.yml

@@ -4,11 +4,12 @@ name: local integration tests
 # against the real AWS Lambda base images + the Runtime Interface Emulator
 # (RIE). No AWS account or credentials required.
 #
-# One job per Node major version; each job builds and tests both container
-# variants (cjs, esm), which share the same ~1 GB base image, so each job
-# pulls it exactly once. If public.ecr.aws anonymous pull limits ever bite,
-# the fix is to mirror the base images to GHCR (same pattern dd-trace-js
-# uses in its mirror-image.yml workflow).
+# One job per Node major version; each job builds the container/layer fixture
+# images and runs every case (default: all of them) against that runtime.
+# The fixtures share the same ~1 GB base image, so each job pulls it exactly
+# once. If public.ecr.aws anonymous pull limits ever bite, the fix is to
+# mirror the base images to GHCR (same pattern dd-trace-js uses in its
+# mirror-image.yml workflow).
 
 on:
   pull_request:

integration_tests/container/cjs/Dockerfile

@@ -1,7 +1,11 @@
 ARG NODE_VERSION=22
 FROM public.ecr.aws/lambda/nodejs:${NODE_VERSION}
 
-COPY package.json handler.js ${LAMBDA_TASK_ROOT}/
+# handler.js is the default redirect-mode fixture (DD_LAMBDA_HANDLER=handler.handle).
+# The remaining *.js files are the manual-wrap fixture handlers ported from the
+# AWS suite (throw-error, status-code-500s, send-metrics, process-input,
+# http-requests) plus extractor.js for the DD_TRACE_EXTRACTOR case.
+COPY package.json *.js ${LAMBDA_TASK_ROOT}/
 COPY datadog-lambda-js-local.tgz /tmp/datadog-lambda-js-local.tgz
 RUN cd ${LAMBDA_TASK_ROOT} \
     && npm install --omit=dev \

integration_tests/container/cjs/extractor.js

@@ -0,0 +1,15 @@
+// Custom trace-context extractor fixture, loaded via
+// DD_TRACE_EXTRACTOR=extractor.extract (handler-style "module.function"
+// string, resolved by the shim's user-function loader).
+//
+// Returns a fixed context on every event: the extracted parent and the
+// `_dd.parent_source` span tag must show up in the trace, which the golden
+// asserts deterministically (ids are normalized to XXXX; the tag is not).
+module.exports.extract = async (event, context) => {
+  return {
+    traceId: "1234567890123456789",
+    parentId: "9876543210987654321",
+    sampleMode: 1,
+    source: "event",
+  };
+};

integration_tests/container/cjs/http-requests-manual.js

@@ -0,0 +1,18 @@
+// Manual-wrap companion to http-requests.js, run as the manual-http-requests
+// case — the exact shape of the AWS suite's http-requests function
+// (manual datadog() wrap, no userland dd-trace init).
+//
+// With no tracer initialized, TraceListener falls back to the library's own
+// patchHttp (src/trace/patch-http.ts), which wraps http/https and logs
+// "HTTP GET <url> TraceHeaders: [...]" per request. On real Lambda the
+// injected headers come from the platform's pass-through _X_AMZN_TRACE_ID;
+// under RIE that variable cannot be emulated (the RIC owns it), so the
+// golden pins the empty-context form (TraceHeaders: []) plus the mock echo
+// of the exact header set sent. The path under test — patchHttp wiring —
+// fails this golden the moment it stops wrapping or logging; the
+// context-dependent header values are pinned by the patch-http unit tests
+// and the AWS suite.
+const { datadog } = require("datadog-lambda-js");
+const { handle } = require("./http-requests");
+
+module.exports.handle = datadog(handle);

integration_tests/container/cjs/http-requests.js

@@ -0,0 +1,40 @@
+// Ported from integration_tests/http-requests.js with one seam adaptation:
+// the AWS suite hits the real ip-ranges.datadoghq.com/.eu endpoints; the
+// local harness is hermetic, so URLs come from MOCK_HTTP_URLS (set by run.sh
+// to the mock-http container on the case's docker network). The default keeps
+// this file runnable outside the harness.
+//
+// This entry point is UNWRAPPED and runs through the npm redirect entry
+// (dist/handler.handler) in the cjs-http-requests case. Redirect mode
+// initializes dd-trace first, so the tracer's http plugin patches axios and
+// the mock echo below shows the injected x-datadog-* headers in the golden.
+// The manual-wrap variant of the AWS suite's function — which exercises the
+// library's own patchHttp fallback instead — lives in http-requests-manual.js
+// and runs as the manual-http-requests case.
+//
+// The mock echoes the request headers it received, and this handler logs them,
+// so both goldens show the injected downstream trace context
+// (x-datadog-trace-id / x-datadog-parent-id, normalized to XXXX).
+const { sendDistributionMetric } = require("datadog-lambda-js");
+const axios = require("axios");
+
+const urls = (process.env.MOCK_HTTP_URLS ||
+  "https://ip-ranges.datadoghq.com,https://ip-ranges.datadoghq.eu"
+).split(",");
+
+async function handle(event, context) {
+  const responsePayload = { message: "hello, dog!" };
+
+  sendDistributionMetric("serverless.integration_test.execution", 1, "function:http-request");
+
+  for (let index = 0; index < urls.length; index++) {
+    const response = await axios({ url: urls[index], method: "get" });
+    console.log(`mock-http saw headers for ${urls[index]}: ${JSON.stringify(response.data.headers)}`);
+  }
+
+  console.log(`Snapshot test http requests successfully made to URLs: ${urls}`);
+
+  return responsePayload;
+}
+
+module.exports.handle = handle;

integration_tests/container/cjs/package.json

@@ -3,6 +3,7 @@
   "version": "1.0.0",
   "private": true,
   "dependencies": {
+    "axios": "1.18.1",
     "dd-trace": "5.105.0"
   }
 }

integration_tests/container/cjs/process-input.js

@@ -0,0 +1,91 @@
+const tracer = require("dd-trace").init();
+
+const { datadog, sendDistributionMetric } = require("datadog-lambda-js");
+
+// Minimal example of tracing to reduce flakiness
+async function handle(event, context) {
+  const span = tracer.scope().active();
+
+  const responsePayload = { message: "hello, dog!" };
+
+  sendDistributionMetric("serverless.integration_test.execution", 1, "function:process-input-traced");
+
+  const { recordIds, eventType } = getRecordIds(event);
+
+  if (recordIds) {
+    responsePayload.recordIds = recordIds;
+  }
+
+  if (eventType) {
+    responsePayload.eventType = eventType;
+  } else {
+    const requestId = getAPIGatewayRequestId(event);
+
+    if (requestId) {
+      responsePayload.eventType = "APIGateway";
+
+      if (span) {
+        span.setTag("api_gateway_request_id", requestId);
+      }
+    }
+  }
+
+  if (span) {
+    span.setTag("event_type", responsePayload.eventType);
+  }
+
+  return responsePayload;
+}
+
+const getRecordIds = tracer.wrap("getRecordIds", (event) => {
+  const recordIds = [];
+  let eventType = null;
+  if (event.Records) {
+    event.Records.forEach((record) => {
+      if (record.messageId) {
+        eventType = "SQS";
+        recordIds.push(record.messageId);
+      }
+      if (record.Sns) {
+        eventType = "SNS";
+        recordIds.push(record.Sns.MessageId);
+      }
+      if (record.s3) {
+        eventType = "S3";
+        const bucketName = record.s3.bucket.name;
+        const objectKey = record.s3.object.key;
+        recordIds.push(`${bucketName}/${objectKey}`);
+      }
+    });
+  }
+
+  if (eventType) {
+    const span = tracer.scope().active();
+
+    if (span) {
+      span.setTag("record_event_type", eventType);
+      span.setTag("record_ids", recordIds.join());
+    }
+  }
+
+  return { recordIds, eventType };
+});
+
+const getAPIGatewayRequestId = tracer.wrap("getAPIGatewayRequestId", (event) => {
+  let requestId;
+
+  if (event.requestContext) {
+    requestId = event.requestContext.requestId;
+
+    const span = tracer.scope().active();
+
+    if (span) {
+      span.setTag("api_gateway_request_id", requestId);
+      span.setTag("event_type", "APIGateway");
+    }
+  }
+
+  return requestId;
+});
+
+module.exports.handle = datadog(handle);

integration_tests/container/cjs/send-metrics.js

@@ -0,0 +1,57 @@
+const { datadog, sendDistributionMetric } = require("datadog-lambda-js");
+
+sendDistributionMetric(
+  "serverless.integration_test.outside_handler",
+  1,
+  "tagkey:tagvalue",
+  `eventsource:outside_handler`,
+);
+
+async function handle(event, context) {
+  const responsePayload = { message: "hello, dog!" };
+
+  if (event.requestContext) {
+    responsePayload.eventType = "APIGateway";
+    responsePayload.requestId = event.requestContext.requestId;
+  }
+
+  if (event.Records) {
+    responsePayload.recordIds = [];
+
+    event.Records.forEach((record) => {
+      if (record.messageId) {
+        responsePayload.eventType = "SQS";
+        responsePayload.recordIds.push(record.messageId);
+        sendDistributionMetric(
+          "serverless.integration_test.records_processed",
+          1,
+          "tagkey:tagvalue",
+          `eventsource:${responsePayload.eventType}`,
+        );
+      }
+      if (record.Sns) {
+        responsePayload.eventType = "SNS";
+        responsePayload.recordIds.push(record.Sns.MessageId);
+        sendDistributionMetric(
+          "serverless.integration_test.records_processed",
+          1,
+          "tagkey:tagvalue",
+          `eventsource:${responsePayload.eventType}`,
+        );
+      }
+    });
+  }
+
+  sendDistributionMetric(
+    "serverless.integration_test.execution",
+    1,
+    "tagkey:tagvalue",
+    `eventsource:${responsePayload.eventType}`,
+  );
+
+  console.log(`Processed ${responsePayload.eventType} request`);
+
+  return responsePayload;
+}
+
+module.exports.handle = datadog(handle);

integration_tests/container/cjs/status-code-500s.js

@@ -0,0 +1,15 @@
+// Ported as-is from integration_tests/status-code-500s.js (the AWS suite's
+// API Gateway 500 case): returns statusCode 500 so the span gets the error
+// tag and the aws.lambda.enhanced.errors metric is emitted.
+const tracer = require("dd-trace").init();
+const { datadog } = require("datadog-lambda-js");
+
+async function handle(event, context) {
+  const span = tracer.scope().active();
+  return {
+    "statusCode": 500,
+    "body": {}
+  };
+}
+
+module.exports.handle = datadog(handle);

integration_tests/container/cjs/throw-error.js

@@ -0,0 +1,10 @@
+// Ported as-is from integration_tests/throw-error-traced.js (the AWS suite's
+// manual-wrap error case): a handler that always throws, wrapped with
+// datadog() manually. Exercises thrown-error -> span error + rethrow through
+// the RIE invoke path.
+const { datadog } = require("datadog-lambda-js");
+
+async function handle(event, context) {
+  throw new Error("Hello");
+}
+module.exports.handle = datadog(handle);

integration_tests/container/layer/.gitignore

@@ -0,0 +1,3 @@
+# Generated by integration_tests_local/run.sh from the repo build; never committed.
+layer_pkg/
+deps.package.json

integration_tests/container/layer/Dockerfile

@@ -0,0 +1,28 @@
+# Layer-mode fixture: exercises the customer-facing layer entrypoint
+# /opt/nodejs/node_modules/datadog-lambda-js/handler.handler instead of the
+# npm-installed dist/handler.handler path. The layer tree (layer_pkg/) and the
+# dependency manifest (deps.package.json) are generated by
+# integration_tests_local/run.sh from the repo build, mirroring the release
+# Dockerfile's layer layout:
+#   dist/ contents  -> /opt/nodejs/node_modules/datadog-lambda-js/
+#   src/handler.mjs -> .../datadog-lambda-js/handler.mjs
+#   src/runtime/module_importer.js -> .../datadog-lambda-js/runtime/ (overlay)
+#   production deps + dd-trace -> /opt/nodejs/node_modules/
+ARG NODE_VERSION=22
+FROM public.ecr.aws/lambda/nodejs:${NODE_VERSION}
+
+# Install dependencies BEFORE copying layer_pkg: npm prunes anything in
+# node_modules that is not in deps.package.json, so copying the layer tree
+# first would silently delete it.
+COPY deps.package.json /opt/nodejs/package.json
+# --omit=optional matches the release build's --ignore-optional: pprof/appsec
+# natives are not exercised by these fixtures and skipping them keeps the
+# build fast.
+RUN cd /opt/nodejs && npm install --omit=dev --omit=optional --no-audit --no-fund
+COPY layer_pkg /opt/nodejs/node_modules/datadog-lambda-js/
+
+# handler.js (CJS) and esm.mjs (top-level await) are plain unwrapped user
+# handlers; the layer's handler.mjs wraps them via DD_LAMBDA_HANDLER.
+COPY handler.js esm.mjs ${LAMBDA_TASK_ROOT}/
+
+CMD ["/opt/nodejs/node_modules/datadog-lambda-js/handler.handler"]

integration_tests/container/layer/esm.mjs

@@ -0,0 +1,11 @@
+// Plain unwrapped ESM user handler with top-level await, exercising the
+// layer's ESM loader registration (Module.register in handler.mjs) via
+// DD_LAMBDA_HANDLER=esm.handle.
+import { promisify } from "util";
+
+// Verify top level await
+await promisify(setTimeout)(100);
+
+export function handle(ev) {
+  return { message: "hello, dog!" };
+}

integration_tests/container/layer/handler.js

@@ -0,0 +1,6 @@
+// Plain unwrapped CJS user handler. The layer entrypoint
+// (/opt/nodejs/node_modules/datadog-lambda-js/handler.handler) wraps it via
+// DD_LAMBDA_HANDLER=handler.handle.
+exports.handle = async (event) => {
+  return { message: "hello, dog!" };
+};