Skip to content

Commit c817f4e

Browse files
committed
Merge branch 'main' of github.com:DataDog/datadog-lambda-extension
2 parents 56a529b + 8f99bda commit c817f4e

12 files changed

Lines changed: 243 additions & 32 deletions

File tree

‎.github/CODEOWNERS‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
.github/chainguard/serverless-init-ci-publish.sts.yaml @DataDog/serverless
2+
.github/publish-serverless-init-to-ghcr.yaml @DataDog/serverless
Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
# DD Octo STS Trust Policy for serverless-init-ci GitLab pipeline
2+
#
3+
# This policy allows the serverless-init-ci GitLab pipeline to trigger
4+
# GitHub Actions workflows that publish serverless-init images to GHCR.
5+
#
6+
# Reference: https://datadoghq.atlassian.net/wiki/spaces/SECENG/pages/5138645099
7+
# Pipeline: https://gitlab.ddbuild.io/DataDog/serverless-init-ci
8+
9+
issuer: https://gitlab.ddbuild.io
10+
11+
# Subject pattern matches the serverless-init-ci repo on any protected branch or tag
12+
subject_pattern: "project_path:DataDog/serverless-init-ci:ref_type:(branch|tag):ref:.*"
13+
14+
# Only allow protected branches and tags (security control)
15+
claim_pattern:
16+
project_path: "DataDog/serverless-init-ci"
17+
ref_type: "^(branch|tag)$"
18+
ref_protected: "true"
19+
pipeline_source: "^(web|pipeline)$"
20+
ci_config_ref_uri: "^gitlab\\.ddbuild\\.io/DataDog/serverless-init-ci//\\.gitlab-ci\\.yml@refs/(heads|tags)/.*$"
21+
22+
# Minimal permissions: only trigger GitHub Actions workflows
23+
# The workflow itself uses GITHUB_TOKEN for GHCR access
24+
permissions:
25+
actions: write
Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
name: Publish serverless-init to GHCR
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
source_image:
7+
description: 'Source image from registry.datadoghq.com (e.g., registry.datadoghq.com/serverless-init:1.7.8)'
8+
required: true
9+
type: string
10+
version:
11+
description: 'Version tag (e.g., 1.7.8 or 1.7.8-rc1)'
12+
required: true
13+
type: string
14+
image_suffix:
15+
description: 'Image suffix (empty for standard, -alpine for alpine)'
16+
required: false
17+
type: string
18+
default: ''
19+
pipeline_id:
20+
description: 'GitLab pipeline ID'
21+
required: true
22+
type: string
23+
is_latest:
24+
description: 'Tag as latest (true for prod releases, false for RCs)'
25+
required: true
26+
type: boolean
27+
default: false
28+
29+
permissions:
30+
packages: write
31+
contents: read
32+
33+
jobs:
34+
publish-to-ghcr:
35+
runs-on: ubuntu-latest
36+
steps:
37+
- name: Install crane
38+
run: |
39+
cd /tmp
40+
wget https://github.com/google/go-containerregistry/releases/download/v0.20.2/go-containerregistry_Linux_x86_64.tar.gz
41+
tar -xzf go-containerregistry_Linux_x86_64.tar.gz
42+
sudo mv crane /usr/local/bin/
43+
crane version
44+
45+
- name: Login to GHCR
46+
uses: docker/login-action@v3
47+
with:
48+
registry: ghcr.io
49+
username: ${{ github.actor }}
50+
password: ${{ secrets.GITHUB_TOKEN }}
51+
52+
- name: Wait for image availability
53+
run: |
54+
SOURCE_IMAGE="${{ inputs.source_image }}"
55+
MAX_ATTEMPTS=20
56+
RETRY_DELAY=30
57+
# Maximum wait time: 20 attempts × 30s = 600s (10 minutes)
58+
59+
echo "⏳ Waiting for image to be available: ${SOURCE_IMAGE}"
60+
echo "Will check every ${RETRY_DELAY}s for up to $((MAX_ATTEMPTS * RETRY_DELAY))s"
61+
62+
for i in $(seq 1 $MAX_ATTEMPTS); do
63+
echo "Attempt $i/$MAX_ATTEMPTS: Checking if image exists..."
64+
65+
if crane manifest ${SOURCE_IMAGE} >/dev/null 2>&1; then
66+
echo "✅ Image is available!"
67+
exit 0
68+
fi
69+
70+
if [ $i -lt $MAX_ATTEMPTS ]; then
71+
echo "⏳ Image not yet available, waiting ${RETRY_DELAY}s..."
72+
sleep $RETRY_DELAY
73+
fi
74+
done
75+
76+
echo "❌ Image did not become available after $((MAX_ATTEMPTS * RETRY_DELAY))s"
77+
exit 1
78+
79+
- name: Copy image to GHCR
80+
run: |
81+
SOURCE_IMAGE="${{ inputs.source_image }}"
82+
VERSION="${{ inputs.version }}"
83+
IMAGE_SUFFIX="${{ inputs.image_suffix }}"
84+
PIPELINE_ID="${{ inputs.pipeline_id }}"
85+
IS_LATEST="${{ inputs.is_latest }}"
86+
87+
DEST_BASE="ghcr.io/datadog/datadog-lambda-extension/serverless-init"
88+
89+
echo "📦 Publishing serverless-init image to GHCR"
90+
echo " Source: ${SOURCE_IMAGE}"
91+
echo " Destinations:"
92+
echo " - ${DEST_BASE}:${VERSION}${IMAGE_SUFFIX}"
93+
echo " - ${DEST_BASE}:v${PIPELINE_ID}${IMAGE_SUFFIX}"
94+
95+
# Copy with version tag (with retry logic)
96+
# Maximum retry duration: 3 attempts with 10s delays between retries
97+
# This workflow is triggered in parallel with the publish attempt to registry.datadoghq.com
98+
# Registry.datadoghq.com should normally need about ~30 seconds to recieve the new image
99+
MAX_COPY_ATTEMPTS=3
100+
COPY_RETRY_DELAY=10
101+
102+
for i in $(seq 1 $MAX_COPY_ATTEMPTS); do
103+
echo "Copying image (attempt $i/$MAX_COPY_ATTEMPTS)..."
104+
105+
if crane copy ${SOURCE_IMAGE} ${DEST_BASE}:${VERSION}${IMAGE_SUFFIX}; then
106+
echo "✅ Image copied successfully!"
107+
break
108+
fi
109+
110+
if [ $i -lt $MAX_COPY_ATTEMPTS ]; then
111+
echo "⚠️ Copy failed, retrying in ${COPY_RETRY_DELAY}s..."
112+
sleep $COPY_RETRY_DELAY
113+
else
114+
echo "❌ Failed to copy image after $MAX_COPY_ATTEMPTS attempts"
115+
exit 1
116+
fi
117+
done
118+
119+
# Tag for pipeline ID
120+
crane tag ${DEST_BASE}:${VERSION}${IMAGE_SUFFIX} v${PIPELINE_ID}${IMAGE_SUFFIX}
121+
122+
# Tag as latest if this is a production release
123+
if [ "$IS_LATEST" = "true" ]; then
124+
echo " - ${DEST_BASE}:latest${IMAGE_SUFFIX}"
125+
crane tag ${DEST_BASE}:${VERSION}${IMAGE_SUFFIX} latest${IMAGE_SUFFIX}
126+
fi
127+
128+
echo "✅ Successfully published image to GHCR!"
129+
echo "📍 View at: https://github.com/DataDog/datadog-lambda-extension/pkgs/container/datadog-lambda-extension%2Fserverless-init"

‎bottlecap/src/config/env.rs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -418,6 +418,13 @@ pub struct EnvConfig {
418418
/// Default is `false`.
419419
#[serde(deserialize_with = "deserialize_optional_bool_from_anything")]
420420
pub compute_trace_stats_on_extension: Option<bool>,
421+
/// @env `DD_SPAN_DEDUP_TIMEOUT`
422+
///
423+
/// The timeout for the span deduplication service to check if a span key exists, in seconds.
424+
/// For now, this is a temporary field added to debug the failure of `check_and_add()` in span dedup service.
425+
/// Do not use this field extensively in production.
426+
#[serde(deserialize_with = "deserialize_optional_duration_from_seconds_ignore_zero")]
427+
pub span_dedup_timeout: Option<Duration>,
421428
/// @env `DD_API_KEY_SECRET_RELOAD_INTERVAL`
422429
///
423430
/// The interval at which the Datadog API key is reloaded, in seconds.
@@ -654,6 +661,7 @@ fn merge_config(config: &mut Config, env_config: &EnvConfig) {
654661
merge_option_to_value!(config, env_config, capture_lambda_payload);
655662
merge_option_to_value!(config, env_config, capture_lambda_payload_max_depth);
656663
merge_option_to_value!(config, env_config, compute_trace_stats_on_extension);
664+
merge_option!(config, env_config, span_dedup_timeout);
657665
merge_option!(config, env_config, api_key_secret_reload_interval);
658666
merge_option_to_value!(config, env_config, serverless_appsec_enabled);
659667
merge_option!(config, env_config, appsec_rules);
@@ -853,6 +861,7 @@ mod tests {
853861
jail.set_env("DD_CAPTURE_LAMBDA_PAYLOAD", "true");
854862
jail.set_env("DD_CAPTURE_LAMBDA_PAYLOAD_MAX_DEPTH", "5");
855863
jail.set_env("DD_COMPUTE_TRACE_STATS_ON_EXTENSION", "true");
864+
jail.set_env("DD_SPAN_DEDUP_TIMEOUT", "5");
856865
jail.set_env("DD_API_KEY_SECRET_RELOAD_INTERVAL", "10");
857866
jail.set_env("DD_SERVERLESS_APPSEC_ENABLED", "true");
858867
jail.set_env("DD_APPSEC_RULES", "/path/to/rules.json");
@@ -1006,6 +1015,7 @@ mod tests {
10061015
capture_lambda_payload: true,
10071016
capture_lambda_payload_max_depth: 5,
10081017
compute_trace_stats_on_extension: true,
1018+
span_dedup_timeout: Some(Duration::from_secs(5)),
10091019
api_key_secret_reload_interval: Some(Duration::from_secs(10)),
10101020
serverless_appsec_enabled: true,
10111021
appsec_rules: Some("/path/to/rules.json".to_string()),

‎bottlecap/src/config/mod.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -348,6 +348,7 @@ pub struct Config {
348348
pub capture_lambda_payload: bool,
349349
pub capture_lambda_payload_max_depth: u32,
350350
pub compute_trace_stats_on_extension: bool,
351+
pub span_dedup_timeout: Option<Duration>,
351352
pub api_key_secret_reload_interval: Option<Duration>,
352353

353354
pub serverless_appsec_enabled: bool,
@@ -457,6 +458,7 @@ impl Default for Config {
457458
capture_lambda_payload: false,
458459
capture_lambda_payload_max_depth: 10,
459460
compute_trace_stats_on_extension: false,
461+
span_dedup_timeout: None,
460462
api_key_secret_reload_interval: None,
461463

462464
serverless_appsec_enabled: false,

‎bottlecap/src/config/yaml.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -995,6 +995,7 @@ api_security_sample_delay: 60 # Seconds
995995
capture_lambda_payload: true,
996996
capture_lambda_payload_max_depth: 5,
997997
compute_trace_stats_on_extension: true,
998+
span_dedup_timeout: None,
998999
api_key_secret_reload_interval: None,
9991000

10001001
serverless_appsec_enabled: true,

‎bottlecap/src/tags/lambda/tags.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ const FUNCTION_TAGS_KEY: &str = "_dd.tags.function";
4545
// TODO(astuyve) decide what to do with the version
4646
const EXTENSION_VERSION_KEY: &str = "dd_extension_version";
4747
// TODO(duncanista) figure out a better way to not hardcode this
48-
pub const EXTENSION_VERSION: &str = "91-next";
48+
pub const EXTENSION_VERSION: &str = "92-next";
4949

5050
const REGION_KEY: &str = "region";
5151
const ACCOUNT_ID_KEY: &str = "account_id";

‎bottlecap/src/traces/span_dedup_service.rs‎

Lines changed: 35 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@
22
// SPDX-License-Identifier: Apache-2.0
33

44
use tokio::sync::{mpsc, oneshot};
5-
use tracing::error;
5+
use tokio::time::Duration;
6+
use tracing::warn;
67

78
use crate::traces::span_dedup::{DedupKey, Deduper};
89

@@ -12,6 +13,8 @@ pub enum DedupError {
1213
SendError(mpsc::error::SendError<DedupCommand>),
1314
#[error("Failed to receive response from deduper: {0}")]
1415
RecvError(oneshot::error::RecvError),
16+
#[error("Timeout waiting for response from deduper")]
17+
Timeout,
1518
}
1619

1720
pub enum DedupCommand {
@@ -34,14 +37,29 @@ impl DedupHandle {
3437
///
3538
/// # Errors
3639
///
37-
/// Returns an error if the command cannot be sent to the deduper service
38-
/// or if the response cannot be received.
39-
pub async fn check_and_add(&self, key: DedupKey) -> Result<bool, DedupError> {
40+
/// Returns an error if the command cannot be sent to the deduper service,
41+
/// if the response cannot be received, or if the operation times out after 5 seconds.
42+
pub async fn check_and_add(
43+
&self,
44+
key: DedupKey,
45+
timeout: Option<Duration>,
46+
) -> Result<bool, DedupError> {
4047
let (response_tx, response_rx) = oneshot::channel();
4148
self.tx
4249
.send(DedupCommand::CheckAndAdd(key, response_tx))
4350
.map_err(DedupError::SendError)?;
44-
response_rx.await.map_err(DedupError::RecvError)
51+
52+
// Sometimes the dedup service fails to send a response for unknown reasons, so we
53+
// add a timeout to avoid blocking the caller forever. We may remove the
54+
// timeout if we can figure out and fix the root cause.
55+
if let Some(timeout) = timeout {
56+
tokio::time::timeout(timeout, response_rx)
57+
.await
58+
.map_err(|_| DedupError::Timeout)?
59+
.map_err(DedupError::RecvError)
60+
} else {
61+
response_rx.await.map_err(DedupError::RecvError)
62+
}
4563
}
4664
}
4765

@@ -77,7 +95,7 @@ impl DedupService {
7795
DedupCommand::CheckAndAdd(key, response_tx) => {
7896
let was_added = self.deduper.check_and_add(key);
7997
if let Err(e) = response_tx.send(was_added) {
80-
error!("Failed to send check_and_add response: {e:?}");
98+
warn!("Failed to send check_and_add response: {e:?}");
8199
}
82100
}
83101
}
@@ -112,17 +130,17 @@ mod tests {
112130
let key2 = DedupKey::new(100, 456);
113131

114132
// First call should return true (key was added)
115-
assert!(handle.check_and_add(key1).await.unwrap());
133+
assert!(handle.check_and_add(key1, None).await.unwrap());
116134

117135
// Second call should return false (key already exists)
118-
assert!(!handle.check_and_add(key1).await.unwrap());
136+
assert!(!handle.check_and_add(key1, None).await.unwrap());
119137

120138
// Different key should return true again
121-
assert!(handle.check_and_add(key2).await.unwrap());
139+
assert!(handle.check_and_add(key2, None).await.unwrap());
122140

123141
// Calling again on already-added keys should return false
124-
assert!(!handle.check_and_add(key1).await.unwrap());
125-
assert!(!handle.check_and_add(key2).await.unwrap());
142+
assert!(!handle.check_and_add(key1, None).await.unwrap());
143+
assert!(!handle.check_and_add(key2, None).await.unwrap());
126144
}
127145

128146
#[tokio::test]
@@ -139,17 +157,17 @@ mod tests {
139157
let key4 = DedupKey::new(4, 40);
140158

141159
// Add 3 keys
142-
assert!(handle.check_and_add(key1).await.unwrap());
143-
assert!(handle.check_and_add(key2).await.unwrap());
144-
assert!(handle.check_and_add(key3).await.unwrap());
160+
assert!(handle.check_and_add(key1, None).await.unwrap());
161+
assert!(handle.check_and_add(key2, None).await.unwrap());
162+
assert!(handle.check_and_add(key3, None).await.unwrap());
145163

146164
// Add a 4th key, should evict the oldest (key1)
147-
assert!(handle.check_and_add(key4).await.unwrap());
165+
assert!(handle.check_and_add(key4, None).await.unwrap());
148166

149167
// Now key1 should be addable again (was evicted)
150-
assert!(handle.check_and_add(key1).await.unwrap());
168+
assert!(handle.check_and_add(key1, None).await.unwrap());
151169

152170
// But key2 should now be evicted
153-
assert!(handle.check_and_add(key2).await.unwrap());
171+
assert!(handle.check_and_add(key2, None).await.unwrap());
154172
}
155173
}

‎bottlecap/src/traces/trace_agent.rs‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ use tokio::sync::{
1919
};
2020
use tokio_util::sync::CancellationToken;
2121
use tower_http::limit::RequestBodyLimitLayer;
22-
use tracing::{debug, error};
22+
use tracing::{debug, error, warn};
2323

2424
use crate::traces::trace_processor::SendingTraceProcessor;
2525
use crate::{
@@ -545,7 +545,8 @@ impl TraceAgent {
545545
for mut span in original_chunk {
546546
// Check for duplicates
547547
let key = DedupKey::new(span.trace_id, span.span_id);
548-
let should_keep = match deduper.check_and_add(key).await {
548+
let should_keep = match deduper.check_and_add(key, config.span_dedup_timeout).await
549+
{
549550
Ok(should_keep) => {
550551
if !should_keep {
551552
debug!(
@@ -556,7 +557,7 @@ impl TraceAgent {
556557
should_keep
557558
}
558559
Err(e) => {
559-
error!("Failed to check span in deduper, keeping span: {e}");
560+
warn!("Failed to check span in deduper, keeping span: {e}");
560561
true
561562
}
562563
};

‎integration-tests/tests/lmi.test.ts‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,12 +67,23 @@ describe('LMI Integration Tests', () => {
6767
expect(awsLambdaSpan?.attributes.custom.init_type).toBe('lambda-managed-instances')
6868
})
6969
// SVLS-8232
70-
test.skip('aws.lambda.span should have cold_start set to false', () => {
70+
it('aws.lambda.span should have cold_start set to false if no cold_start operation', () => {
7171
const trace = results[runtime].traces![0];
72+
73+
// The presence of 'aws.lambda.cold_start' span indicates a cold start
74+
const coldStartSpan = trace.spans.find((span: any) =>
75+
span.attributes.operation_name === 'aws.lambda.cold_start'
76+
);
77+
7278
const awsLambdaSpan = trace.spans.find((span: any) =>
7379
span.attributes.operation_name === 'aws.lambda'
7480
);
75-
expect(awsLambdaSpan?.attributes.custom.cold_start).toBe('false')
81+
expect(awsLambdaSpan).toBeDefined(); // Ensure span exists before checking attributes
82+
83+
// Only check cold_start attribute when no dedicated cold_start span exists
84+
if (!coldStartSpan) {
85+
expect(awsLambdaSpan?.attributes.custom.cold_start).toBe('false');
86+
}
7687
})
7788

7889
});

0 commit comments

Comments
 (0)