diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5a1ba7456c9..6d683e09bb4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,20 +1,6 @@ -include: - - local: ".gitlab/one-pipeline.locked.yml" - - local: ".gitlab/benchmarks.yml" - - local: ".gitlab/macrobenchmarks.yml" - - local: ".gitlab/exploration-tests.yml" - - local: ".gitlab/ci-visibility-tests.yml" - stages: - build - - publish - - shared-pipeline - - benchmarks - - macrobenchmarks - tests - - exploration-tests - - ci-visibility-tests - - generate-signing-key variables: # Gitlab runner features; see https://docs.gitlab.com/runner/configuration/feature-flags.html @@ -30,7 +16,7 @@ variables: GRADLE_PLUGIN_PROXY: "https://depot-read-api-java.us1.ddbuild.io/magicmirror/magicmirror/@current/" BUILDER_IMAGE_VERSION_PREFIX: "v25.09-" # use either an empty string (e.g. "") for latest images or a version followed by a hyphen (e.g. "v25.05-") REPO_NOTIFICATION_CHANNEL: "#apm-java-escalations" - DEFAULT_TEST_JVMS: /^(8|11|17|21|25)$/ # the latest "stable" version is LTS v25 + DEFAULT_TEST_JVMS: /^(8|11|17|21|25)$/ PROFILE_TESTS: description: "Enable profiling of tests" value: "false" @@ -40,6 +26,9 @@ variables: RUN_FLAKY_TESTS: description: "Enable flaky tests" value: "false" + TEST_ITERATIONS: + description: "Number of times to run tests (for flaky test investigation)" + value: "50" # trigger new commit cancel workflow: @@ -55,47 +44,12 @@ workflow: - when: always .test_matrix: &test_matrix - - testJvm: &test_jvms + - testJvm: - "8" - "11" - "17" - "21" - "25" - - "semeru11" - - "oracle8" - - "zulu8" - - "semeru8" - - "ibm8" - - "zulu11" - - "semeru17" - # - "stable" - CI_SPLIT: ["1/1"] - -# Gitlab doesn't support "parallel" and "parallel:matrix" at the same time -# These blocks emulate "parallel" by including it in the matrix -.test_matrix_2: &test_matrix_2 - - testJvm: *test_jvms - CI_SPLIT: ["1/2", "2/2"] - -.test_matrix_4: &test_matrix_4 - - testJvm: *test_jvms - CI_SPLIT: ["1/4", "2/4", "3/4", "4/4"] - -.test_matrix_6: &test_matrix_6 - - testJvm: *test_jvms - CI_SPLIT: ["1/6", "2/6", "3/6", "4/6", "5/6", "6/6"] - -.test_matrix_8: &test_matrix_8 - - testJvm: *test_jvms - CI_SPLIT: ["1/8", "2/8", "3/8", "4/8", "5/8", "6/8", "7/8", "8/8"] - -.test_matrix_12: &test_matrix_12 - - testJvm: *test_jvms - CI_SPLIT: [ "1/12", "2/12", "3/12", "4/12", "5/12", "6/12", "7/12", "8/12", "9/12", "10/12", "11/12", "12/12" ] - -.master_only: &master_only - - if: $CI_COMMIT_BRANCH == "master" - when: on_success default: tags: [ "arch:amd64" ] @@ -195,60 +149,22 @@ default: after_script: - *cgroup_info -# Check and fail early if maven central credentials are incorrect. When a new token is generated -# on the central publisher portal, it invalidates the old one. This check prevents going further. -# See https://datadoghq.atlassian.net/wiki/x/Oog5OgE -maven-central-pre-release-check: - image: ghcr.io/datadog/dd-trace-java-docker-build:${BUILDER_IMAGE_VERSION_PREFIX}base - stage: .pre - rules: - - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/' - when: on_success - allow_failure: false - script: - - | - MAVEN_CENTRAL_USERNAME=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_username --with-decryption --query "Parameter.Value" --out text) - MAVEN_CENTRAL_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_password --with-decryption --query "Parameter.Value" --out text) - # See https://central.sonatype.org/publish/publish-portal-api/ - # 15e0cbbb-deff-421e-9e02-296a24d0cada is deployment, any deployment id listed in central work, the idea is to check whether the token can authenticate - curl --request POST --include --fail https://central.sonatype.com/api/v1/publisher/status?id=15e0cbbb-deff-421e-9e02-296a24d0cada --header "Authorization: Bearer $(printf "$MAVEN_CENTRAL_USERNAME:$MAVEN_CENTRAL_PASSWORD" | base64)" - if [ $? -ne 0 ]; then - echo "Failed to authenticate against central. Check credentials, see https://datadoghq.atlassian.net/wiki/x/Oog5OgE" - exit 1 - fi +# In Gitlab, DD_* variables are set because the build runner is instrumented with Datadog telemetry +# To have a pristine environment for the tests, these variables are saved before the test run and restored afterwards +.prepare_test_env: &prepare_test_env + - export gitlabVariables=("DD_SERVICE" "DD_ENTITY_ID" "DD_SITE" "DD_ENV" "DD_DATACENTER" "DD_PARTITION" "DD_CLOUDPROVIDER") + - '[ ! -e pretest.env ] || rm pretest.env' + - | + for VARIABLE in "${gitlabVariables[@]}" + do + echo "export $VARIABLE=${!VARIABLE}" >> pretest.env + unset "$VARIABLE" + done -dd-octo-sts-pre-release-check: - image: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1 - stage: .pre - tags: [ "arch:amd64" ] - id_tokens: - DDOCTOSTS_ID_TOKEN: - aud: dd-octo-sts - rules: - - if: '$POPULATE_CACHE' - when: never - - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/' - when: on_success - allow_failure: false - before_script: - - dd-octo-sts version - - dd-octo-sts debug --scope DataDog/dd-trace-java --policy self.gitlab.release - - dd-octo-sts token --scope DataDog/dd-trace-java --policy self.gitlab.release > test-github-token.txt - script: - - gh auth login --with-token < test-github-token.txt - - gh auth status - after_script: - - dd-octo-sts revoke -t $(cat test-github-token.txt) - retry: - max: 2 - when: always +.restore_pretest_env: &restore_pretest_env + - source pretest.env build: - needs: - - job: maven-central-pre-release-check - optional: true - - job: dd-octo-sts-pre-release-check - optional: true extends: .gradle_build variables: BUILD_CACHE_POLICY: push @@ -280,238 +196,16 @@ build_tests: GRADLE_WORKERS: 3 KUBERNETES_MEMORY_REQUEST: 18Gi KUBERNETES_MEMORY_LIMIT: 18Gi - parallel: - matrix: - - GRADLE_TARGET: ":baseTest" - CACHE_TYPE: "base" - - GRADLE_TARGET: ":profilingTest" - CACHE_TYPE: "profiling" - - GRADLE_TARGET: ":instrumentationTest" - CACHE_TYPE: "inst" - - GRADLE_TARGET: ":instrumentationLatestDepTest" - CACHE_TYPE: "latestdep" - - GRADLE_TARGET: ":smokeTest" - CACHE_TYPE: "smoke" - MAVEN_OPTS: "-Xms64M -Xmx512M -Dorg.slf4j.simpleLogger.defaultLogLevel=debug" # FIXME: Build :smokeTest build fails unless mvn debug logging is on - + GRADLE_TARGET: ":instrumentationTest" + CACHE_TYPE: "inst" script: - *gitlab_base_ref_params - ./gradlew clean $GRADLE_TARGET $GRADLE_PARAMS -PskipTests $GRADLE_ARGS -populate_dep_cache: - extends: build_tests - variables: - BUILD_CACHE_POLICY: pull - DEPENDENCY_CACHE_POLICY: push - rules: - - if: '$POPULATE_CACHE' - when: on_success - - when: manual - allow_failure: true - parallel: - matrix: - - GRADLE_TARGET: ":dd-java-agent:shadowJar :dd-trace-api:jar :dd-trace-ot:shadowJar" - CACHE_TYPE: "lib" - - GRADLE_TARGET: ":baseTest" - CACHE_TYPE: "base" - - GRADLE_TARGET: ":profilingTest" - CACHE_TYPE: "profiling" -# FIXME: Gitlab doesn't support s3 based caches >5GB. Fixed in Gitlab 17.5 -# See: https://gitlab.com/gitlab-org/gitlab-runner/-/issues/26921#note_2132307223 -# - GRADLE_TARGET: ":instrumentationTest" -# CACHE_TYPE: "inst" -# - GRADLE_TARGET: ":instrumentationLatestDepTest" -# CACHE_TYPE: "latestdep" -# - GRADLE_TARGET: ":smokeTest" -# CACHE_TYPE: "smoke" - -publish-artifacts-to-s3: - image: registry.ddbuild.io/images/mirror/amazon/aws-cli:2.4.29 - stage: publish - needs: [ build ] - script: - - source upstream.env - - export VERSION="${UPSTREAM_TRACER_VERSION%~*}" # remove ~githash from the end of version - - aws s3 cp workspace/dd-java-agent/build/libs/dd-java-agent-${VERSION}.jar s3://dd-trace-java-builds/${CI_COMMIT_REF_NAME}/dd-java-agent.jar - - aws s3 cp workspace/dd-trace-api/build/libs/dd-trace-api-${VERSION}.jar s3://dd-trace-java-builds/${CI_COMMIT_REF_NAME}/dd-trace-api.jar - - aws s3 cp workspace/dd-trace-ot/build/libs/dd-trace-ot-${VERSION}.jar s3://dd-trace-java-builds/${CI_COMMIT_REF_NAME}/dd-trace-ot.jar - - aws s3 cp workspace/dd-java-agent/build/libs/dd-java-agent-${VERSION}.jar s3://dd-trace-java-builds/${CI_PIPELINE_ID}/dd-java-agent.jar - - aws s3 cp workspace/dd-trace-api/build/libs/dd-trace-api-${VERSION}.jar s3://dd-trace-java-builds/${CI_PIPELINE_ID}/dd-trace-api.jar - - aws s3 cp workspace/dd-trace-ot/build/libs/dd-trace-ot-${VERSION}.jar s3://dd-trace-java-builds/${CI_PIPELINE_ID}/dd-trace-ot.jar - - | - cat << EOF > links.json - { - "S3 Links": [ - { - "external_link": { - "label": "Public Link to dd-java-agent.jar", - "url": "https://s3.us-east-1.amazonaws.com/dd-trace-java-builds/${CI_PIPELINE_ID}/dd-java-agent.jar" - } - } - ] - } - EOF - artifacts: - reports: - annotations: - - links.json - - -spotless: - extends: .gradle_build - stage: tests - needs: [] - script: - - export JAVA_HOME=$JAVA_11_HOME - - ./gradlew spotlessCheck $GRADLE_ARGS - -test_published_artifacts: - extends: .gradle_build - image: ghcr.io/datadog/dd-trace-java-docker-build:${BUILDER_IMAGE_VERSION_PREFIX}7 # Needs Java7 for some tests - stage: tests - needs: [ build ] - variables: - CACHE_TYPE: lib - script: - - mvn_local_repo=$(./mvnw help:evaluate -Dexpression=settings.localRepository -q -DforceStdout) - - rm -rf "${mvn_local_repo}/com/datadoghq" - - export GPG_PRIVATE_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_private_key --with-decryption --query "Parameter.Value" --out text) - - export GPG_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_passphrase --with-decryption --query "Parameter.Value" --out text) - - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx2G -Xms2G -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'" - - ./gradlew publishToMavenLocal $GRADLE_ARGS - - cd test-published-dependencies - - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx1G -Xms1G -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'" - - ./gradlew check --info $GRADLE_ARGS - after_script: - - *cgroup_info - - source .gitlab/gitlab-utils.sh - - gitlab_section_start "collect-reports" "Collecting reports" - - .gitlab/collect_reports.sh - - gitlab_section_end "collect-reports" - artifacts: - when: always - paths: - - ./check_reports - -.check_job: - extends: .gradle_build - needs: [ build ] - stage: tests - variables: - CACHE_TYPE: lib - script: - - *gitlab_base_ref_params - - ./gradlew $GRADLE_TARGET $GRADLE_PARAMS -PskipTests -PrunBuildSrcTests -PskipSpotless -PtaskPartitionCount=$NORMALIZED_NODE_TOTAL -PtaskPartition=$NORMALIZED_NODE_INDEX $GRADLE_ARGS - after_script: - - *cgroup_info - - source .gitlab/gitlab-utils.sh - - gitlab_section_start "collect-reports" "Collecting reports" - - .gitlab/collect_reports.sh --destination ./check_reports --move - - gitlab_section_end "collect-reports" - artifacts: - when: always - paths: - - ./check_reports - - '.gradle/daemon/*/*.out.log' - retry: - max: 2 - when: - - unknown_failure - - stuck_or_timeout_failure - - runner_system_failure - - unmet_prerequisites - - scheduler_failure - - data_integrity_failure - -check_base: - extends: .check_job - variables: - GRADLE_TARGET: ":baseCheck" - -check_inst: - extends: .check_job - parallel: 4 - variables: - GRADLE_TARGET: ":instrumentationCheck" - -check_smoke: - extends: .check_job - parallel: 4 - variables: - GRADLE_TARGET: ":smokeCheck" - -check_profiling: - extends: .check_job - variables: - GRADLE_TARGET: ":profilingCheck" - -check_debugger: - extends: .check_job - variables: - GRADLE_TARGET: ":debuggerCheck" - -muzzle: - extends: .gradle_build - needs: [ build_tests ] - stage: tests - parallel: - matrix: - - CI_SPLIT: ["1/8", "2/8", "3/8", "4/8", "5/8", "6/8", "7/8", "8/8"] - variables: - CACHE_TYPE: inst - script: - - export SKIP_BUILDSCAN="true" - - ./gradlew :runMuzzle -PtaskPartitionCount=$NORMALIZED_NODE_TOTAL -PtaskPartition=$NORMALIZED_NODE_INDEX $GRADLE_ARGS - after_script: - - *cgroup_info - - source .gitlab/gitlab-utils.sh - - gitlab_section_start "collect-reports" "Collecting reports" - - .gitlab/collect_reports.sh - - gitlab_section_end "collect-reports" - artifacts: - when: always - paths: - - ./reports - - '.gradle/daemon/*/*.out.log' - -muzzle-dep-report: - extends: .gradle_build - needs: [ build_tests ] - stage: tests - variables: - CACHE_TYPE: inst - script: - - export SKIP_BUILDSCAN="true" - - ./gradlew generateMuzzleReport muzzleInstrumentationReport $GRADLE_ARGS - after_script: - - *cgroup_info - - .gitlab/collect_muzzle_deps.sh - artifacts: - when: always - paths: - - ./reports - - '.gradle/daemon/*/*.out.log' - -# In Gitlab, DD_* variables are set because the build runner is instrumented with Datadog telemetry -# To have a pristine environment for the tests, these variables are saved before the test run and restored afterwards -.prepare_test_env: &prepare_test_env - - export gitlabVariables=("DD_SERVICE" "DD_ENTITY_ID" "DD_SITE" "DD_ENV" "DD_DATACENTER" "DD_PARTITION" "DD_CLOUDPROVIDER") - - '[ ! -e pretest.env ] || rm pretest.env' - - | - for VARIABLE in "${gitlabVariables[@]}" - do - echo "export $VARIABLE=${!VARIABLE}" >> pretest.env - unset "$VARIABLE" - done - -.restore_pretest_env: &restore_pretest_env - - source pretest.env - -.test_job: +test_all: extends: .gradle_build image: ghcr.io/datadog/dd-trace-java-docker-build:${BUILDER_IMAGE_VERSION_PREFIX}$testJvm tags: [ "docker-in-docker:amd64" ] # use docker-in-docker runner for testcontainers - needs: [ build_tests ] stage: tests variables: KUBERNETES_MEMORY_REQUEST: 17Gi @@ -519,12 +213,13 @@ muzzle-dep-report: KUBERNETES_CPU_REQUEST: 10 GRADLE_WORKERS: 4 GRADLE_MEM: 3G - GRADLE_PARAMS: "-PskipFlakyTests" - CONTINUE_ON_FAILURE: "false" + CACHE_TYPE: "inst" + CONTINUE_ON_FAILURE: "true" TESTCONTAINERS_CHECKS_DISABLE: "true" TESTCONTAINERS_RYUK_DISABLED: "true" TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX: "registry.ddbuild.io/images/mirror/" JETTY_AVAILABLE_PROCESSORS: 4 # Jetty incorrectly calculates processor count in containers + needs: [ build_tests ] rules: - if: $testJvm =~ $DEFAULT_TEST_JVMS when: on_success @@ -532,16 +227,51 @@ muzzle-dep-report: when: on_success - if: $CI_COMMIT_BRANCH == "master" when: on_success + parallel: + matrix: *test_matrix script: - *gitlab_base_ref_params - - > - if [ "$PROFILE_TESTS" == "true" ] && [ "$testJvm" != "ibm8" ] && [ "$testJvm" != "oracle8" ]; - then - export PROFILER_COMMAND="-XX:StartFlightRecording=settings=profile,filename=/tmp/${CI_JOB_NAME_SLUG}.jfr,dumponexit=true"; - fi - *prepare_test_env - - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xms$GRADLE_MEM -Xmx$GRADLE_MEM $PROFILER_COMMAND -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp' -Ddatadog.forkedMaxHeapSize=1024M -Ddatadog.forkedMinHeapSize=128M" - - ./gradlew $GRADLE_TARGET $GRADLE_PARAMS -PtestJvm=$testJvm -PtaskPartitionCount=$NORMALIZED_NODE_TOTAL -PtaskPartition=$NORMALIZED_NODE_INDEX $GRADLE_ARGS --continue || $CONTINUE_ON_FAILURE + - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xms$GRADLE_MEM -Xmx$GRADLE_MEM -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp' -Ddatadog.forkedMaxHeapSize=1024M -Ddatadog.forkedMinHeapSize=128M" + - echo "=== SETUP, Pre-compiling all test dependencies ===" + - ./gradlew :dd-java-agent:instrumentation:pekko:pekko-http-1.0:testClasses :dd-java-agent:instrumentation:pekko:pekko-http-1.0:baseTestClasses -PtestJvm=$testJvm --no-daemon --build-cache --stacktrace --parallel --max-workers=$GRADLE_WORKERS --configure-on-demand + - echo "=== SETUP COMPLETE, Starting ${TEST_ITERATIONS} test iterations ===" + - | + TOTAL_RUNS=${TEST_ITERATIONS} + PASSED=0 + FAILED=0 + mkdir -p flaky_test_results + + for i in $(seq 1 $TOTAL_RUNS); do + echo "=== Running iteration $i/$TOTAL_RUNS ===" + + ./gradlew :dd-java-agent:instrumentation:pekko:pekko-http-1.0:baseTest -PtestJvm=$testJvm --no-daemon --build-cache --stacktrace --parallel --max-workers=$GRADLE_WORKERS --configure-on-demand --quiet > flaky_test_results/iteration_${i}_base.log 2>&1 & + BASE_PID=$! + ./gradlew :dd-java-agent:instrumentation:pekko:pekko-http-1.0:forkedTest -PtestJvm=$testJvm --no-daemon --build-cache --stacktrace --parallel --max-workers=$GRADLE_WORKERS --configure-on-demand --quiet > flaky_test_results/iteration_${i}_forked.log 2>&1 & + FORKED_PID=$! + + if wait $BASE_PID && wait $FORKED_PID; then + PASSED=$((PASSED + 1)) + echo "Iteration $i: PASSED" | tee -a flaky_test_results/summary.log + else + FAILED=$((FAILED + 1)) + echo "Iteration $i: FAILED" | tee -a flaky_test_results/summary.log + fi + + echo "Progress: $PASSED passed, $FAILED failed out of $i total runs" + done + + echo "=== FINAL RESULTS ===" | tee -a flaky_test_results/summary.log + echo "Total runs: $TOTAL_RUNS" | tee -a flaky_test_results/summary.log + echo "Passed: $PASSED" | tee -a flaky_test_results/summary.log + echo "Failed: $FAILED" | tee -a flaky_test_results/summary.log + + # Exit with error code if there were any failures - this helps quickly identify problematic JVMs + if [ $FAILED -gt 0 ]; then + exit 1 + else + exit 0 + fi after_script: - *restore_pretest_env - *set_datadog_api_keys @@ -549,7 +279,6 @@ muzzle-dep-report: - source .gitlab/gitlab-utils.sh - gitlab_section_start "collect-reports" "Collecting reports" - .gitlab/collect_reports.sh - - if [ "$PROFILE_TESTS" == "true" ]; then .gitlab/collect_profiles.sh; fi - .gitlab/collect_results.sh - .gitlab/upload_ciapp.sh $CACHE_TYPE $testJvm - gitlab_section_end "collect-reports" @@ -561,413 +290,7 @@ muzzle-dep-report: - ./reports.tar - ./profiles.tar - ./results + - ./flaky_test_results/ - '.gradle/daemon/*/*.out.log' reports: junit: results/*.xml - retry: - max: 2 - when: - - unknown_failure - - stuck_or_timeout_failure - - runner_system_failure - - unmet_prerequisites - - scheduler_failure - - data_integrity_failure - -.test_job_with_test_agent: - extends: .test_job - variables: - CI_USE_TEST_AGENT: "true" - CI_AGENT_HOST: local-agent - services: - - name: ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.33.1 - alias: local-agent - variables: - LOG_LEVEL: "DEBUG" - TRACE_LANGUAGE: "java" - DD_SUPPRESS_TRACE_PARSE_ERRORS: "true" - DD_POOL_TRACE_CHECK_FAILURES: "true" - DD_DISABLE_ERROR_RESPONSES: "true" - ENABLED_CHECKS: "trace_content_length,trace_stall,meta_tracer_version_header,trace_count_header,trace_peer_service,trace_dd_service" - script: - - !reference [.test_job, script] - - .gitlab/check_test_agent_results.sh - -agent_integration_tests: - extends: .test_job - tags: [ "arch:amd64" ] - variables: - testJvm: "8" - CI_AGENT_HOST: local-agent - GRADLE_TARGET: "traceAgentTest" - CACHE_TYPE: "base" - services: - - name: registry.ddbuild.io/images/mirror/datadog/agent:7.40.1 - alias: local-agent - variables: - DD_APM_ENABLED: "true" - DD_BIND_HOST: "0.0.0.0" - DD_HOSTNAME: "local-agent" - DD_API_KEY: "invalid_key_but_this_is_fine" - -test_base: - extends: .test_job - variables: - GRADLE_TARGET: ":baseTest" - CACHE_TYPE: "base" - parallel: - matrix: *test_matrix_4 - script: - - if [ "$testJvm" == "8" ]; then export GRADLE_PARAMS="-PskipFlakyTests -PcheckCoverage"; fi - - !reference [.test_job, script] - -test_inst: - extends: .test_job_with_test_agent - variables: - GRADLE_TARGET: ":instrumentationTest" - CACHE_TYPE: "inst" - parallel: - matrix: *test_matrix_6 - -test_inst_latest: - extends: .test_job_with_test_agent - variables: - GRADLE_TARGET: ":instrumentationLatestDepTest" - CACHE_TYPE: "latestDep" - parallel: - matrix: - - testJvm: ["8", "17", "21", "25"] # the latest "stable" version is LTS v25 - # Gitlab doesn't support "parallel" and "parallel:matrix" at the same time - # This emulates "parallel" by including it in the matrix - CI_SPLIT: [ "1/6", "2/6", "3/6", "4/6", "5/6", "6/6"] - -test_flaky: - extends: .test_job_with_test_agent - variables: - GRADLE_PARAMS: "-PrunFlakyTests" - CACHE_TYPE: "base" - testJvm: "8" - CONTINUE_ON_FAILURE: "true" - rules: - - *master_only - - if: $RUN_FLAKY_TESTS == "true" - when: on_success - parallel: - matrix: - - GRADLE_TARGET: [":baseTest", ":smokeTest", ":debuggerTest"] - # Gitlab doesn't support "parallel" and "parallel:matrix" at the same time - # This emulates "parallel" by including it in the matrix - CI_SPLIT: [ "1/4", "2/4", "3/4", "4/4" ] - -test_flaky_inst: - extends: .test_job - variables: - GRADLE_TARGET: ":instrumentationTest" - GRADLE_PARAMS: "-PrunFlakyTests" - CACHE_TYPE: "inst" - testJvm: "8" - CONTINUE_ON_FAILURE: "true" - rules: - - *master_only - - if: $RUN_FLAKY_TESTS == "true" - when: on_success - parallel: 6 - -test_profiling: - extends: .test_job - variables: - GRADLE_TARGET: ":profilingTest" - CACHE_TYPE: "profiling" - parallel: - matrix: *test_matrix - -# specific jvms list for debugger project because J9-based JVMs have issues with local vars -# so need to test at least against one J9-based JVM -test_debugger: - extends: .test_job - variables: - GRADLE_TARGET: ":debuggerTest" - CACHE_TYPE: "base" - DEFAULT_TEST_JVMS: /^(8|11|17|21|25|semeru8)$/ # the latest "stable" version is LTS v25 - parallel: - matrix: *test_matrix - -test_smoke: - extends: .test_job - variables: - GRADLE_TARGET: "stageMainDist :smokeTest" - GRADLE_PARAMS: "-PskipFlakyTests" - CACHE_TYPE: "smoke" - parallel: - matrix: *test_matrix_4 - -test_ssi_smoke: - extends: .test_job - rules: *master_only - variables: - GRADLE_TARGET: "stageMainDist :smokeTest" - CACHE_TYPE: "smoke" - DD_INJECT_FORCE: "true" - DD_INJECTION_ENABLED: "tracer" - parallel: - matrix: *test_matrix_4 - -test_smoke_graalvm: - extends: .test_job - tags: [ "arch:amd64" ] - variables: - GRADLE_TARGET: "stageMainDist :dd-smoke-test:spring-boot-3.0-native:test" - CACHE_TYPE: "smoke" - CI_NO_SPLIT: "true" - NON_DEFAULT_JVMS: "true" - parallel: - matrix: - - testJvm: ["graalvm17", "graalvm21"] - -test_smoke_semeru8_debugger: - extends: .test_job - tags: [ "arch:amd64" ] - variables: - GRADLE_TARGET: "stageMainDist dd-smoke-tests:debugger-integration-tests:test" - CACHE_TYPE: "smoke" - NON_DEFAULT_JVMS: "true" - testJvm: "semeru8" - -deploy_to_profiling_backend: - stage: publish - needs: [ build ] - rules: - - if: '$POPULATE_CACHE' - when: never - - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' - when: on_success - - if: '$CI_COMMIT_TAG =~ /^v.*/' - when: on_success - - when: manual - allow_failure: true - trigger: - project: DataDog/profiling-backend - branch: dogfooding - variables: - UPSTREAM_PACKAGE_JOB: $BUILD_JOB_NAME - UPSTREAM_PACKAGE_JOB_ID: $BUILD_JOB_ID - UPSTREAM_PROJECT_ID: $CI_PROJECT_ID - UPSTREAM_PROJECT_NAME: $CI_PROJECT_NAME - UPSTREAM_PIPELINE_ID: $CI_PIPELINE_ID - UPSTREAM_BRANCH: $CI_COMMIT_BRANCH - UPSTREAM_TAG: $CI_COMMIT_TAG - -trigger_tibco_tests: - stage: tests - needs: [ build ] - rules: - - if: '$POPULATE_CACHE' - when: never - - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' - when: on_success - - if: '$CI_COMMIT_TAG =~ /^v.*/' - when: on_success - - when: manual - allow_failure: true - trigger: - project: DataDog/tibco-testing - branch: main - strategy: depend - variables: - UPSTREAM_PACKAGE_JOB: $BUILD_JOB_NAME - UPSTREAM_PACKAGE_JOB_ID: $BUILD_JOB_ID - UPSTREAM_PROJECT_ID: $CI_PROJECT_ID - UPSTREAM_PROJECT_NAME: $CI_PROJECT_NAME - UPSTREAM_PIPELINE_ID: $CI_PIPELINE_ID - UPSTREAM_BRANCH: $CI_COMMIT_BRANCH - UPSTREAM_TAG: $CI_COMMIT_TAG - FORCE_TRIGGER: $FORCE_TRIGGER - -deploy_to_di_backend:manual: - stage: publish - needs: [ build ] - rules: - - if: '$POPULATE_CACHE' - when: never - - when: manual - allow_failure: true - trigger: - project: DataDog/debugger-demos - branch: main - variables: - UPSTREAM_PACKAGE_JOB: build - UPSTREAM_PROJECT_ID: $CI_PROJECT_ID - UPSTREAM_PROJECT_NAME: $CI_PROJECT_NAME - UPSTREAM_PIPELINE_ID: $CI_PIPELINE_ID - UPSTREAM_BRANCH: $CI_COMMIT_BRANCH - UPSTREAM_TAG: $CI_COMMIT_TAG - UPSTREAM_COMMIT_AUTHOR: $CI_COMMIT_AUTHOR - UPSTREAM_COMMIT_SHORT_SHA: $CI_COMMIT_SHORT_SHA - -# If the deploy_to_maven_central job is re-run, re-trigger the deploy_artifacts_to_github job as well so that the artifacts match. -deploy_to_maven_central: - extends: .gradle_build - stage: publish - needs: [ build ] - variables: - CACHE_TYPE: lib - rules: - - if: '$POPULATE_CACHE' - when: never - - if: '$CI_COMMIT_BRANCH == "master"' - when: on_success - # Do not deploy release candidate versions - - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/' - when: on_success - - when: manual - allow_failure: true - script: - - export MAVEN_CENTRAL_USERNAME=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_username --with-decryption --query "Parameter.Value" --out text) - - export MAVEN_CENTRAL_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_password --with-decryption --query "Parameter.Value" --out text) - - export GPG_PRIVATE_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_private_key --with-decryption --query "Parameter.Value" --out text) - - export GPG_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_passphrase --with-decryption --query "Parameter.Value" --out text) - - ./gradlew -PbuildInfo.build.number=$CI_JOB_ID publishToSonatype closeSonatypeStagingRepository -PskipTests $GRADLE_ARGS - artifacts: - paths: - - 'workspace/dd-java-agent/build/libs/*.jar' - - 'workspace/dd-trace-api/build/libs/*.jar' - - 'workspace/dd-trace-ot/build/libs/*.jar' - -deploy_artifacts_to_github: - stage: publish - image: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1 - tags: [ "arch:amd64" ] - id_tokens: - DDOCTOSTS_ID_TOKEN: - aud: dd-octo-sts - rules: - - if: '$POPULATE_CACHE' - when: never - - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/' - when: on_success - # Requires the deploy_to_maven_central job to have run first (the UP-TO-DATE gradle check across jobs is broken) - # This will deploy the artifacts built from the publishToSonatype task to the GitHub release - needs: - - job: deploy_to_maven_central - # The deploy_to_maven_central job is not run for release candidate versions - optional: true - before_script: - - dd-octo-sts version - - dd-octo-sts debug --scope DataDog/dd-trace-java --policy self.gitlab.release - - dd-octo-sts token --scope DataDog/dd-trace-java --policy self.gitlab.release > github-token.txt - script: - - gh auth login --with-token < github-token.txt - - gh auth status - - export VERSION=${CI_COMMIT_TAG##v} # remove "v" from front of tag to get version - - cp workspace/dd-java-agent/build/libs/dd-java-agent-${VERSION}.jar workspace/dd-java-agent/build/libs/dd-java-agent.jar # upload two filenames - - gh release upload --clobber --repo DataDog/dd-trace-java $CI_COMMIT_TAG workspace/dd-java-agent/build/libs/dd-java-agent.jar - - gh release upload --clobber --repo DataDog/dd-trace-java $CI_COMMIT_TAG workspace/dd-java-agent/build/libs/dd-java-agent-${VERSION}.jar - - gh release upload --clobber --repo DataDog/dd-trace-java $CI_COMMIT_TAG workspace/dd-trace-api/build/libs/dd-trace-api-${VERSION}.jar - - gh release upload --clobber --repo DataDog/dd-trace-java $CI_COMMIT_TAG workspace/dd-trace-ot/build/libs/dd-trace-ot-${VERSION}.jar - after_script: - - dd-octo-sts revoke -t $(cat github-token.txt) - retry: - max: 2 - when: always - -requirements_json_test: - rules: - - when: on_success - variables: - REQUIREMENTS_BLOCK_JSON_PATH: "metadata/requirements-block.json" - REQUIREMENTS_ALLOW_JSON_PATH: "metadata/requirements-allow.json" - -package-oci: - needs: [ build ] - -override_verify_maven_central: - image: registry.ddbuild.io/images/base/gbi-ubuntu_2204:release - stage: publish - needs: [ ] - rules: - - if: '$POPULATE_CACHE' - when: never - - when: manual - allow_failure: true - script: - - touch OVERRIDE_MAVEN_VERIFY - cache: # Cache is used to signal between the override_verify_maven_central and verify_maven_central_deployment jobs - - key: $CI_PIPELINE_ID-OVERRIDE_SIGNAL - paths: - - OVERRIDE_MAVEN_VERIFY - policy: push - unprotect: true - -# Verify Maven Central deployment is publicly available before publishing OCI images -verify_maven_central_deployment: - image: registry.ddbuild.io/images/base/gbi-ubuntu_2204:release - stage: publish - needs: [ deploy_to_maven_central ] - rules: - - if: '$POPULATE_CACHE' - when: never - - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/' - when: on_success - cache: # Cache is used to signal between the override_verify_maven_central and verify_maven_central_deployment jobs - - key: $CI_PIPELINE_ID-OVERRIDE_SIGNAL - paths: - - OVERRIDE_MAVEN_VERIFY - policy: pull - unprotect: true - script: - - if [ -f OVERRIDE_MAVEN_VERIFY ]; then echo "SKIPPING MAVEN VERIFICATION"; exit 0; fi - - | - export VERSION=${CI_COMMIT_TAG##v} - ARTIFACT_URLS=( - "https://repo1.maven.org/maven2/com/datadoghq/dd-java-agent/${VERSION}/dd-java-agent-${VERSION}.jar" - "https://repo1.maven.org/maven2/com/datadoghq/dd-trace-api/${VERSION}/dd-trace-api-${VERSION}.jar" - "https://repo1.maven.org/maven2/com/datadoghq/dd-trace-ot/${VERSION}/dd-trace-ot-${VERSION}.jar" - ) - # Wait 5 mins initially, then try 5 times with a minute delay between each retry to see if the release artifacts are available - sleep 300 - TRY=0 - MAX_TRIES=5 - DELAY=60 - while [ $TRY -lt $MAX_TRIES ]; do - ARTIFACTS_AVAILABLE=true - for URL in "${ARTIFACT_URLS[@]}"; do - if ! curl --location --fail --silent --show-error -I "$URL"; then - ARTIFACTS_AVAILABLE=false - break - fi - done - if [ "$ARTIFACTS_AVAILABLE" = true ]; then - break - fi - TRY=$((TRY + 1)) - if [ $TRY -eq $MAX_TRIES ]; then - echo "The release was not available after 10 mins. Manually re-run the job to try again." - exit 1 - fi - sleep $DELAY - done - -publishing-gate: - stage: publish - needs: - - job: verify_maven_central_deployment - optional: true # Required for releases only - -configure_system_tests: - variables: - SYSTEM_TESTS_SCENARIOS_GROUPS: "simple_onboarding,simple_onboarding_profiling,simple_onboarding_appsec,docker-ssi,lib-injection" - -create_key: - stage: generate-signing-key - when: manual - needs: [ ] - variables: - PROJECT_NAME: "dd-trace-java" - EXPORT_TO_KEYSERVER: "true" - image: $REGISTRY/ci/agent-key-management-tools/gpg:1 - script: - - /create.sh - artifacts: - expire_in: 13 mos - paths: - - pubkeys