Skip to content

Commit 2812c6f

Browse files
olegsuolegsu
authored
feat(agentless): add agentless release pipelines (#7701)
1 parent 0d0e70d commit 2812c6f

8 files changed

+380
-1
lines changed

‎.buildkite/pipeline.agentless-app-release.yaml

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
env:
2+
VERSION: "${BUILDKITE_COMMIT:0:12}"
3+
steps:
4+
- label: "Mirror Elastic-Agent Snapshot DRA to internal registry"
5+
key: "mirror-elastic-agent"
6+
command: "./build/ecp-internal-release.sh"
7+
agents:
8+
image: docker.elastic.co/ci-agent-images/serverless-helm-builder:0.0.2@sha256:d00e8a7a0ab3618cfaacb0a7b1e1b06ee29728eb2b44de602374bd8f6b9b92ac
9+
10+
11+
12+
# wait for metadata to be set
13+
- wait: ~
14+
15+
- label: ":grey_question: Promote agentless app release if validation passes"
16+
depends_on: "mirror-elastic-agent"
17+
command: |
18+
export COMMIT_HASH=$$(buildkite-agent meta-data get git-short-commit)
19+
if [ $$(buildkite-agent step get "outcome" --step "mirror-elastic-agent") == "passed" ]; then
20+
cat <<- YAML | buildkite-agent pipeline upload
21+
steps:
22+
- label: ":serverless::argo: Run synthetics tests and update agentless to $${COMMIT_HASH} in serverless-gitops"
23+
async: true
24+
branches: main
25+
trigger: gpctl-promote-after-serverless-devenv-synthetics
26+
build:
27+
env:
28+
SERVICE_COMMIT_HASH: $${COMMIT_HASH}
29+
SERVICE: agentless
30+
YAML
31+
fi
32+
agents:
33+
image: docker.elastic.co/ci-agent-images/serverless-helm-builder:0.0.2@sha256:d00e8a7a0ab3618cfaacb0a7b1e1b06ee29728eb2b44de602374bd8f6b9b92ac

‎.buildkite/pipeline.agentless-tests.yaml

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# This pipeline serves as the entry point for your service's quality gates definitions. When
2+
# properly configured, it will be invoked automatically as part of the automated
3+
# promotion process once a new version was rolled out in one of the various cloud stages.
4+
#
5+
# The updated environment is provided via ENVIRONMENT variable. The seedling
6+
# step will branch and execute pipeline snippets at the following location:
7+
# .buildkite/pipeline.tests-qa.yaml
8+
# .buildkite/pipeline.tests-staging.yaml
9+
# .buildkite/pipeline.tests-production.yaml
10+
#
11+
# Docs: https://docs.elastic.dev/serverless/qualitygates
12+
13+
env:
14+
ENVIRONMENT: ${ENVIRONMENT?}
15+
TEAM_CHANNEL: "#agentless-alerts"
16+
17+
steps:
18+
- label: ":pipeline::grey_question::seedling: Trigger service tests for ${ENVIRONMENT}"
19+
command: ".buildkite/scripts/steps/run-agentless-tests.sh"
20+
agents:
21+
image: "docker.elastic.co/ci-agent-images/quality-gate-seedling:0.0.4@sha256:b15aa65183fd9ac4b3ad2b01287ee8c47382a74450485b012bade5331fefeae9"
22+
env:
23+
SERVICE_VERSION: "${BUILDKITE_COMMIT:0:12}"
24+
25+
notify:
26+
- slack: "${TEAM_CHANNEL}"
27+
if: build.branch == "main" && build.state == "failed"

‎.buildkite/pipeline.tests-production.yaml

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
# These pipeline steps constitute the quality gate for your service within the Production environment.
2+
# Incorporate any necessary additional logic to validate the service's integrity. A failure in
3+
# this pipeline build will prevent further progression to the subsequent stage.
4+
5+
steps:
6+
- command:
7+
- echo "Waiting for 10m for indicative health metrics"
8+
- sleep 600
9+
- wait
10+
- label: ":rocket: Run observability gates"
11+
if: build.env("ENVIRONMENT") == "production-canary"
12+
trigger: "serverless-quality-gates"
13+
build:
14+
branch: main
15+
commit: HEAD
16+
message: "${BUILDKITE_MESSAGE}"
17+
env:
18+
TARGET_ENV: production
19+
SERVICE: agentless-controller
20+
CONTAINER_NAME: agentless-controller
21+
CHECK_ALERTS: true
22+
CHECK_RECONCILE_ERROR_PERCENTAGE: true
23+
CHECK_CONTAINER_RESTART_COUNT: true
24+
CHECK_LOG_ERROR_RATE: true
25+
CHECK_SLO: true
26+
CHECK_SLO_TAG: agentless-controller
27+
CHECK_SLO_BURN_RATE_THRESHOLD: 0.1
28+
MAX_ERROR_PERCENT: 2
29+
SERVICE_VERSION: ${SERVICE_VERSION:0:12}
30+
CHECK_SYNTHETICS: true
31+
CHECK_SYNTHETICS_TAG: agentless
32+
CHECK_SYNTHETICS_MINIMUM_RUNS: 2
33+
CHECK_SYNTHETICS_MAX_POLL: 70
34+
CHECK_SYNTHETIC_POLL_INTERVAL: 180
35+
MAX_FAILURES: 1
36+
DEPLOYMENT_SLICES: ${DEPLOYMENT_SLICES:-""}
37+
38+
- label: ":cookie: 1h bake period before continuing promotion"
39+
if: build.env("ENVIRONMENT") == "production-canary"
40+
command: "sleep 3600"

‎.buildkite/pipeline.tests-qa.yaml

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
# These pipeline steps constitute the quality gate for your service within the QA environment.
2+
# Incorporate any necessary additional logic to validate the service's integrity. A failure in
3+
# this pipeline build will prevent further progression to the subsequent stage.
4+
steps:
5+
- command:
6+
- echo "Waiting for 10m for indicative health metrics"
7+
- sleep 600
8+
- wait
9+
- label: ":rocket: Run observability gates"
10+
trigger: "serverless-quality-gates"
11+
build:
12+
branch: main
13+
commit: HEAD
14+
message: "${BUILDKITE_MESSAGE}"
15+
env:
16+
TARGET_ENV: qa
17+
SERVICE: agentless-controller
18+
CONTAINER_NAME: agentless-controller
19+
CHECK_RECONCILE_ERROR_PERCENTAGE: true
20+
CHECK_CONTAINER_RESTART_COUNT: true
21+
CHECK_LOG_ERROR_RATE: true
22+
CHECK_SLO: true
23+
CHECK_SLO_TAG: agentless-controller
24+
CHECK_SLO_BURN_RATE_THRESHOLD: 0.1
25+
MAX_ERROR_PERCENT: 10
26+
SERVICE_VERSION: ${SERVICE_VERSION:0:12}
27+
CHECK_SYNTHETICS: true
28+
CHECK_SYNTHETICS_TAG: agentless
29+
CHECK_SYNTHETICS_MINIMUM_RUNS: 2
30+
CHECK_SYNTHETICS_MAX_POLL: 70
31+
CHECK_SYNTHETIC_POLL_INTERVAL: 180
32+
MAX_FAILURES: 1
33+
DEPLOYMENT_SLICES: ${DEPLOYMENT_SLICES:-""}

‎.buildkite/pipeline.tests-staging.yaml

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
# These pipeline steps constitute the quality gate for your service within the Staging environment.
2+
# Incorporate any necessary additional logic to validate the service's integrity. A failure in
3+
# this pipeline build will prevent further progression to the subsequent stage.
4+
steps:
5+
- command:
6+
- echo "Waiting for 10m for indicative health metrics"
7+
- sleep 600
8+
- wait
9+
- label: ":rocket: Run observability gates"
10+
trigger: "serverless-quality-gates"
11+
build:
12+
branch: main
13+
commit: HEAD
14+
message: "${BUILDKITE_MESSAGE}"
15+
env:
16+
TARGET_ENV: staging
17+
SERVICE: agentless-controller
18+
CONTAINER_NAME: agentless-controller
19+
CHECK_RECONCILE_ERROR_PERCENTAGE: true
20+
CHECK_CONTAINER_RESTART_COUNT: true
21+
CHECK_LOG_ERROR_RATE: true
22+
CHECK_SLO: true
23+
CHECK_SLO_TAG: agentless-controller
24+
CHECK_SLO_BURN_RATE_THRESHOLD: 0.1
25+
MAX_ERROR_PERCENT: 10
26+
SERVICE_VERSION: ${SERVICE_VERSION:0:12}
27+
CHECK_SYNTHETICS: true
28+
CHECK_SYNTHETICS_TAG: agentless
29+
CHECK_SYNTHETICS_MINIMUM_RUNS: 2
30+
CHECK_SYNTHETICS_MAX_POLL: 70
31+
CHECK_SYNTHETIC_POLL_INTERVAL: 180
32+
MAX_FAILURES: 1
33+
DEPLOYMENT_SLICES: ${DEPLOYMENT_SLICES:-""}

‎.buildkite/scripts/steps/ecp-internal-release.sh

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
#!/usr/bin/env bash
2+
3+
# ELASTICSEARCH CONFIDENTIAL
4+
# __________________
5+
#
6+
# Copyright Elasticsearch B.V. All rights reserved.
7+
#
8+
# NOTICE: All information contained herein is, and remains
9+
# the property of Elasticsearch B.V. and its suppliers, if any.
10+
# The intellectual and technical concepts contained herein
11+
# are proprietary to Elasticsearch B.V. and its suppliers and
12+
# may be covered by U.S. and Foreign Patents, patents in
13+
# process, and are protected by trade secret or copyright
14+
# law. Dissemination of this information or reproduction of
15+
# this material is strictly forbidden unless prior written
16+
# permission is obtained from Elasticsearch B.V.
17+
18+
set -eu
19+
20+
_SELF=$(dirname $0)
21+
source "${_SELF}/../common.sh"
22+
23+
24+
# annotate create temp markdown file if not exists
25+
# this file will be later used to annotate the build
26+
# it appends to the file the message passed as argument
27+
BUILDKITE_ANNOTATE_FILE="buildkite-annotate.md"
28+
annotate() {
29+
echo "$1" >>$BUILDKITE_ANNOTATE_FILE
30+
}
31+
32+
write_annotation() {
33+
cat $BUILDKITE_ANNOTATE_FILE | buildkite-agent annotate --style info
34+
}
35+
36+
DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod"
37+
DOCKER_REGISTRY="docker.elastic.co"
38+
PRIVATE_REPO="docker.elastic.co/observability-ci/ecp-elastic-agent-service"
39+
SNAPSHOT_DRA_URL=https://snapshots.elastic.co/latest/master.json
40+
41+
DRA_RESULT=$(curl -s -X GET "$SNAPSHOT_DRA_URL")
42+
echo "$DRA_RESULT"
43+
BUILD_ID=$(echo "$DRA_RESULT" | jq '.build_id' | tr -d '"')
44+
BUILD_VERSION=$(echo "$DRA_RESULT" | jq '.version' | tr -d '"')
45+
46+
MANIFEST_URL="https://snapshots.elastic.co/$BUILD_ID/agent-package/agent-artifacts-$BUILD_VERSION.json"
47+
GIT_COMMIT=$(curl -s -X GET "$MANIFEST_URL" | jq '.projects["elastic-agent-core"]["commit_hash"]' | tr -d '"')
48+
GIT_SHORT_COMMIT=$(echo "$GIT_COMMIT" | cut -c1-12)
49+
50+
DOCKER_TAG="git-${GIT_SHORT_COMMIT}"
51+
PRIVATE_IMAGE="${PRIVATE_REPO}:${DOCKER_TAG}"
52+
53+
DOCKER_USERNAME_SECRET=$(retry 5 vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}")
54+
DOCKER_PASSWORD_SECRET=$(retry 5 vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}")
55+
skopeo login --username "${DOCKER_USERNAME_SECRET}" --password "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY}"
56+
skopeo copy --all "docker://docker.elastic.co/cloud-release/elastic-agent-service:$BUILD_ID-SNAPSHOT" "docker://$PRIVATE_IMAGE"
57+
58+
annotate "* Image: $PRIVATE_IMAGE"
59+
annotate "* Short commit: $GIT_SHORT_COMMIT"
60+
annotate "* Commit: https://github.com/elastic/elastic-agent/commit/$GIT_COMMIT"
61+
annotate "* Manifest: $MANIFEST_URL"
62+
63+
buildkite-agent meta-data set "git-short-commit" "$GIT_SHORT_COMMIT"
64+
65+
write_annotation

‎.buildkite/scripts/steps/run-agentless-tests.sh

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
#!/usr/bin/env bash
2+
3+
# ELASTICSEARCH CONFIDENTIAL
4+
# __________________
5+
#
6+
# Copyright Elasticsearch B.V. All rights reserved.
7+
#
8+
# NOTICE: All information contained herein is, and remains
9+
# the property of Elasticsearch B.V. and its suppliers, if any.
10+
# The intellectual and technical concepts contained herein
11+
# are proprietary to Elasticsearch B.V. and its suppliers and
12+
# may be covered by U.S. and Foreign Patents, patents in
13+
# process, and are protected by trade secret or copyright
14+
# law. Dissemination of this information or reproduction of
15+
# this material is strictly forbidden unless prior written
16+
# permission is obtained from Elasticsearch B.V.
17+
18+
set -eox pipefail
19+
20+
_SELF=$(dirname $0)
21+
source "${_SELF}/../common.sh"
22+
23+
extract_sha() {
24+
local env=$1
25+
local sha
26+
27+
# Ensure repo is available
28+
git clone --depth 1 [email protected]:elastic/serverless-gitops.git || (
29+
cd serverless-gitops
30+
git pull
31+
)
32+
33+
go install github.com/mikefarah/yq/[email protected]
34+
35+
# Extract first matching SHA for the environment pattern
36+
sha=$(yq eval ".services.agentless-controller.versions | to_entries | .[] | select(.key | test(\"^${env}.*\")) | .value" serverless-gitops/services/agentless-controller/versions.yaml | head -1)
37+
38+
echo "$sha"
39+
}
40+
41+
# Check environment variable
42+
if [ -z "${ENVIRONMENT:-}" ]; then
43+
echo "ENVIRONMENT variable is not set"
44+
exit 1
45+
fi
46+
47+
# Extract agentless_controller_sha for the specified environment
48+
agentless_controller_sha=$(extract_sha "$ENVIRONMENT")
49+
50+
if [ -z "$agentless_controller_sha" ]; then
51+
echo "No SHA found for environment: $ENVIRONMENT"
52+
exit 1
53+
fi
54+
55+
echo "Running agentless tests for environment $ENVIRONMENT with Agentless-Controller version: $agentless_controller_sha"
56+
export SERVICE_VERSION="$agentless_controller_sha"
57+
make -C /agent run-environment-tests # part of docker.elastic.co/ci-agent-images/quality-gate-seedling image

‎catalog-info.yaml

Lines changed: 92 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -412,4 +412,95 @@ spec:
412412
ELASTIC_SLACK_NOTIFICATIONS_ENABLED: "true"
413413
SLACK_NOTIFICATIONS_CHANNEL: "#ingest-notifications"
414414
SLACK_NOTIFICATIONS_ON_SUCCESS: "false"
415-
SLACK_NOTIFICATIONS_ALL_BRANCHES: "false" # only notify for failures on `main` or \d+.\d+ (release) branches
415+
SLACK_NOTIFICATIONS_ALL_BRANCHES: "false" # only notify for failures on `main` or \d+.\d+ (release) branches
416+
417+
---
418+
# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
419+
apiVersion: backstage.io/v1alpha1
420+
kind: Resource
421+
metadata:
422+
name: agentless-serverless-release
423+
description: Initiate agentless serverless release
424+
links:
425+
- title: Pipeline
426+
url: https://buildkite.com/elastic/agentless-serverless-release
427+
spec:
428+
type: buildkite-pipeline
429+
owner: group:ingest-fp
430+
system: buildkite
431+
implementation:
432+
apiVersion: buildkite.elastic.dev/v1
433+
kind: Pipeline
434+
metadata:
435+
name: agentless / serverless / release
436+
description: Initiate Agentless serverless release
437+
spec:
438+
env:
439+
SLACK_NOTIFICATIONS_CHANNEL: '#agentless-alerts'
440+
ELASTIC_SLACK_NOTIFICATIONS_ENABLED: "true"
441+
SLACK_NOTIFICATIONS_ON_SUCCESS: "false"
442+
SLACK_NOTIFICATIONS_ALL_BRANCHES: "false" # only notify for failures on `main` or \d+.\d+ (release) branches
443+
default_branch: main
444+
allow_rebuilds: false
445+
skip_intermediate_builds: false
446+
repository: elastic/elastic-agent
447+
pipeline_file: .buildkite/pipeline.agentless-app-release.yaml
448+
provider_settings:
449+
build_branches: false
450+
build_pull_requests: false
451+
publish_commit_status: false
452+
trigger_mode: none
453+
build_tags: false
454+
prefix_pull_request_fork_branch_names: false
455+
skip_pull_request_builds_for_existing_commits: false
456+
teams:
457+
agentless-team:
458+
access_level: MANAGE_BUILD_AND_READ
459+
cloud-tooling:
460+
access_level: MANAGE_BUILD_AND_READ
461+
everyone:
462+
access_level: READ_ONLY
463+
ingest-fp:
464+
access_level: MANAGE_BUILD_AND_READ
465+
---
466+
467+
# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
468+
apiVersion: backstage.io/v1alpha1
469+
kind: Resource
470+
metadata:
471+
name: agentless-tests
472+
description: Tests the service ntegration in a specific environment
473+
links:
474+
- title: Pipeline
475+
url: https://buildkite.com/elastic/agentless-tests
476+
spec:
477+
type: buildkite-pipeline
478+
owner: group:ingest-fp
479+
system: buildkite
480+
implementation:
481+
apiVersion: buildkite.elastic.dev/v1
482+
kind: Pipeline
483+
metadata:
484+
name: agentless-tests
485+
description: agentless tests
486+
spec:
487+
env:
488+
SLACK_NOTIFICATIONS_CHANNEL: '#agentless-alerts'
489+
ELASTIC_SLACK_NOTIFICATIONS_ENABLED: "true"
490+
SLACK_NOTIFICATIONS_ON_SUCCESS: "false"
491+
SLACK_NOTIFICATIONS_ALL_BRANCHES: "false" # only notify for failures on `main` or \d+.\d+ (release) branches
492+
repository: elastic/elastic-agent
493+
pipeline_file: .buildkite/pipeline.agentless-tests.yaml
494+
branch_configuration: "main"
495+
provider_settings:
496+
build_pull_requests: false
497+
trigger_mode: none
498+
teams:
499+
agentless-team:
500+
access_level: MANAGE_BUILD_AND_READ
501+
cloud-tooling:
502+
access_level: MANAGE_BUILD_AND_READ
503+
ingest-fp:
504+
access_level: MANAGE_BUILD_AND_READ
505+
everyone:
506+
access_level: READ_ONLY

0 commit comments

Comments
 (0)