From 35e99ce7b76cd16f9834011f9e878b95811f5a9b Mon Sep 17 00:00:00 2001 From: Katya Titkova Date: Thu, 26 Mar 2026 09:53:12 +0100 Subject: [PATCH 1/4] partially parallelize acceptance tests --- .bazelrc | 1 + .buildkite/pipeline_lib.sh | 56 ++- MODULE.bazel.lock | 523 +++++++++++++++++++++++- Makefile | 2 +- acceptance/README.md | 14 +- acceptance/common/BUILD.bazel | 6 + acceptance/common/base.py | 56 ++- acceptance/common/docker.py | 36 +- acceptance/common/raw.bzl | 2 +- acceptance/common/slot.py | 140 +++++++ acceptance/common/topogen.bzl | 3 +- acceptance/hidden_paths/BUILD.bazel | 1 + acceptance/router_benchmark/BUILD.bazel | 1 + acceptance/router_multi/BUILD.bazel | 2 + acceptance/stun/BUILD.bazel | 1 + demo/file_transfer/BUILD.bazel | 1 + tools/topogen.py | 3 + tools/topology/docker.py | 9 +- 18 files changed, 826 insertions(+), 31 deletions(-) create mode 100644 acceptance/common/slot.py diff --git a/.bazelrc b/.bazelrc index d29eec7d4e..95d266b198 100644 --- a/.bazelrc +++ b/.bazelrc @@ -35,6 +35,7 @@ test:unit_all --config=unit //... test:integration --test_tag_filters=integration,-lint test:integration_all --config=integration //... +test:integration --local_test_jobs=HOST_CPUS*.5 test:lint --test_tag_filters=lint,write_src --test_summary=terse --noshow_progress --experimental_convenience_symlinks=ignore diff --git a/.buildkite/pipeline_lib.sh b/.buildkite/pipeline_lib.sh index bf61b3cdea..8fb34feb6f 100644 --- a/.buildkite/pipeline_lib.sh +++ b/.buildkite/pipeline_lib.sh @@ -7,8 +7,60 @@ gen_bazel_test_steps() { echo " key: integration-tests" echo " steps:" - targets="$(bazel query "attr(tags, integration, tests(//...)) except attr(tags, \"lint|manual\", tests(//...))" 2>/dev/null)" - for test in $targets; do + # Split tests into exclusive (must run alone) and parallel-safe groups. + all_targets="$(bazel query "attr(tags, integration, tests(//...)) except attr(tags, \"lint|manual\", tests(//...))" 2>/dev/null)" + exclusive_targets="$(bazel query "attr(tags, exclusive, attr(tags, integration, tests(//...))) except attr(tags, \"lint|manual\", tests(//...))" 2>/dev/null)" + parallel_targets="" + for test in $all_targets; do + if [ -n "${SINGLE_TEST}" ]; then + name=${test#//} + if [[ ! "${name}" =~ "${SINGLE_TEST}" ]]; then + continue + fi + fi + is_exclusive=false + for ex in $exclusive_targets; do + if [ "$test" = "$ex" ]; then + is_exclusive=true + break + fi + done + if [ "$is_exclusive" = "false" ]; then + parallel_targets="$parallel_targets $test" + fi + done + + # Emit a single step for all parallel-safe tests. + if [ -n "$parallel_targets" ]; then + cache="" + if [ -n "${SINGLE_TEST}" ] || [ "$parallel" != "1" ]; then + cache="--nocache_tesx§t_results" + fi + + echo " - label: \"Integration tests (parallel)\"" + echo " command:" + echo " - bazel test --config=integration --local_test_jobs=HOST_CPUS*.5 $cache $parallel_targets" + echo " key: \"integration_parallel\"" + echo " plugins:" + echo " - scionproto/metahook#v0.3.0:" + echo " pre-command: .buildkite/cleanup-leftovers.sh" + echo " pre-artifact: tar -chaf bazel-testlogs.tar.gz bazel-testlogs" + echo " pre-exit: .buildkite/cleanup-leftovers.sh" + echo " artifact_paths:" + echo " - \"bazel-testlogs.tar.gz\"" + echo " timeout_in_minutes: 20" + echo " retry:" + echo " manual:" + echo " permit_on_passed: true" + echo " automatic:" + echo " - exit_status: -1 # Agent was lost" + echo " - exit_status: 255 # Forced agent shutdown" + echo " - exit_status: 3 # Test may be flaky or it just didn't pass" + echo " limit: 2" + fi + + # Emit individual steps for exclusive tests. + for test in $exclusive_targets; do name=${test#//} cache="" args="" diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 542c06fb80..791bf11303 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -405,8 +405,8 @@ }, "@@aspect_rules_js+//npm:extensions.bzl%pnpm": { "general": { - "bzlTransitiveDigest": "cYE3QE61mjLIM5IdiE+TCpLJPIIsMr0Plg/2aOdcHOA=", - "usagesDigest": "8ggfSW1eOJDzHp5oo9r33sAB0sGvuQg4MjIOnRBgj2E=", + "bzlTransitiveDigest": "oYNqMRBQZT4eUO64w7LUy20WAgJZy2VJVMHy+qXr6Og=", + "usagesDigest": "EFKKU7eUgTryCbo1pjmIBAfRNXSqYCgsTqt5M8RIwhE=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -415,11 +415,11 @@ "repoRuleId": "@@aspect_rules_js+//npm/private:npm_import.bzl%npm_import_rule", "attributes": { "package": "pnpm", - "version": "8.6.7", + "version": "8.15.9", "root_package": "", "link_workspace": "", "link_packages": {}, - "integrity": "sha512-vRIWpD/L4phf9Bk2o/O2TDR8fFoJnpYrp2TKqTIZF/qZ2/rgL3qKXzHofHgbXsinwMoSEigz28sqk3pQ+yMEQQ==", + "integrity": "sha512-SZQ0ydj90aJ5Tr9FUrOyXApjOrzuW7Fee13pDzL0e1E6ypjNXP0AHDHw20VLw4BO3M1XhQHkyik6aBYWa72fgQ==", "url": "", "commit": "", "patch_args": [ @@ -435,14 +435,14 @@ "extra_build_content": "load(\"@aspect_rules_js//js:defs.bzl\", \"js_binary\")\njs_binary(name = \"pnpm\", data = glob([\"package/**\"]), entry_point = \"package/dist/pnpm.cjs\", visibility = [\"//visibility:public\"])", "generate_bzl_library_targets": false, "extract_full_archive": true, - "system_tar": "auto" + "exclude_package_contents": [] } }, "pnpm__links": { "repoRuleId": "@@aspect_rules_js+//npm/private:npm_import.bzl%npm_import_links", "attributes": { "package": "pnpm", - "version": "8.6.7", + "version": "8.15.9", "dev": false, "root_package": "", "link_packages": {}, @@ -464,6 +464,11 @@ } }, "recordedRepoMappingEntries": [ + [ + "aspect_bazel_lib+", + "bazel_lib", + "bazel_lib+" + ], [ "aspect_bazel_lib+", "bazel_skylib", @@ -474,16 +479,36 @@ "bazel_tools", "bazel_tools" ], + [ + "aspect_bazel_lib+", + "tar.bzl", + "tar.bzl+" + ], [ "aspect_rules_js+", "aspect_bazel_lib", "aspect_bazel_lib+" ], + [ + "aspect_rules_js+", + "aspect_rules_js", + "aspect_rules_js+" + ], + [ + "aspect_rules_js+", + "aspect_tools_telemetry_report", + "aspect_tools_telemetry++telemetry+aspect_tools_telemetry_report" + ], [ "aspect_rules_js+", "bazel_features", "bazel_features+" ], + [ + "aspect_rules_js+", + "bazel_lib", + "bazel_lib+" + ], [ "aspect_rules_js+", "bazel_skylib", @@ -503,6 +528,31 @@ "bazel_features+", "bazel_features_version", "bazel_features++version_extension+bazel_features_version" + ], + [ + "bazel_lib+", + "bazel_skylib", + "bazel_skylib+" + ], + [ + "bazel_lib+", + "bazel_tools", + "bazel_tools" + ], + [ + "tar.bzl+", + "aspect_bazel_lib", + "aspect_bazel_lib+" + ], + [ + "tar.bzl+", + "bazel_skylib", + "bazel_skylib+" + ], + [ + "tar.bzl+", + "tar.bzl", + "tar.bzl+" ] ] } @@ -539,6 +589,156 @@ ] } }, + "@@pybind11_bazel+//:internal_configure.bzl%internal_configure_extension": { + "general": { + "bzlTransitiveDigest": "yA4GkX1zdUTOXU4qKsh/YkjZKHAWkG8Fd7adNeGwiRc=", + "usagesDigest": "D1r3lfzMuUBFxgG8V6o0bQTLMk3GkaGOaPzw53wrwyw=", + "recordedFileInputs": { + "@@pybind11_bazel+//MODULE.bazel": "e6f4c20442eaa7c90d7190d8dc539d0ab422f95c65a57cc59562170c58ae3d34" + }, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "pybind11": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "build_file": "@@pybind11_bazel+//:pybind11-BUILD.bazel", + "strip_prefix": "pybind11-2.12.0", + "urls": [ + "https://github.com/pybind/pybind11/archive/v2.12.0.zip" + ] + } + } + }, + "recordedRepoMappingEntries": [ + [ + "pybind11_bazel+", + "bazel_tools", + "bazel_tools" + ] + ] + } + }, + "@@rules_apple+//apple:apple.bzl%provisioning_profile_repository_extension": { + "general": { + "bzlTransitiveDigest": "pdRt+Wm+XQmS427nAIg9z7qfm56CnZChO+HJ4zu3Xa8=", + "usagesDigest": "vsJl8Rw5NL+5Ag2wdUDoTeRF/5klkXO8545Iy7U1Q08=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "local_provisioning_profiles": { + "repoRuleId": "@@rules_apple+//apple/internal:local_provisioning_profiles.bzl%provisioning_profile_repository", + "attributes": {} + } + }, + "recordedRepoMappingEntries": [ + [ + "apple_support+", + "bazel_skylib", + "bazel_skylib+" + ], + [ + "bazel_tools", + "rules_cc", + "rules_cc+" + ], + [ + "rules_apple+", + "bazel_skylib", + "bazel_skylib+" + ], + [ + "rules_apple+", + "bazel_tools", + "bazel_tools" + ], + [ + "rules_apple+", + "build_bazel_apple_support", + "apple_support+" + ], + [ + "rules_apple+", + "build_bazel_rules_swift", + "rules_swift+" + ], + [ + "rules_cc+", + "bazel_tools", + "bazel_tools" + ], + [ + "rules_cc+", + "cc_compatibility_proxy", + "rules_cc++compatibility_proxy+cc_compatibility_proxy" + ], + [ + "rules_cc+", + "rules_cc", + "rules_cc+" + ], + [ + "rules_cc++compatibility_proxy+cc_compatibility_proxy", + "rules_cc", + "rules_cc+" + ], + [ + "rules_swift+", + "bazel_skylib", + "bazel_skylib+" + ], + [ + "rules_swift+", + "bazel_tools", + "bazel_tools" + ], + [ + "rules_swift+", + "build_bazel_apple_support", + "apple_support+" + ], + [ + "rules_swift+", + "build_bazel_rules_swift", + "rules_swift+" + ], + [ + "rules_swift+", + "build_bazel_rules_swift_local_config", + "rules_swift++non_module_deps+build_bazel_rules_swift_local_config" + ] + ] + } + }, + "@@rules_apple+//apple:extensions.bzl%non_module_deps": { + "general": { + "bzlTransitiveDigest": "rdeowIB16n42a/MEy5gV/a/kpjt45s1nmLJCD/kXsPU=", + "usagesDigest": "M3VqFpeTCo4qmrNKGZw0dxBHvTYDrfV3cscGzlSAhQ4=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "xctestrunner": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/google/xctestrunner/archive/b7698df3d435b6491b4b4c0f9fc7a63fbed5e3a6.tar.gz" + ], + "strip_prefix": "xctestrunner-b7698df3d435b6491b4b4c0f9fc7a63fbed5e3a6", + "sha256": "ae3a063c985a8633cb7eb566db21656f8db8eb9a0edb8c182312c7f0db53730d" + } + } + }, + "recordedRepoMappingEntries": [ + [ + "rules_apple+", + "bazel_tools", + "bazel_tools" + ] + ] + } + }, "@@rules_kotlin+//src/main/starlark/core/repositories:bzlmod_setup.bzl%rules_kotlin_extensions": { "general": { "bzlTransitiveDigest": "sFhcgPbDQehmbD1EOXzX4H1q/CD5df8zwG4kp4jbvr8=", @@ -1243,6 +1443,317 @@ ] } }, + "@@rules_rust+//crate_universe/private:internal_extensions.bzl%cu_nr": { + "general": { + "bzlTransitiveDigest": "dTMqlNf5XNNms3D6jnwfyN3A189YbUbRc2OuxgtL/IM=", + "usagesDigest": "h9wqdPgSnpDxdRNP6kO+DMoTuAWtxtek2SnJOOL321c=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "cargo_bazel_bootstrap": { + "repoRuleId": "@@rules_rust+//cargo/private:cargo_bootstrap.bzl%cargo_bootstrap_repository", + "attributes": { + "srcs": [ + "@@rules_rust+//crate_universe:src/api.rs", + "@@rules_rust+//crate_universe:src/api/lockfile.rs", + "@@rules_rust+//crate_universe:src/cli.rs", + "@@rules_rust+//crate_universe:src/cli/generate.rs", + "@@rules_rust+//crate_universe:src/cli/query.rs", + "@@rules_rust+//crate_universe:src/cli/render.rs", + "@@rules_rust+//crate_universe:src/cli/splice.rs", + "@@rules_rust+//crate_universe:src/cli/vendor.rs", + "@@rules_rust+//crate_universe:src/config.rs", + "@@rules_rust+//crate_universe:src/context.rs", + "@@rules_rust+//crate_universe:src/context/crate_context.rs", + "@@rules_rust+//crate_universe:src/context/platforms.rs", + "@@rules_rust+//crate_universe:src/lib.rs", + "@@rules_rust+//crate_universe:src/lockfile.rs", + "@@rules_rust+//crate_universe:src/main.rs", + "@@rules_rust+//crate_universe:src/metadata.rs", + "@@rules_rust+//crate_universe:src/metadata/cargo_bin.rs", + "@@rules_rust+//crate_universe:src/metadata/cargo_tree_resolver.rs", + "@@rules_rust+//crate_universe:src/metadata/cargo_tree_rustc_wrapper.bat", + "@@rules_rust+//crate_universe:src/metadata/cargo_tree_rustc_wrapper.sh", + "@@rules_rust+//crate_universe:src/metadata/dependency.rs", + "@@rules_rust+//crate_universe:src/metadata/metadata_annotation.rs", + "@@rules_rust+//crate_universe:src/rendering.rs", + "@@rules_rust+//crate_universe:src/rendering/template_engine.rs", + "@@rules_rust+//crate_universe:src/rendering/templates/module_bzl.j2", + "@@rules_rust+//crate_universe:src/rendering/templates/partials/header.j2", + "@@rules_rust+//crate_universe:src/rendering/templates/partials/module/aliases_map.j2", + "@@rules_rust+//crate_universe:src/rendering/templates/partials/module/deps_map.j2", + "@@rules_rust+//crate_universe:src/rendering/templates/partials/module/repo_git.j2", + "@@rules_rust+//crate_universe:src/rendering/templates/partials/module/repo_http.j2", + "@@rules_rust+//crate_universe:src/rendering/templates/vendor_module.j2", + "@@rules_rust+//crate_universe:src/rendering/verbatim/alias_rules.bzl", + "@@rules_rust+//crate_universe:src/select.rs", + "@@rules_rust+//crate_universe:src/splicing.rs", + "@@rules_rust+//crate_universe:src/splicing/cargo_config.rs", + "@@rules_rust+//crate_universe:src/splicing/crate_index_lookup.rs", + "@@rules_rust+//crate_universe:src/splicing/splicer.rs", + "@@rules_rust+//crate_universe:src/test.rs", + "@@rules_rust+//crate_universe:src/utils.rs", + "@@rules_rust+//crate_universe:src/utils/starlark.rs", + "@@rules_rust+//crate_universe:src/utils/starlark/glob.rs", + "@@rules_rust+//crate_universe:src/utils/starlark/label.rs", + "@@rules_rust+//crate_universe:src/utils/starlark/select.rs", + "@@rules_rust+//crate_universe:src/utils/starlark/select_dict.rs", + "@@rules_rust+//crate_universe:src/utils/starlark/select_list.rs", + "@@rules_rust+//crate_universe:src/utils/starlark/select_scalar.rs", + "@@rules_rust+//crate_universe:src/utils/starlark/select_set.rs", + "@@rules_rust+//crate_universe:src/utils/starlark/serialize.rs", + "@@rules_rust+//crate_universe:src/utils/starlark/target_compatible_with.rs", + "@@rules_rust+//crate_universe:src/utils/symlink.rs", + "@@rules_rust+//crate_universe:src/utils/target_triple.rs" + ], + "binary": "cargo-bazel", + "cargo_lockfile": "@@rules_rust+//crate_universe:Cargo.lock", + "cargo_toml": "@@rules_rust+//crate_universe:Cargo.toml", + "version": "1.86.0", + "timeout": 900, + "rust_toolchain_cargo_template": "@rust_host_tools//:bin/{tool}", + "rust_toolchain_rustc_template": "@rust_host_tools//:bin/{tool}", + "compressed_windows_toolchain_names": false + } + } + }, + "moduleExtensionMetadata": { + "explicitRootModuleDirectDeps": [ + "cargo_bazel_bootstrap" + ], + "explicitRootModuleDirectDevDeps": [], + "useAllRepos": "NO", + "reproducible": false + }, + "recordedRepoMappingEntries": [ + [ + "bazel_features+", + "bazel_features_globals", + "bazel_features++version_extension+bazel_features_globals" + ], + [ + "bazel_features+", + "bazel_features_version", + "bazel_features++version_extension+bazel_features_version" + ], + [ + "rules_cc+", + "bazel_tools", + "bazel_tools" + ], + [ + "rules_cc+", + "cc_compatibility_proxy", + "rules_cc++compatibility_proxy+cc_compatibility_proxy" + ], + [ + "rules_cc+", + "rules_cc", + "rules_cc+" + ], + [ + "rules_cc++compatibility_proxy+cc_compatibility_proxy", + "rules_cc", + "rules_cc+" + ], + [ + "rules_rust+", + "bazel_features", + "bazel_features+" + ], + [ + "rules_rust+", + "bazel_skylib", + "bazel_skylib+" + ], + [ + "rules_rust+", + "bazel_tools", + "bazel_tools" + ], + [ + "rules_rust+", + "cui", + "rules_rust++cu+cui" + ], + [ + "rules_rust+", + "rules_cc", + "rules_cc+" + ], + [ + "rules_rust+", + "rules_rust", + "rules_rust+" + ], + [ + "rules_rust+", + "rules_rust_ctve", + "rules_rust++i2+rules_rust_ctve" + ] + ] + } + }, + "@@rules_swift+//swift:extensions.bzl%non_module_deps": { + "general": { + "bzlTransitiveDigest": "X3B53n1AmUggqagsfW3TwgDmiWEQnqWY1izDXQ3mZtY=", + "usagesDigest": "mhACFnrdMv9Wi0Mt67bxocJqviRkDSV+Ee5Mqdj5akA=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "com_github_apple_swift_protobuf": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-protobuf/archive/1.20.2.tar.gz" + ], + "sha256": "3fb50bd4d293337f202d917b6ada22f9548a0a0aed9d9a4d791e6fbd8a246ebb", + "strip_prefix": "swift-protobuf-1.20.2/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_protobuf/BUILD.overlay" + } + }, + "com_github_grpc_grpc_swift": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/grpc/grpc-swift/archive/1.16.0.tar.gz" + ], + "sha256": "58b60431d0064969f9679411264b82e40a217ae6bd34e17096d92cc4e47556a5", + "strip_prefix": "grpc-swift-1.16.0/", + "build_file": "@@rules_swift+//third_party:com_github_grpc_grpc_swift/BUILD.overlay" + } + }, + "com_github_apple_swift_docc_symbolkit": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-docc-symbolkit/archive/refs/tags/swift-5.10-RELEASE.tar.gz" + ], + "sha256": "de1d4b6940468ddb53b89df7aa1a81323b9712775b0e33e8254fa0f6f7469a97", + "strip_prefix": "swift-docc-symbolkit-swift-5.10-RELEASE", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_docc_symbolkit/BUILD.overlay" + } + }, + "com_github_apple_swift_nio": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio/archive/2.42.0.tar.gz" + ], + "sha256": "e3304bc3fb53aea74a3e54bd005ede11f6dc357117d9b1db642d03aea87194a0", + "strip_prefix": "swift-nio-2.42.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_http2": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-http2/archive/1.26.0.tar.gz" + ], + "sha256": "f0edfc9d6a7be1d587e5b403f2d04264bdfae59aac1d74f7d974a9022c6d2b25", + "strip_prefix": "swift-nio-http2-1.26.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_http2/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_transport_services": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-transport-services/archive/1.15.0.tar.gz" + ], + "sha256": "f3498dafa633751a52b9b7f741f7ac30c42bcbeb3b9edca6d447e0da8e693262", + "strip_prefix": "swift-nio-transport-services-1.15.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_transport_services/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_extras": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-extras/archive/1.4.0.tar.gz" + ], + "sha256": "4684b52951d9d9937bb3e8ccd6b5daedd777021ef2519ea2f18c4c922843b52b", + "strip_prefix": "swift-nio-extras-1.4.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_extras/BUILD.overlay" + } + }, + "com_github_apple_swift_log": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-log/archive/1.4.4.tar.gz" + ], + "sha256": "48fe66426c784c0c20031f15dc17faf9f4c9037c192bfac2f643f65cb2321ba0", + "strip_prefix": "swift-log-1.4.4/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_log/BUILD.overlay" + } + }, + "com_github_apple_swift_nio_ssl": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-nio-ssl/archive/2.23.0.tar.gz" + ], + "sha256": "4787c63f61dd04d99e498adc3d1a628193387e41efddf8de19b8db04544d016d", + "strip_prefix": "swift-nio-ssl-2.23.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_nio_ssl/BUILD.overlay" + } + }, + "com_github_apple_swift_collections": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-collections/archive/1.0.4.tar.gz" + ], + "sha256": "d9e4c8a91c60fb9c92a04caccbb10ded42f4cb47b26a212bc6b39cc390a4b096", + "strip_prefix": "swift-collections-1.0.4/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_collections/BUILD.overlay" + } + }, + "com_github_apple_swift_atomics": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "urls": [ + "https://github.com/apple/swift-atomics/archive/1.1.0.tar.gz" + ], + "sha256": "1bee7f469f7e8dc49f11cfa4da07182fbc79eab000ec2c17bfdce468c5d276fb", + "strip_prefix": "swift-atomics-1.1.0/", + "build_file": "@@rules_swift+//third_party:com_github_apple_swift_atomics/BUILD.overlay" + } + }, + "build_bazel_rules_swift_index_import": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "build_file": "@@rules_swift+//third_party:build_bazel_rules_swift_index_import/BUILD.overlay", + "canonical_id": "index-import-5.8", + "urls": [ + "https://github.com/MobileNativeFoundation/index-import/releases/download/5.8.0.1/index-import.tar.gz" + ], + "sha256": "28c1ffa39d99e74ed70623899b207b41f79214c498c603915aef55972a851a15" + } + }, + "build_bazel_rules_swift_local_config": { + "repoRuleId": "@@rules_swift+//swift/internal:swift_autoconfiguration.bzl%swift_autoconfiguration", + "attributes": {} + } + }, + "recordedRepoMappingEntries": [ + [ + "rules_swift+", + "bazel_tools", + "bazel_tools" + ], + [ + "rules_swift+", + "build_bazel_rules_swift", + "rules_swift+" + ] + ] + } + }, "@@yq.bzl+//yq:extensions.bzl%yq": { "general": { "bzlTransitiveDigest": "61Uz+o5PnlY0jJfPZEUNqsKxnM/UCLeWsn5VVCc8u5Y=", diff --git a/Makefile b/Makefile index ad4eae3535..72723a4888 100644 --- a/Makefile +++ b/Makefile @@ -60,7 +60,7 @@ test: bazel test --config=unit_all test-integration: - bazel test --config=integration_all + bazel test --config=integration_all $(ARGS) go.mod: bazel run --config=quiet @rules_go//go -- mod tidy diff --git a/acceptance/README.md b/acceptance/README.md index 49b8b435a3..6450194f10 100644 --- a/acceptance/README.md +++ b/acceptance/README.md @@ -1,7 +1,10 @@ # Acceptance testing framework This directory contains a set of integration tests. -Each test is defined as a bazel test target, with tags `integration` and `exclusive`. +Each test is defined as a bazel test target, with tag `integration`. +Tests run in parallel, unless they need any global resources (like +network namespaces or hardcoded IPs). For such tests, +`exclusive` tag is used. Some integration tests use code outside this directory. For example, the `router_multi` acceptance test cases and main executable are in `tools/braccept`. @@ -24,10 +27,17 @@ bazel test --config=integration //acceptance/cert_renewal:all //acceptance/trc_u bazel test --config=integration //acceptance/router_multi:all --cache_test_results=no ``` -The following the flags to bazel test can be helpful when running individual tests: +The following flags can be helpful when running individual tests: - `--test_output=streamed` to display test output to the screen immediately - `--cache_test_results=no` or `-t-` to re-run tests after a cached success +- `--local_test_jobs=N` to control how many tests run in parallel + +When using `make test-integration`, extra Bazel flags can be passed via `ARGS`: + +```bash +make test-integration ARGS="--local_test_jobs=3" +``` ## Manual Testing diff --git a/acceptance/common/BUILD.bazel b/acceptance/common/BUILD.bazel index 2396fbc908..2a178d8998 100644 --- a/acceptance/common/BUILD.bazel +++ b/acceptance/common/BUILD.bazel @@ -11,6 +11,7 @@ py_library( "docker", "log", "scion", + "slot", ], ) @@ -22,6 +23,11 @@ py_library( ], ) +py_library( + name = "slot", + srcs = ["slot.py"], +) + py_library( name = "log", srcs = ["log.py"], diff --git a/acceptance/common/base.py b/acceptance/common/base.py index 73e07f2fa0..efb5f1bd88 100644 --- a/acceptance/common/base.py +++ b/acceptance/common/base.py @@ -15,6 +15,7 @@ import logging import os import re +import subprocess import traceback from abc import abstractmethod, ABC @@ -24,6 +25,7 @@ from plumbum import LocalPath from acceptance.common import docker, log +from acceptance.common import slot from tools.topology.scion_addr import ISD_AS logger = logging.getLogger(__name__) @@ -86,6 +88,7 @@ def _set_executables(self, executables): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._setup_prepare_failed = False + self._slot = None def init(self): """ init is called first. The Test object can be initialized here. @@ -110,13 +113,37 @@ def _run(self): pass def teardown(self): - pass + if self._slot is not None: + self._slot.release() + + def _load_persisted_slot(self): + """Load slot from .slot file for manual run/teardown subcommands.""" + slot_file = self.artifacts / ".slot" + if slot_file.exists() and self._slot is None: + with open(str(slot_file), "r") as f: + slot_id = int(f.read().strip()) + self._slot = slot.Slot(slot_id, lock_fd=None) def setup_prepare(self): """Unpacks loads local docker images and generates the topology. """ - docker.assert_no_networks() + self._slot = slot.acquire() + # Override artifacts dir with slot-derived path when not running + # under Bazel (which sets TEST_UNDECLARED_OUTPUTS_DIR to a unique dir). + if not os.environ.get("TEST_UNDECLARED_OUTPUTS_DIR"): + self.artifacts = LocalPath(self._slot.artifacts_dir) + # Pre-cleanup: remove orphaned containers from a previous crashed run. + subprocess.run( + ["docker", "compose", "-p", self._slot.project_name, "down", "-v"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + docker.assert_no_networks(prefix=self._slot.project_name) self._setup_artifacts() + # Persist slot ID for manual setup/run/teardown mode. + slot_file = self.artifacts / ".slot" + with open(str(slot_file), "w") as f: + f.write(str(self._slot.id)) self._setup_docker_images() # Define where coredumps will be stored. print( @@ -157,6 +184,13 @@ def init(self): def setup_prepare(self): super().setup_prepare() + # Re-initialize Compose with the slot's project name now that + # the slot is acquired (init() runs before setup, before slot + # acquisition). + self.dc = docker.Compose( + compose_file=self.artifacts / "gen/scion-dc.yml", + project_name=self._slot.project_name, + ) self._setup_generate() def _setup_generate(self): @@ -181,6 +215,8 @@ def copy_file(src, dst): "-o=" + self.artifacts + "/gen", "-c=topology.json", "-d", + "--project-name=" + self._slot.project_name, + "--network=" + self._slot.network, *self.setup_params, ) for support_dir in ["logs", "gen-cache", "gen-data", "traces"]: @@ -199,14 +235,24 @@ def setup_start(self): def teardown(self): # Avoid running docker compose teardown if setup_prepare failed if self._setup_prepare_failed: + super().teardown() return out_dir = self.artifacts / "logs" self.dc.collect_logs(out_dir=out_dir) ps = self.dc("ps") print(self.dc("down", "-v")) + super().teardown() if re.search(r"Exit\s+[1-9]\d*", ps): raise Exception("Failed services.\n" + ps) + def _reinit_compose_from_slot(self): + """Reinitialize Compose with slot's project name for manual mode.""" + if self._slot is not None: + self.dc = docker.Compose( + compose_file=self.artifacts / "gen/scion-dc.yml", + project_name=self._slot.project_name, + ) + def await_connectivity(self, quiet_seconds=None, timeout_seconds=None): """ Wait for the beaconing process in a local topology to establish full connectivity, i.e. at @@ -266,6 +312,9 @@ def main(self): class _TestRun(test_class, cli.Application): def main(self): self.init() + self._load_persisted_slot() + if hasattr(self, '_reinit_compose_from_slot'): + self._reinit_compose_from_slot() try: self._run() except Exception: @@ -275,6 +324,9 @@ def main(self): class _TestTeardown(test_class, cli.Application): def main(self): self.init() + self._load_persisted_slot() + if hasattr(self, '_reinit_compose_from_slot'): + self._reinit_compose_from_slot() try: self.teardown() except Exception: diff --git a/acceptance/common/docker.py b/acceptance/common/docker.py index 33b898aa46..16bb51e304 100644 --- a/acceptance/common/docker.py +++ b/acceptance/common/docker.py @@ -40,15 +40,18 @@ class Compose(object): def __init__(self, - compose_file: str = SCION_DC_FILE): + compose_file: str = SCION_DC_FILE, + project_name: str = "scion"): self.compose_file = compose_file + self.project_name = project_name def __call__(self, *args, **kwargs) -> str: """Runs docker compose with the given arguments""" # Note: not using plumbum here due to complications with encodings in the captured output try: res = subprocess.run( - ["docker", "compose", "-f", self.compose_file, *args], + ["docker", "compose", "-f", self.compose_file, + "-p", self.project_name, *args], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8") except subprocess.CalledProcessError as e: raise _CalledProcessErrorWithOutput(e) from None @@ -63,18 +66,18 @@ def collect_logs(self, out_dir: str = "logs/docker"): dst_f = out_p / "%s.log" % svc print(svc) with open(dst_f, "w") as log_file: - cmd.docker.run(args=("logs", "scion-"+svc+"-1"), stdout=log_file, + cmd.docker.run(args=("logs", self.project_name+"-"+svc+"-1"), stdout=log_file, stderr=subprocess.STDOUT, retcode=None) # Collect coredupms. coredump_f = out_p / "%s.coredump" % svc try: - cmd.docker.run(args=("cp", svc+":/share/coredump", coredump_f)) + cmd.docker.run(args=("cp", self.project_name+"-"+svc+"-1:/share/coredump", coredump_f)) except Exception: # If the coredump does not exist, do nothing. pass # Collect tshark traces. try: - cmd.docker.run(args=("cp", svc+":/share/tshark", out_p)) + cmd.docker.run(args=("cp", self.project_name+"-"+svc+"-1:/share/tshark", out_p)) cmd.mv(out_p / "tshark" // "*", out_p) cmd.rmdir(out_p / "tshark") except Exception: @@ -188,12 +191,16 @@ class UnexpectedNetworkError(Exception): pass -def assert_no_networks(writer=None): +def assert_no_networks(writer=None, prefix=None): """Raises an exception if unexpected docker networks are found. The default bridge, host and none networks are always ignored. - If the SCION_TESTING_DOCKER_ASSERTIONS_OFF environmnent variable + If prefix is set, only networks starting with '{prefix}_scn_' are + considered unexpected. If prefix is None, all non-default networks + are flagged. + + If the SCION_TESTING_DOCKER_ASSERTIONS_OFF environment variable is set to 1, the assertion is not executed. A warning message is printed to the writer if the writer is set. @@ -201,10 +208,11 @@ def assert_no_networks(writer=None): writer: If specified, the writer's write method is used to print detailed information about all unexpected networks before raising the exception. + prefix: If specified, only flag networks with this prefix. Raises: UnexpectedNetworkError: An unexpected network name was found. The - message will contain the names of all unexepcted networks. + message will contain the names of all unexpected networks. plumbum.commands.processes.ProcessExecutionError: One of the docker commands returned a non-zero exit code. """ @@ -216,10 +224,14 @@ def assert_no_networks(writer=None): allowed_nets = ['bridge', 'host', 'none', 'benchmark'] unexpected_nets = [] for net in _get_networks(): - if net.name not in allowed_nets: - if writer: - writer.write(f'{net.name} {net.driver} {net.containers}\n') - unexpected_nets.append(net.name) + if net.name in allowed_nets: + continue + if prefix is not None: + if not net.name.startswith(prefix + '_scn_'): + continue + if writer: + writer.write(f'{net.name} {net.driver} {net.containers}\n') + unexpected_nets.append(net.name) if unexpected_nets: raise UnexpectedNetworkError(str(unexpected_nets)) diff --git a/acceptance/common/raw.bzl b/acceptance/common/raw.bzl index 7eb6718a53..7dc684a181 100644 --- a/acceptance/common/raw.bzl +++ b/acceptance/common/raw.bzl @@ -67,7 +67,7 @@ def raw_test( deps = [":%s_lib" % name], imports = imports, data = data, - tags = tags + ["integration", "exclusive"], + tags = tags + ["integration"], local = local, env = { # Ensure output appears immediately (in particular with --test_output=streamed) diff --git a/acceptance/common/slot.py b/acceptance/common/slot.py new file mode 100644 index 0000000000..0ee2bc8539 --- /dev/null +++ b/acceptance/common/slot.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +# +# Copyright 2026 SCION Association +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Slot pool manager for parallel acceptance test isolation. + +Each slot provides a unique Docker Compose project name, network prefix, +IPv4 subnet, and artifacts directory. Slots are acquired via filesystem +locks and released on teardown or process exit. +""" + +import fcntl +import logging +import os + +logger = logging.getLogger(__name__) + +NUM_SLOTS = 16 +LOCK_DIR = "/tmp" +LOCK_PREFIX = "scion-test-slot-" + + +class Slot: + """Represents an acquired test slot with derived resource names.""" + + def __init__(self, slot_id: int, lock_fd: int): + self.id = slot_id + self._lock_fd = lock_fd + + @property + def project_name(self) -> str: + return "scion%d" % self.id + + @property + def network(self) -> str: + return "172.20.%d.0/20" % (self.id * 16) + + @property + def artifacts_dir(self) -> str: + return "/tmp/artifacts-scion-%d" % self.id + + def release(self): + """Release the slot by closing the lock file descriptor.""" + if self._lock_fd is not None: + try: + os.close(self._lock_fd) + except OSError: + pass + self._lock_fd = None + logger.info("Released slot %d", self.id) + + +def acquire() -> Slot: + """Acquire a free slot from the pool. + + Tries each slot in order, using flock(LOCK_EX | LOCK_NB) to acquire + without blocking. Writes the current PID to the lock file for stale + detection. + + If all slots are taken, checks for stale locks (dead PIDs) and reclaims + the first one found. + + Returns: + A Slot object with the acquired slot ID. + + Raises: + RuntimeError: If no slot could be acquired (all 16 in use by live + processes). + """ + # First pass: try to acquire a free slot. + for slot_id in range(NUM_SLOTS): + fd = _try_acquire(slot_id) + if fd is not None: + return Slot(slot_id, fd) + + # Second pass: reclaim stale locks. + for slot_id in range(NUM_SLOTS): + lock_path = _lock_path(slot_id) + try: + with open(lock_path, "r") as f: + pid = int(f.read().strip()) + # Check if the process is still alive. + os.kill(pid, 0) + except (OSError, ValueError): + # Process is dead or PID is invalid. Reclaim. + logger.warning("Reclaiming stale slot %d", slot_id) + try: + os.unlink(lock_path) + except OSError: + pass + fd = _try_acquire(slot_id) + if fd is not None: + return Slot(slot_id, fd) + + raise RuntimeError( + "No test slots available (all %d in use by live processes)" % NUM_SLOTS + ) + + +def _try_acquire(slot_id: int): + """Try to acquire a slot by locking its file. + + Returns: + The file descriptor if successful, None otherwise. + """ + lock_path = _lock_path(slot_id) + fd = -1 + try: + fd = os.open(lock_path, os.O_CREAT | os.O_RDWR, 0o644) + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + # Write our PID for stale detection. + os.ftruncate(fd, 0) + os.lseek(fd, 0, os.SEEK_SET) + os.write(fd, ("%d\n" % os.getpid()).encode()) + logger.info("Acquired slot %d", slot_id) + return fd + except (OSError, IOError): + # Could not lock - slot is taken. + if fd >= 0: + try: + os.close(fd) + except OSError: + pass + return None + + +def _lock_path(slot_id: int) -> str: + return os.path.join(LOCK_DIR, "%s%d.lock" % (LOCK_PREFIX, slot_id)) diff --git a/acceptance/common/topogen.bzl b/acceptance/common/topogen.bzl index bebb575a2a..4659c5e0d6 100644 --- a/acceptance/common/topogen.bzl +++ b/acceptance/common/topogen.bzl @@ -15,6 +15,7 @@ def topogen_test( args = [], deps = [], data = [], + tags = [], homedir = "", tester = "//docker:tester.tarball"): """Creates a test based on a topology file. @@ -114,7 +115,7 @@ def topogen_test( args = args + common_args, deps = [":%s_lib" % name], data = data + common_data, - tags = ["integration", "exclusive"], + tags = tags + ["integration"], env = { # Ensure output appears immediately (in particular with --test_output=streamed) "PYTHONUNBUFFERED": "1", diff --git a/acceptance/hidden_paths/BUILD.bazel b/acceptance/hidden_paths/BUILD.bazel index b2bdfd4c2c..bb9cbfe288 100644 --- a/acceptance/hidden_paths/BUILD.bazel +++ b/acceptance/hidden_paths/BUILD.bazel @@ -9,4 +9,5 @@ topogen_test( "//acceptance/hidden_paths/testdata:hp_groups_as4.yml", ], topo = "//acceptance/hidden_paths/testdata:topology.topo", + tags = ["exclusive"], ) diff --git a/acceptance/router_benchmark/BUILD.bazel b/acceptance/router_benchmark/BUILD.bazel index 21f3f853da..aa39d6e5be 100644 --- a/acceptance/router_benchmark/BUILD.bazel +++ b/acceptance/router_benchmark/BUILD.bazel @@ -40,6 +40,7 @@ raw_test( homedir = "$(rootpath //docker:router.tarball)", # This test uses sudo and accesses /var/run/netns. local = True, + tags = ["exclusive"], deps = ["benchmarklib"], ) diff --git a/acceptance/router_multi/BUILD.bazel b/acceptance/router_multi/BUILD.bazel index 21b526d717..06aa49cdee 100644 --- a/acceptance/router_multi/BUILD.bazel +++ b/acceptance/router_multi/BUILD.bazel @@ -32,6 +32,7 @@ raw_test( homedir = "$(rootpath :conf)", # This test uses sudo and accesses /var/run/netns. local = True, + tags = ["exclusive"], ) raw_test( @@ -42,4 +43,5 @@ raw_test( homedir = "$(rootpath :conf)", # This test uses sudo and accesses /var/run/netns. local = True, + tags = ["exclusive"], ) diff --git a/acceptance/stun/BUILD.bazel b/acceptance/stun/BUILD.bazel index 4dec92ea5f..f0fa13b7f6 100644 --- a/acceptance/stun/BUILD.bazel +++ b/acceptance/stun/BUILD.bazel @@ -13,4 +13,5 @@ topogen_test( "//acceptance/stun/test-server", ], topo = "//topology:tiny.topo", + tags = ["exclusive"], ) diff --git a/demo/file_transfer/BUILD.bazel b/demo/file_transfer/BUILD.bazel index 842860c5dd..ea6b32801b 100644 --- a/demo/file_transfer/BUILD.bazel +++ b/demo/file_transfer/BUILD.bazel @@ -7,4 +7,5 @@ topogen_test( data = ["tc_setup.sh"], gateway = True, topo = "topo.topo", + tags = ["exclusive"], ) diff --git a/tools/topogen.py b/tools/topogen.py index db365f43fb..2b311b6b23 100755 --- a/tools/topogen.py +++ b/tools/topogen.py @@ -53,6 +53,9 @@ def add_arguments(parser): parser.add_argument('--sig', action='store_true', help='Generate a SIG per AS (only available with -d, the SIG image needs\ to be built manually e.g. when running acceptance tests)') + parser.add_argument('--project-name', + help='Docker Compose project name (default: scion)', + default='scion') parser.add_argument('--rpc_server_protocol', help='Configures services that can to accept RPCs over the given protocol \ (grpc, connectrpc, all)', default='all') diff --git a/tools/topology/docker.py b/tools/topology/docker.py index ba59b9db58..d453e4bd6a 100644 --- a/tools/topology/docker.py +++ b/tools/topology/docker.py @@ -54,7 +54,7 @@ def __init__(self, args): self.args = args self.dc_conf = { 'version': DOCKER_COMPOSE_CONFIG_VERSION, - 'name': 'scion', + 'name': getattr(self.args, 'project_name', 'scion'), 'services': {}, 'networks': {}, 'volumes': {} @@ -119,8 +119,9 @@ def _create_networks(self): ipv = 'ipv6' self.elem_networks[elem].append({'net': str(network), ipv: ip}) # Create docker networks - prefix = 'scn_' - net_name = "%s%03d" % (prefix, len(self.bridges)) + project_name = getattr(self.args, 'project_name', 'scion') + net_name = "scn_%03d" % len(self.bridges) + bridge_name = "%s_%s" % (project_name, net_name) self.bridges[str(network)] = net_name self.dc_conf['networks'][net_name] = { 'ipam': { @@ -130,7 +131,7 @@ def _create_networks(self): }, 'driver': 'bridge', 'driver_opts': { - 'com.docker.network.bridge.name': net_name + 'com.docker.network.bridge.name': bridge_name } } if net_desc.name in v4nets: From d70f7fab9e6837669ac7500e1db20447c564de8c Mon Sep 17 00:00:00 2001 From: Katya Titkova Date: Thu, 26 Mar 2026 10:12:35 +0100 Subject: [PATCH 2/4] fix lint --- .buildkite/pipeline_lib.sh | 5 +++-- acceptance/hidden_paths/BUILD.bazel | 2 +- acceptance/stun/BUILD.bazel | 2 +- demo/file_transfer/BUILD.bazel | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.buildkite/pipeline_lib.sh b/.buildkite/pipeline_lib.sh index 8fb34feb6f..148525e23e 100644 --- a/.buildkite/pipeline_lib.sh +++ b/.buildkite/pipeline_lib.sh @@ -37,9 +37,10 @@ gen_bazel_test_steps() { cache="--nocache_tesx§t_results" fi - echo " - label: \"Integration tests (parallel)\"" + echo " - label: \"Other integration tests (parallel)\"" echo " command:" - echo " - bazel test --config=integration --local_test_jobs=HOST_CPUS*.5 $cache $parallel_targets" + echo " - echo '--- Targets' && echo '$parallel_targets' | tr ' ' '\n' | sort" + echo " - bazel test --config=integration --local_test_jobs=HOST_CPUS*.75 $cache $parallel_targets" echo " key: \"integration_parallel\"" echo " plugins:" echo " - scionproto/metahook#v0.3.0:" diff --git a/acceptance/hidden_paths/BUILD.bazel b/acceptance/hidden_paths/BUILD.bazel index bb9cbfe288..e949161bb6 100644 --- a/acceptance/hidden_paths/BUILD.bazel +++ b/acceptance/hidden_paths/BUILD.bazel @@ -8,6 +8,6 @@ topogen_test( "//acceptance/hidden_paths/testdata:hp_groups_as3.yml", "//acceptance/hidden_paths/testdata:hp_groups_as4.yml", ], - topo = "//acceptance/hidden_paths/testdata:topology.topo", tags = ["exclusive"], + topo = "//acceptance/hidden_paths/testdata:topology.topo", ) diff --git a/acceptance/stun/BUILD.bazel b/acceptance/stun/BUILD.bazel index f0fa13b7f6..2b38ea6d55 100644 --- a/acceptance/stun/BUILD.bazel +++ b/acceptance/stun/BUILD.bazel @@ -12,6 +12,6 @@ topogen_test( "//acceptance/stun/test-client", "//acceptance/stun/test-server", ], - topo = "//topology:tiny.topo", tags = ["exclusive"], + topo = "//topology:tiny.topo", ) diff --git a/demo/file_transfer/BUILD.bazel b/demo/file_transfer/BUILD.bazel index ea6b32801b..c88cceac35 100644 --- a/demo/file_transfer/BUILD.bazel +++ b/demo/file_transfer/BUILD.bazel @@ -6,6 +6,6 @@ topogen_test( args = [], data = ["tc_setup.sh"], gateway = True, - topo = "topo.topo", tags = ["exclusive"], + topo = "topo.topo", ) From 952866b7cac479376daed1563c47ea1ef7124f21 Mon Sep 17 00:00:00 2001 From: Katya Titkova Date: Thu, 26 Mar 2026 12:19:27 +0100 Subject: [PATCH 3/4] fix lint --- acceptance/common/docker.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/acceptance/common/docker.py b/acceptance/common/docker.py index 16bb51e304..706f19ab08 100644 --- a/acceptance/common/docker.py +++ b/acceptance/common/docker.py @@ -62,22 +62,23 @@ def collect_logs(self, out_dir: str = "logs/docker"): out_p = plumbum.local.path(out_dir) cmd.mkdir("-p", out_p) for svc in self("config", "--services").splitlines(): + container = self.project_name + "-" + svc + "-1" # Collect logs. dst_f = out_p / "%s.log" % svc print(svc) with open(dst_f, "w") as log_file: - cmd.docker.run(args=("logs", self.project_name+"-"+svc+"-1"), stdout=log_file, + cmd.docker.run(args=("logs", container), stdout=log_file, stderr=subprocess.STDOUT, retcode=None) # Collect coredupms. coredump_f = out_p / "%s.coredump" % svc try: - cmd.docker.run(args=("cp", self.project_name+"-"+svc+"-1:/share/coredump", coredump_f)) + cmd.docker.run(args=("cp", container+":/share/coredump", coredump_f)) except Exception: # If the coredump does not exist, do nothing. pass # Collect tshark traces. try: - cmd.docker.run(args=("cp", self.project_name+"-"+svc+"-1:/share/tshark", out_p)) + cmd.docker.run(args=("cp", container+":/share/tshark", out_p)) cmd.mv(out_p / "tshark" // "*", out_p) cmd.rmdir(out_p / "tshark") except Exception: From 029752ef725c38ce712dd1118daf0dd8b5858fa9 Mon Sep 17 00:00:00 2001 From: Katya Titkova Date: Thu, 26 Mar 2026 12:54:35 +0100 Subject: [PATCH 4/4] move topogen to topogen_with_graph's data deps so it gets proper runfiles --- control/beaconing/BUILD.bazel | 2 -- pkg/private/xtest/graph/BUILD.bazel | 1 + pkg/private/xtest/graph/topogen_with_graph.sh | 10 +++++----- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/control/beaconing/BUILD.bazel b/control/beaconing/BUILD.bazel index dc4570daf2..28d975995a 100644 --- a/control/beaconing/BUILD.bazel +++ b/control/beaconing/BUILD.bazel @@ -110,11 +110,9 @@ run_binary( name = "topogen_topology_big", srcs = [ ":ifids_topology_big", - "//tools:topogen", "//topology:big", ], args = [ - "$(locations //tools:topogen)", "$(location //topology:big)", "$(location :ifids_topology_big)", "$(RULEDIR)/testdata/big", diff --git a/pkg/private/xtest/graph/BUILD.bazel b/pkg/private/xtest/graph/BUILD.bazel index 18de552733..e912712596 100644 --- a/pkg/private/xtest/graph/BUILD.bazel +++ b/pkg/private/xtest/graph/BUILD.bazel @@ -72,5 +72,6 @@ write_source_files( sh_binary( name = "topogen_with_graph", srcs = ["topogen_with_graph.sh"], + data = ["//tools:topogen"], visibility = ["//visibility:public"], ) diff --git a/pkg/private/xtest/graph/topogen_with_graph.sh b/pkg/private/xtest/graph/topogen_with_graph.sh index 802315072d..1bef2e80b8 100755 --- a/pkg/private/xtest/graph/topogen_with_graph.sh +++ b/pkg/private/xtest/graph/topogen_with_graph.sh @@ -1,10 +1,10 @@ #!/bin/bash -topogen="$1" -# $2 and $3 are other files passed by topogen's py_binary to the Bazel rule running this script. -topology="$4" -ifids="$5" -out="$6" +# Locate topogen via runfiles (it's a data dependency of this sh_binary). +topogen="${RUNFILES_DIR:-$0.runfiles}/_main/tools/topogen" +topology="$1" +ifids="$2" +out="$3" tmpdir=$(mktemp -d)