diff --git a/.github/workflows/binary_image.yml b/.github/workflows/binary_image.yml new file mode 100644 index 000000000000..0c07f7426ab0 --- /dev/null +++ b/.github/workflows/binary_image.yml @@ -0,0 +1,157 @@ +name: binary-image + +env: + # Force the stdout and stderr streams to be unbuffered + PYTHONUNBUFFERED: 1 + +on: # yamllint disable-line rule:truthy + push: + # pull_request: + # types: + # - synchronize + # - reopened + # - opened + # branches: + # - master + # paths-ignore: + # - 'CHANGELOG.md' + # - 'README.md' + # - 'SECURITY.md' + # - 'docker/docs/**' + # - 'docs/**' + # - 'utils/check-style/aspell-ignore/**' +jobs: + DockerHubPushAarch64: + runs-on: [self-hosted, style-checker-aarch64] + steps: + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Images check + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 docker_images_check.py --suffix aarch64 + - name: Upload images files to artifacts + uses: actions/upload-artifact@v3 + with: + name: changed_images_aarch64 + path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json + DockerHubPushAmd64: + runs-on: [self-hosted, builder] + steps: + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Images check + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 docker_images_check.py --suffix amd64 + - name: Upload images files to artifacts + uses: actions/upload-artifact@v3 + with: + name: changed_images_amd64 + path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json + DockerHubPush: + needs: [DockerHubPushAmd64, DockerHubPushAarch64] + runs-on: [self-hosted, builder] + steps: + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Download changed aarch64 images + uses: actions/download-artifact@v3 + with: + name: changed_images_aarch64 + path: ${{ runner.temp }} + - name: Download changed amd64 images + uses: actions/download-artifact@v3 + with: + name: changed_images_amd64 + path: ${{ runner.temp }} + - name: Images check + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 + - name: Upload images files to artifacts + uses: actions/upload-artifact@v3 + with: + name: changed_images + path: ${{ runner.temp }}/changed_images.json + # BuilderDebAarch64: + # needs: [DockerHubPush] + # runs-on: [self-hosted, vm2] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/build_check + # IMAGES_PATH=${{runner.temp}}/images_path + # REPO_COPY=${{runner.temp}}/build_check/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # BUILD_NAME=package_aarch64 + # EOF + # - name: Download changed images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images + # path: ${{ runner.temp }}/images_path + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # submodules: true + # fetch-depth: 0 # for performance artifact + # - name: Build + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + # - name: Upload build URLs to artifacts + # if: ${{ success() || failure() }} + # uses: actions/upload-artifact@v3 + # with: + # name: ${{ env.BUILD_URLS }} + # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + # InstallPackagesTestAarch64: + # needs: [BuilderDebAarch64] + # runs-on: [self-hosted, test41] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/test_install + # REPORTS_PATH=${{runner.temp}}/reports_dir + # CHECK_NAME=Install packages (arm64) + # REPO_COPY=${{runner.temp}}/test_install/ClickHouse + # EOF + # - name: Download json reports + # uses: actions/download-artifact@v3 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Test packages installation + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" + # python3 install_check.py "$CHECK_NAME" + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" \ No newline at end of file diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index f0741b5465f3..57c099121219 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -9,19 +9,8 @@ on: # yamllint disable-line rule:truthy branches: - 'master' jobs: - PythonUnitTests: - runs-on: [self-hosted, style-checker] - steps: - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Python unit tests - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 -m unittest discover -s . -p '*_test.py' DockerHubPushAarch64: - runs-on: [self-hosted, style-checker-aarch64] + runs-on: [self-hosted, style-checker] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 @@ -53,7 +42,7 @@ jobs: name: changed_images_amd64 path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json DockerHubPush: - needs: [DockerHubPushAmd64, DockerHubPushAarch64, PythonUnitTests] + needs: [DockerHubPushAmd64, DockerHubPushAarch64] runs-on: [self-hosted, style-checker] steps: - name: Check out repository code diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index afc08f3e6376..35b731c92935 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -23,38 +23,38 @@ on: # yamllint disable-line rule:truthy ##################################### SMALL CHECKS ####################################### ########################################################################################## jobs: - CheckLabels: - runs-on: [self-hosted, style-checker] - # Run the first check always, even if the CI is cancelled - if: ${{ always() }} - steps: - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Labels check - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 run_check.py - PythonUnitTests: - runs-on: [self-hosted, style-checker] - steps: - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Python unit tests - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - echo "Testing the main ci directory" - python3 -m unittest discover -s . -p '*_test.py' - for dir in *_lambda/; do - echo "Testing $dir" - python3 -m unittest discover -s "$dir" -p '*_test.py' - done + # CheckLabels: + # runs-on: [self-hosted, style-checker] + # # Run the first check always, even if the CI is cancelled + # if: ${{ always() }} + # steps: + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Labels check + # run: | + # cd "$GITHUB_WORKSPACE/tests/ci" + # python3 run_check.py + # PythonUnitTests: + # runs-on: [self-hosted, style-checker] + # steps: + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Python unit tests + # run: | + # cd "$GITHUB_WORKSPACE/tests/ci" + # echo "Testing the main ci directory" + # python3 -m unittest discover -s . -p '*_test.py' + # for dir in *_lambda/; do + # echo "Testing $dir" + # python3 -m unittest discover -s "$dir" -p '*_test.py' + # done DockerHubPushAarch64: - needs: CheckLabels - runs-on: [self-hosted, style-checker-aarch64] + # needs: CheckLabels + runs-on: [self-hosted, style-checker] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 @@ -70,7 +70,7 @@ jobs: name: changed_images_aarch64 path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json DockerHubPushAmd64: - needs: CheckLabels + # needs: CheckLabels runs-on: [self-hosted, style-checker] steps: - name: Check out repository code @@ -87,7 +87,7 @@ jobs: name: changed_images_amd64 path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json DockerHubPush: - needs: [DockerHubPushAmd64, DockerHubPushAarch64, PythonUnitTests] + needs: [DockerHubPushAmd64, DockerHubPushAarch64] runs-on: [self-hosted, style-checker] steps: - name: Check out repository code @@ -178,71 +178,71 @@ jobs: docker ps --quiet | xargs --no-run-if-empty docker kill ||: docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - CompatibilityCheckX86: - needs: [BuilderDebRelease] - runs-on: [self-hosted, style-checker] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/compatibility_check - REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse - REPORTS_PATH=${{runner.temp}}/reports_dir - EOF - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: CompatibilityCheckX86 - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (amd64)" --check-glibc --check-distributions - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - CompatibilityCheckAarch64: - needs: [BuilderDebAarch64] - runs-on: [self-hosted, style-checker] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/compatibility_check - REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse - REPORTS_PATH=${{runner.temp}}/reports_dir - EOF - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: CompatibilityCheckAarch64 - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (aarch64)" --check-glibc - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -######################################################################################### -#################################### ORDINARY BUILDS #################################### -######################################################################################### +## CompatibilityCheckX86: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, style-checker] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/compatibility_check +# REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse +# REPORTS_PATH=${{runner.temp}}/reports_dir +# EOF +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: CompatibilityCheckX86 +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (amd64)" --check-glibc --check-distributions +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# CompatibilityCheckAarch64: +# needs: [BuilderDebAarch64] +# runs-on: [self-hosted, style-checker] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/compatibility_check +# REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse +# REPORTS_PATH=${{runner.temp}}/reports_dir +# EOF +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: CompatibilityCheckAarch64 +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (aarch64)" --check-glibc +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ######################################################################################### +# #################################### ORDINARY BUILDS #################################### +# ######################################################################################### BuilderDebRelease: needs: [DockerHubPush, FastTest, StyleCheck] runs-on: [self-hosted, builder] @@ -285,47 +285,47 @@ jobs: docker ps --quiet | xargs --no-run-if-empty docker kill ||: docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" - BuilderBinRelease: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=binary_release - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinRelease: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_release +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderDebAarch64: needs: [DockerHubPush, FastTest, StyleCheck] runs-on: [self-hosted, builder] @@ -580,50 +580,50 @@ jobs: docker ps --quiet | xargs --no-run-if-empty docker kill ||: docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -########################################################################################## -##################################### SPECIAL BUILDS ##################################### -########################################################################################## - BuilderBinClangTidy: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=binary_tidy - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# ########################################################################################## +# ##################################### SPECIAL BUILDS ##################################### +# ########################################################################################## +# BuilderBinClangTidy: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_tidy +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" BuilderBinDarwin: needs: [DockerHubPush, FastTest, StyleCheck] runs-on: [self-hosted, builder] @@ -665,48 +665,89 @@ jobs: docker ps --quiet | xargs --no-run-if-empty docker kill ||: docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderBinAarch64: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=binary_aarch64 - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderBinFreeBSD: +# BuilderBinAarch64: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_aarch64 +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinFreeBSD: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_freebsd +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + BuilderBinDarwinAarch64: needs: [DockerHubPush, FastTest, StyleCheck] runs-on: [self-hosted, builder] steps: @@ -717,7 +758,7 @@ jobs: IMAGES_PATH=${{runner.temp}}/images_path REPO_COPY=${{runner.temp}}/build_check/ClickHouse CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=binary_freebsd + BUILD_NAME=binary_darwin_aarch64 EOF - name: Download changed images uses: actions/download-artifact@v3 @@ -747,4180 +788,4139 @@ jobs: docker ps --quiet | xargs --no-run-if-empty docker kill ||: docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderBinDarwinAarch64: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] +# BuilderBinPPC64: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_ppc64le +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinAmd64Compat: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_amd64_compat +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinAarch64V80Compat: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_aarch64_v80compat +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# ############################################################################################ +# ##################################### Docker images ####################################### +# ############################################################################################ + DockerServerImages: + needs: + - BuilderDebRelease + - BuilderDebAarch64 + runs-on: [self-hosted, style-checker] steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=binary_darwin_aarch64 - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - name: Check out repository code uses: ClickHouse/checkout@v1 with: clear-repository: true - submodules: true - - name: Build + fetch-depth: 0 # It MUST BE THE SAME for all dependencies and the job itself + - name: Check docker clickhouse/clickhouse-server building run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderBinPPC64: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=binary_ppc64le - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderBinAmd64Compat: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=binary_amd64_compat - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderBinAarch64V80Compat: - needs: [DockerHubPush, FastTest, StyleCheck] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=binary_aarch64_v80compat - EOF - - name: Download changed images - uses: actions/download-artifact@v3 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - submodules: true - - name: Build - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -############################################################################################ -##################################### Docker images ####################################### -############################################################################################ - DockerServerImages: - needs: - - BuilderDebRelease - - BuilderDebAarch64 - runs-on: [self-hosted, style-checker] - steps: - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - fetch-depth: 0 # It MUST BE THE SAME for all dependencies and the job itself - - name: Check docker clickhouse/clickhouse-server building - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 docker_server.py --release-type head --no-push \ - --image-repo clickhouse/clickhouse-server --image-path docker/server - python3 docker_server.py --release-type head --no-push \ - --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -############################################################################################ -##################################### BUILD REPORTER ####################################### -############################################################################################ - BuilderReport: - needs: - - BuilderBinRelease - - BuilderDebAarch64 - - BuilderDebAsan - - BuilderDebDebug - - BuilderDebMsan - - BuilderDebRelease - - BuilderDebTsan - - BuilderDebUBsan - runs-on: [self-hosted, style-checker] - if: ${{ success() || failure() }} - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - CHECK_NAME=ClickHouse build check - REPORTS_PATH=${{runner.temp}}/reports_dir - TEMP_PATH=${{runner.temp}}/report_check - NEEDS_DATA_PATH=${{runner.temp}}/needs.json - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Report Builder - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cat > "$NEEDS_DATA_PATH" << 'EOF' - ${{ toJSON(needs) }} - EOF - cd "$GITHUB_WORKSPACE/tests/ci" - python3 build_report_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - BuilderSpecialReport: - needs: - - BuilderBinAarch64 - - BuilderBinDarwin - - BuilderBinDarwinAarch64 - - BuilderBinFreeBSD - - BuilderBinPPC64 - - BuilderBinAmd64Compat - - BuilderBinAarch64V80Compat - - BuilderBinClangTidy - runs-on: [self-hosted, style-checker] - if: ${{ success() || failure() }} - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/report_check - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=ClickHouse special build check - NEEDS_DATA_PATH=${{runner.temp}}/needs.json - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Report Builder - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cat > "$NEEDS_DATA_PATH" << 'EOF' - ${{ toJSON(needs) }} - EOF - cd "$GITHUB_WORKSPACE/tests/ci" - python3 build_report_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -############################################################################################ -#################################### INSTALL PACKAGES ###################################### -############################################################################################ - InstallPackagesTestRelease: - needs: [BuilderDebRelease] - runs-on: [self-hosted, style-checker] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/test_install - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Install packages (amd64) - REPO_COPY=${{runner.temp}}/test_install/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Test packages installation - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 install_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - InstallPackagesTestAarch64: - needs: [BuilderDebAarch64] - runs-on: [self-hosted, style-checker-aarch64] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/test_install - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Install packages (arm64) - REPO_COPY=${{runner.temp}}/test_install/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Test packages installation - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 install_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -############################################################################################## -########################### FUNCTIONAl STATELESS TESTS ####################################### -############################################################################################## - FunctionalStatelessTestRelease: - needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (release) - REPO_COPY=${{runner.temp}}/stateless_release/ClickHouse - KILL_TIMEOUT=10800 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestReleaseDatabaseReplicated0: - needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_database_replicated - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (release, DatabaseReplicated) - REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestReleaseDatabaseReplicated1: - needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_database_replicated - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (release, DatabaseReplicated) - REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestReleaseDatabaseReplicated2: - needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_database_replicated - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (release, DatabaseReplicated) - REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestReleaseDatabaseReplicated3: - needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_database_replicated - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (release, DatabaseReplicated) - REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=3 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestReleaseWideParts: - needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_wide_parts - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (release, wide parts enabled) - REPO_COPY=${{runner.temp}}/stateless_wide_parts/ClickHouse - KILL_TIMEOUT=10800 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestReleaseAnalyzer: - needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_analyzer - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (release, analyzer) - REPO_COPY=${{runner.temp}}/stateless_analyzer/ClickHouse - KILL_TIMEOUT=10800 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestReleaseS3_0: - needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_s3_storage - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (release, s3 storage) - REPO_COPY=${{runner.temp}}/stateless_s3_storage/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=2 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestReleaseS3_1: - needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_s3_storage - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (release, s3 storage) - REPO_COPY=${{runner.temp}}/stateless_s3_storage/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=2 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestS3Debug0: - needs: [BuilderDebDebug] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (debug, s3 storage) - REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestS3Debug1: - needs: [BuilderDebDebug] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (debug, s3 storage) - REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestS3Debug2: - needs: [BuilderDebDebug] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (debug, s3 storage) - REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestS3Debug3: - needs: [BuilderDebDebug] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (debug, s3 storage) - REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=3 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestS3Debug4: - needs: [BuilderDebDebug] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (debug, s3 storage) - REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=4 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestS3Debug5: - needs: [BuilderDebDebug] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (debug, s3 storage) - REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=5 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestS3Tsan0: - needs: [BuilderDebTsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (tsan, s3 storage) - REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestS3Tsan1: - needs: [BuilderDebTsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (tsan, s3 storage) - REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestS3Tsan2: - needs: [BuilderDebTsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (tsan, s3 storage) - REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestS3Tsan3: - needs: [BuilderDebTsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (tsan, s3 storage) - REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=3 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestS3Tsan4: - needs: [BuilderDebTsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (tsan, s3 storage) - REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=4 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestAarch64: - needs: [BuilderDebAarch64] - runs-on: [self-hosted, func-tester-aarch64] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (aarch64) - REPO_COPY=${{runner.temp}}/stateless_release/ClickHouse - KILL_TIMEOUT=10800 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestAsan0: - needs: [BuilderDebAsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (asan) - REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestAsan1: - needs: [BuilderDebAsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (asan) - REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestAsan2: - needs: [BuilderDebAsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (asan) - REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestAsan3: - needs: [BuilderDebAsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (asan) - REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=3 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestTsan0: - needs: [BuilderDebTsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (tsan) - REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestTsan1: - needs: [BuilderDebTsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (tsan) - REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestTsan2: - needs: [BuilderDebTsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (tsan) - REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestTsan3: - needs: [BuilderDebTsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (tsan) - REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=3 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestTsan4: - needs: [BuilderDebTsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (tsan) - REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=4 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestUBsan0: - needs: [BuilderDebUBsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_ubsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (ubsan) - REPO_COPY=${{runner.temp}}/stateless_ubsan/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=2 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestUBsan1: - needs: [BuilderDebUBsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_ubsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (ubsan) - REPO_COPY=${{runner.temp}}/stateless_ubsan/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=2 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestMsan0: - needs: [BuilderDebMsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_memory - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (msan) - REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestMsan1: - needs: [BuilderDebMsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_memory - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (msan) - REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestMsan2: - needs: [BuilderDebMsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_memory - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (msan) - REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestMsan3: - needs: [BuilderDebMsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_memory - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (msan) - REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=3 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestMsan4: - needs: [BuilderDebMsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_memory - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (msan) - REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=4 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestMsan5: - needs: [BuilderDebMsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_memory - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (msan) - REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=5 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestDebug0: - needs: [BuilderDebDebug] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (debug) - REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestDebug1: - needs: [BuilderDebDebug] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (debug) - REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestDebug2: - needs: [BuilderDebDebug] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (debug) - REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestDebug3: - needs: [BuilderDebDebug] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (debug) - REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=3 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestDebug4: - needs: [BuilderDebDebug] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (debug) - REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=4 - RUN_BY_HASH_TOTAL=5 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestFlakyCheck: - needs: [BuilderDebAsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_flaky_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests flaky check (asan) - REPO_COPY=${{runner.temp}}/stateless_flaky_asan/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - TestsBugfixCheck: - needs: [CheckLabels, StyleCheck] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/tests_bugfix_check - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=tests bugfix validate check - KILL_TIMEOUT=3600 - REPO_COPY=${{runner.temp}}/tests_bugfix_check/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Bugfix test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - - TEMP_PATH="${TEMP_PATH}/integration" \ - REPORTS_PATH="${REPORTS_PATH}/integration" \ - python3 integration_test_check.py "Integration $CHECK_NAME" \ - --validate-bugfix --post-commit-status=file || echo 'ignore exit code' - - TEMP_PATH="${TEMP_PATH}/stateless" \ - REPORTS_PATH="${REPORTS_PATH}/stateless" \ - python3 functional_test_check.py "Stateless $CHECK_NAME" "$KILL_TIMEOUT" \ - --validate-bugfix --post-commit-status=file || echo 'ignore exit code' - - python3 bugfix_validate_check.py "${TEMP_PATH}/stateless/functional_commit_status.tsv" "${TEMP_PATH}/integration/integration_commit_status.tsv" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -############################################################################################## -############################ FUNCTIONAl STATEFUL TESTS ####################################### -############################################################################################## - FunctionalStatefulTestRelease: - needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (release) - REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestAarch64: - needs: [BuilderDebAarch64] - runs-on: [self-hosted, func-tester-aarch64] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (aarch64) - REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestAsan: - needs: [BuilderDebAsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (asan) - REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestTsan: - needs: [BuilderDebTsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (tsan) - REPO_COPY=${{runner.temp}}/stateful_tsan/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestMsan: - needs: [BuilderDebMsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_msan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (msan) - REPO_COPY=${{runner.temp}}/stateful_msan/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestUBsan: - needs: [BuilderDebUBsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_ubsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (ubsan) - REPO_COPY=${{runner.temp}}/stateful_ubsan/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestDebug: - needs: [BuilderDebDebug] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (debug) - REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - # Parallel replicas - FunctionalStatefulTestDebugParallelReplicas: - needs: [BuilderDebDebug] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (debug, ParallelReplicas) - REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestUBsanParallelReplicas: - needs: [BuilderDebUBsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_ubsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (ubsan, ParallelReplicas) - REPO_COPY=${{runner.temp}}/stateful_ubsan/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestMsanParallelReplicas: - needs: [BuilderDebMsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_msan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (msan, ParallelReplicas) - REPO_COPY=${{runner.temp}}/stateful_msan/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestTsanParallelReplicas: - needs: [BuilderDebTsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (tsan, ParallelReplicas) - REPO_COPY=${{runner.temp}}/stateful_tsan/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestAsanParallelReplicas: - needs: [BuilderDebAsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (asan, ParallelReplicas) - REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestReleaseParallelReplicas: - needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (release, ParallelReplicas) - REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -############################################################################################## -######################################### STRESS TESTS ####################################### -############################################################################################## - StressTestAsan: - needs: [BuilderDebAsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stress_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stress test (asan) - REPO_COPY=${{runner.temp}}/stress_asan/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Stress test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 stress_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - StressTestTsan: - needs: [BuilderDebTsan] - # func testers have 16 cores + 128 GB memory - # while stress testers have 36 cores + 72 memory - # It would be better to have something like 32 + 128, - # but such servers almost unavailable as spot instances. - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stress_thread - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stress test (tsan) - REPO_COPY=${{runner.temp}}/stress_thread/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Stress test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 stress_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - StressTestMsan: - needs: [BuilderDebMsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stress_memory - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stress test (msan) - REPO_COPY=${{runner.temp}}/stress_memory/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Stress test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 stress_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - StressTestUBsan: - needs: [BuilderDebUBsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stress_undefined - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stress test (ubsan) - REPO_COPY=${{runner.temp}}/stress_undefined/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Stress test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 stress_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - StressTestDebug: - needs: [BuilderDebDebug] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stress_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stress test (debug) - REPO_COPY=${{runner.temp}}/stress_debug/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Stress test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 stress_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - ############################################################################################## - ######################################### UPGRADE CHECK ###################################### - ############################################################################################## - UpgradeCheckAsan: - needs: [BuilderDebAsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/upgrade_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Upgrade check (asan) - REPO_COPY=${{runner.temp}}/upgrade_asan/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Upgrade check - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 upgrade_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - UpgradeCheckTsan: - needs: [BuilderDebTsan] - # same as for stress test with tsan - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/upgrade_thread - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Upgrade check (tsan) - REPO_COPY=${{runner.temp}}/upgrade_thread/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Upgrade check - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 upgrade_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - UpgradeCheckMsan: - needs: [BuilderDebMsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/upgrade_memory - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Upgrade check (msan) - REPO_COPY=${{runner.temp}}/upgrade_memory/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Upgrade check - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 upgrade_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - UpgradeCheckDebug: - needs: [BuilderDebDebug] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/upgrade_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Upgrade check (debug) - REPO_COPY=${{runner.temp}}/upgrade_debug/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Upgrade check - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 upgrade_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -############################################################################################## -##################################### AST FUZZERS ############################################ -############################################################################################## - ASTFuzzerTestAsan: - needs: [BuilderDebAsan] - runs-on: [self-hosted, fuzzer-unit-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/ast_fuzzer_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=AST fuzzer (asan) - REPO_COPY=${{runner.temp}}/ast_fuzzer_asan/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Fuzzer - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 ast_fuzzer_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - ASTFuzzerTestTsan: - needs: [BuilderDebTsan] - runs-on: [self-hosted, fuzzer-unit-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/ast_fuzzer_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=AST fuzzer (tsan) - REPO_COPY=${{runner.temp}}/ast_fuzzer_tsan/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Fuzzer - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 ast_fuzzer_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - ASTFuzzerTestUBSan: - needs: [BuilderDebUBsan] - runs-on: [self-hosted, fuzzer-unit-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/ast_fuzzer_ubsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=AST fuzzer (ubsan) - REPO_COPY=${{runner.temp}}/ast_fuzzer_ubsan/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Fuzzer - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 ast_fuzzer_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - ASTFuzzerTestMSan: - needs: [BuilderDebMsan] - runs-on: [self-hosted, fuzzer-unit-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/ast_fuzzer_msan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=AST fuzzer (msan) - REPO_COPY=${{runner.temp}}/ast_fuzzer_msan/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Fuzzer - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 ast_fuzzer_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - ASTFuzzerTestDebug: - needs: [BuilderDebDebug] - runs-on: [self-hosted, fuzzer-unit-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/ast_fuzzer_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=AST fuzzer (debug) - REPO_COPY=${{runner.temp}}/ast_fuzzer_debug/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Fuzzer - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 ast_fuzzer_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -############################################################################################# -############################# INTEGRATION TESTS ############################################# -############################################################################################# - IntegrationTestsAsan0: - needs: [BuilderDebAsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (asan) - REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsAsan1: - needs: [BuilderDebAsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (asan) - REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsAsan2: - needs: [BuilderDebAsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (asan) - REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsAsan3: - needs: [BuilderDebAsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (asan) - REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse - RUN_BY_HASH_NUM=3 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsAsan4: - needs: [BuilderDebAsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (asan) - REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse - RUN_BY_HASH_NUM=4 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsAsan5: - needs: [BuilderDebAsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (asan) - REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse - RUN_BY_HASH_NUM=5 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsTsan0: - needs: [BuilderDebTsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (tsan) - REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsTsan1: - needs: [BuilderDebTsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (tsan) - REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsTsan2: - needs: [BuilderDebTsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (tsan) - REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsTsan3: - needs: [BuilderDebTsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (tsan) - REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse - RUN_BY_HASH_NUM=3 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsTsan4: - needs: [BuilderDebTsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (tsan) - REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse - RUN_BY_HASH_NUM=4 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsTsan5: - needs: [BuilderDebTsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (tsan) - REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse - RUN_BY_HASH_NUM=5 - RUN_BY_HASH_TOTAL=6 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsRelease0: - needs: [BuilderDebRelease] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (release) - REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsRelease1: - needs: [BuilderDebRelease] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (release) - REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsRelease2: - needs: [BuilderDebRelease] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (release) - REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsRelease3: - needs: [BuilderDebRelease] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (release) - REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse - RUN_BY_HASH_NUM=3 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsFlakyCheck: - needs: [BuilderDebAsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_asan_flaky_check - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests flaky check (asan) - REPO_COPY=${{runner.temp}}/integration_tests_asan_flaky_check/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -############################################################################################# -#################################### UNIT TESTS ############################################# -############################################################################################# - UnitTestsAsan: - needs: [BuilderDebAsan] - runs-on: [self-hosted, fuzzer-unit-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/unit_tests_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Unit tests (asan) - REPO_COPY=${{runner.temp}}/unit_tests_asan/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Unit test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 unit_tests_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - UnitTestsReleaseClang: - needs: [BuilderBinRelease] - runs-on: [self-hosted, fuzzer-unit-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/unit_tests_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Unit tests (release-clang) - REPO_COPY=${{runner.temp}}/unit_tests_asan/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Unit test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 unit_tests_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - UnitTestsTsan: - needs: [BuilderDebTsan] - runs-on: [self-hosted, fuzzer-unit-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/unit_tests_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Unit tests (tsan) - REPO_COPY=${{runner.temp}}/unit_tests_tsan/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Unit test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 unit_tests_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - UnitTestsMsan: - needs: [BuilderDebMsan] - runs-on: [self-hosted, fuzzer-unit-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/unit_tests_msan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Unit tests (msan) - REPO_COPY=${{runner.temp}}/unit_tests_msan/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Unit test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 unit_tests_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - UnitTestsUBsan: - needs: [BuilderDebUBsan] - runs-on: [self-hosted, fuzzer-unit-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/unit_tests_ubsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Unit tests (ubsan) - REPO_COPY=${{runner.temp}}/unit_tests_ubsan/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Unit test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 unit_tests_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -############################################################################################# -#################################### PERFORMANCE TESTS ###################################### -############################################################################################# - PerformanceComparisonX86-0: - needs: [BuilderDebRelease] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/performance_comparison - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Performance Comparison - REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Performance Comparison - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 performance_comparison_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - PerformanceComparisonX86-1: - needs: [BuilderDebRelease] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/performance_comparison - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Performance Comparison - REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Performance Comparison - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 performance_comparison_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - PerformanceComparisonX86-2: - needs: [BuilderDebRelease] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/performance_comparison - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Performance Comparison - REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Performance Comparison - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 performance_comparison_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - PerformanceComparisonX86-3: - needs: [BuilderDebRelease] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/performance_comparison - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Performance Comparison - REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse - RUN_BY_HASH_NUM=3 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Performance Comparison - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 performance_comparison_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - PerformanceComparisonAarch-0: - needs: [BuilderDebAarch64] - runs-on: [self-hosted, func-tester-aarch64] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/performance_comparison - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Performance Comparison Aarch64 - REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Performance Comparison - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 performance_comparison_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - PerformanceComparisonAarch-1: - needs: [BuilderDebAarch64] - runs-on: [self-hosted, func-tester-aarch64] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/performance_comparison - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Performance Comparison Aarch64 - REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Performance Comparison - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 performance_comparison_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - PerformanceComparisonAarch-2: - needs: [BuilderDebAarch64] - runs-on: [self-hosted, func-tester-aarch64] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/performance_comparison - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Performance Comparison Aarch64 - REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Performance Comparison - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 performance_comparison_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - PerformanceComparisonAarch-3: - needs: [BuilderDebAarch64] - runs-on: [self-hosted, func-tester-aarch64] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/performance_comparison - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Performance Comparison Aarch64 - REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse - RUN_BY_HASH_NUM=3 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Performance Comparison - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 performance_comparison_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -############################################################################################## -###################################### SQLANCER FUZZERS ###################################### -############################################################################################## - SQLancerTestRelease: - needs: [BuilderDebRelease] - runs-on: [self-hosted, fuzzer-unit-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/sqlancer_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=SQLancer (release) - REPO_COPY=${{runner.temp}}/sqlancer_release/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: SQLancer - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 sqlancer_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" - SQLancerTestDebug: - needs: [BuilderDebDebug] - runs-on: [self-hosted, fuzzer-unit-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/sqlancer_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=SQLancer (debug) - REPO_COPY=${{runner.temp}}/sqlancer_debug/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v3 - with: - path: ${{ env.REPORTS_PATH }} - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: SQLancer - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 sqlancer_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - docker ps --quiet | xargs --no-run-if-empty docker kill ||: - docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: - sudo rm -fr "$TEMP_PATH" -############################################################################################# -###################################### JEPSEN TESTS ######################################### -############################################################################################# - Jepsen: - # This is special test NOT INCLUDED in FinishCheck - # When it's skipped, all dependent tasks will be skipped too. - # DO NOT add it there - if: contains(github.event.pull_request.labels.*.name, 'jepsen-test') - needs: [BuilderBinRelease] - uses: ./.github/workflows/jepsen.yml - FinishCheck: - needs: - - StyleCheck - - DockerHubPush - - DockerServerImages - - CheckLabels - - BuilderReport - - BuilderSpecialReport - - FastTest - - FunctionalStatelessTestDebug0 - - FunctionalStatelessTestDebug1 - - FunctionalStatelessTestDebug2 - - FunctionalStatelessTestDebug3 - - FunctionalStatelessTestDebug4 - - FunctionalStatelessTestRelease - - FunctionalStatelessTestReleaseDatabaseReplicated0 - - FunctionalStatelessTestReleaseDatabaseReplicated1 - - FunctionalStatelessTestReleaseDatabaseReplicated2 - - FunctionalStatelessTestReleaseDatabaseReplicated3 - - FunctionalStatelessTestReleaseWideParts - - FunctionalStatelessTestReleaseAnalyzer - - FunctionalStatelessTestAarch64 - - FunctionalStatelessTestAsan0 - - FunctionalStatelessTestAsan1 - - FunctionalStatelessTestAsan2 - - FunctionalStatelessTestAsan3 - - FunctionalStatelessTestTsan0 - - FunctionalStatelessTestTsan1 - - FunctionalStatelessTestTsan2 - - FunctionalStatelessTestTsan3 - - FunctionalStatelessTestTsan4 - - FunctionalStatelessTestMsan0 - - FunctionalStatelessTestMsan1 - - FunctionalStatelessTestMsan2 - - FunctionalStatelessTestMsan3 - - FunctionalStatelessTestMsan4 - - FunctionalStatelessTestMsan5 - - FunctionalStatelessTestUBsan0 - - FunctionalStatelessTestUBsan1 - - FunctionalStatefulTestDebug - - FunctionalStatefulTestRelease - - FunctionalStatefulTestAarch64 - - FunctionalStatefulTestAsan - - FunctionalStatefulTestTsan - - FunctionalStatefulTestMsan - - FunctionalStatefulTestUBsan - - FunctionalStatelessTestReleaseS3_0 - - FunctionalStatelessTestReleaseS3_1 - - FunctionalStatelessTestS3Debug0 - - FunctionalStatelessTestS3Debug1 - - FunctionalStatelessTestS3Debug2 - - FunctionalStatelessTestS3Debug4 - - FunctionalStatelessTestS3Debug5 - - FunctionalStatelessTestS3Tsan0 - - FunctionalStatelessTestS3Tsan1 - - FunctionalStatelessTestS3Tsan2 - - FunctionalStatelessTestS3Tsan4 - - StressTestDebug - - StressTestAsan - - StressTestTsan - - StressTestMsan - - StressTestUBsan - - ASTFuzzerTestDebug - - ASTFuzzerTestAsan - - ASTFuzzerTestTsan - - ASTFuzzerTestMSan - - ASTFuzzerTestUBSan - - IntegrationTestsAsan0 - - IntegrationTestsAsan1 - - IntegrationTestsAsan2 - - IntegrationTestsAsan3 - - IntegrationTestsAsan4 - - IntegrationTestsAsan5 - - IntegrationTestsRelease0 - - IntegrationTestsRelease1 - - IntegrationTestsRelease2 - - IntegrationTestsRelease3 - - IntegrationTestsTsan0 - - IntegrationTestsTsan1 - - IntegrationTestsTsan2 - - IntegrationTestsTsan3 - - IntegrationTestsTsan4 - - IntegrationTestsTsan5 - - PerformanceComparisonX86-0 - - PerformanceComparisonX86-1 - - PerformanceComparisonX86-2 - - PerformanceComparisonX86-3 - - PerformanceComparisonAarch-0 - - PerformanceComparisonAarch-1 - - PerformanceComparisonAarch-2 - - PerformanceComparisonAarch-3 - - UnitTestsAsan - - UnitTestsTsan - - UnitTestsMsan - - UnitTestsUBsan - - UnitTestsReleaseClang - - CompatibilityCheckX86 - - CompatibilityCheckAarch64 - - IntegrationTestsFlakyCheck - - SQLancerTestRelease - - SQLancerTestDebug - runs-on: [self-hosted, style-checker] - steps: - - name: Check out repository code - uses: ClickHouse/checkout@v1 - with: - clear-repository: true - - name: Finish label - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 finish_check.py - python3 merge_pr.py --check-approved -############################################################################################## -########################### SQLLOGIC TEST ################################################### -############################################################################################## - SQLLogicTestRelease: - needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/sqllogic_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Sqllogic test (release) - REPO_COPY=${{runner.temp}}/sqllogic_debug/ClickHouse - KILL_TIMEOUT=10800 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Sqllogic test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 sqllogic_test.py "$CHECK_NAME" "$KILL_TIMEOUT" + cd "$GITHUB_WORKSPACE/tests/ci" + python3 docker_server.py --release-type head --no-push \ + --image-repo clickhouse/clickhouse-server --image-path docker/server + python3 docker_server.py --release-type head --no-push \ + --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper - name: Cleanup if: always() run: | docker ps --quiet | xargs --no-run-if-empty docker kill ||: docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" +# ############################################################################################ +# ##################################### BUILD REPORTER ####################################### +# ############################################################################################ +# BuilderReport: +# needs: +# - BuilderBinRelease +# - BuilderDebAarch64 +# - BuilderDebAsan +# - BuilderDebDebug +# - BuilderDebMsan +# - BuilderDebRelease +# - BuilderDebTsan +# - BuilderDebUBsan +# runs-on: [self-hosted, style-checker] +# if: ${{ success() || failure() }} +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# CHECK_NAME=ClickHouse build check +# REPORTS_PATH=${{runner.temp}}/reports_dir +# TEMP_PATH=${{runner.temp}}/report_check +# NEEDS_DATA_PATH=${{runner.temp}}/needs.json +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Report Builder +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cat > "$NEEDS_DATA_PATH" << 'EOF' +# ${{ toJSON(needs) }} +# EOF +# cd "$GITHUB_WORKSPACE/tests/ci" +# python3 build_report_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# BuilderSpecialReport: +# needs: +# - BuilderBinAarch64 +# - BuilderBinDarwin +# - BuilderBinDarwinAarch64 +# - BuilderBinFreeBSD +# - BuilderBinPPC64 +# - BuilderBinAmd64Compat +# - BuilderBinAarch64V80Compat +# - BuilderBinClangTidy +# runs-on: [self-hosted, style-checker] +# if: ${{ success() || failure() }} +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/report_check +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=ClickHouse special build check +# NEEDS_DATA_PATH=${{runner.temp}}/needs.json +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Report Builder +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cat > "$NEEDS_DATA_PATH" << 'EOF' +# ${{ toJSON(needs) }} +# EOF +# cd "$GITHUB_WORKSPACE/tests/ci" +# python3 build_report_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################ +# #################################### INSTALL PACKAGES ###################################### +# ############################################################################################ +# InstallPackagesTestRelease: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, style-checker] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/test_install +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Install packages (amd64) +# REPO_COPY=${{runner.temp}}/test_install/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Test packages installation +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 install_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# InstallPackagesTestAarch64: +# needs: [BuilderDebAarch64] +# runs-on: [self-hosted, style-checker-aarch64] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/test_install +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Install packages (arm64) +# REPO_COPY=${{runner.temp}}/test_install/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Test packages installation +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 install_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################## +# ########################### FUNCTIONAl STATELESS TESTS ####################################### +# ############################################################################################## +# FunctionalStatelessTestRelease: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_release +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (release) +# REPO_COPY=${{runner.temp}}/stateless_release/ClickHouse +# KILL_TIMEOUT=10800 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestReleaseDatabaseReplicated0: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_database_replicated +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (release, DatabaseReplicated) +# REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestReleaseDatabaseReplicated1: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_database_replicated +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (release, DatabaseReplicated) +# REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestReleaseDatabaseReplicated2: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_database_replicated +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (release, DatabaseReplicated) +# REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestReleaseDatabaseReplicated3: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_database_replicated +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (release, DatabaseReplicated) +# REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestReleaseWideParts: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_wide_parts +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (release, wide parts enabled) +# REPO_COPY=${{runner.temp}}/stateless_wide_parts/ClickHouse +# KILL_TIMEOUT=10800 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestReleaseAnalyzer: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_analyzer +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (release, analyzer) +# REPO_COPY=${{runner.temp}}/stateless_analyzer/ClickHouse +# KILL_TIMEOUT=10800 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestReleaseS3_0: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (release, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=2 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestReleaseS3_1: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (release, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=2 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Debug0: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Debug1: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Debug2: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Debug3: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Debug4: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=4 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Debug5: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=5 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Tsan0: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Tsan1: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Tsan2: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Tsan3: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Tsan4: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=4 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestAarch64: +# needs: [BuilderDebAarch64] +# runs-on: [self-hosted, func-tester-aarch64] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_release +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (aarch64) +# REPO_COPY=${{runner.temp}}/stateless_release/ClickHouse +# KILL_TIMEOUT=10800 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestAsan0: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (asan) +# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestAsan1: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (asan) +# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestAsan2: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (asan) +# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestAsan3: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (asan) +# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestTsan0: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan) +# REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestTsan1: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan) +# REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestTsan2: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan) +# REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestTsan3: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan) +# REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestTsan4: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan) +# REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=4 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestUBsan0: +# needs: [BuilderDebUBsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_ubsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (ubsan) +# REPO_COPY=${{runner.temp}}/stateless_ubsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=2 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestUBsan1: +# needs: [BuilderDebUBsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_ubsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (ubsan) +# REPO_COPY=${{runner.temp}}/stateless_ubsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=2 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestMsan0: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_memory +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (msan) +# REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestMsan1: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_memory +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (msan) +# REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestMsan2: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_memory +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (msan) +# REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestMsan3: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_memory +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (msan) +# REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestMsan4: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_memory +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (msan) +# REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=4 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestMsan5: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_memory +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (msan) +# REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=5 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestDebug0: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug) +# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestDebug1: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug) +# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestDebug2: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug) +# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestDebug3: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug) +# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestDebug4: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug) +# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=4 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestFlakyCheck: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_flaky_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests flaky check (asan) +# REPO_COPY=${{runner.temp}}/stateless_flaky_asan/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# TestsBugfixCheck: +# needs: [CheckLabels, StyleCheck] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/tests_bugfix_check +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=tests bugfix validate check +# KILL_TIMEOUT=3600 +# REPO_COPY=${{runner.temp}}/tests_bugfix_check/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Bugfix test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" + +# TEMP_PATH="${TEMP_PATH}/integration" \ +# REPORTS_PATH="${REPORTS_PATH}/integration" \ +# python3 integration_test_check.py "Integration $CHECK_NAME" \ +# --validate-bugfix --post-commit-status=file || echo 'ignore exit code' + +# TEMP_PATH="${TEMP_PATH}/stateless" \ +# REPORTS_PATH="${REPORTS_PATH}/stateless" \ +# python3 functional_test_check.py "Stateless $CHECK_NAME" "$KILL_TIMEOUT" \ +# --validate-bugfix --post-commit-status=file || echo 'ignore exit code' + +# python3 bugfix_validate_check.py "${TEMP_PATH}/stateless/functional_commit_status.tsv" "${TEMP_PATH}/integration/integration_commit_status.tsv" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################## +# ############################ FUNCTIONAl STATEFUL TESTS ####################################### +# ############################################################################################## +# FunctionalStatefulTestRelease: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_release +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (release) +# REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatefulTestAarch64: +# needs: [BuilderDebAarch64] +# runs-on: [self-hosted, func-tester-aarch64] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_release +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (aarch64) +# REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatefulTestAsan: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (asan) +# REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatefulTestTsan: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (tsan) +# REPO_COPY=${{runner.temp}}/stateful_tsan/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatefulTestMsan: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_msan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (msan) +# REPO_COPY=${{runner.temp}}/stateful_msan/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatefulTestUBsan: +# needs: [BuilderDebUBsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_ubsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (ubsan) +# REPO_COPY=${{runner.temp}}/stateful_ubsan/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatefulTestDebug: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (debug) +# REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# # Parallel replicas +# FunctionalStatefulTestDebugParallelReplicas: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (debug, ParallelReplicas) +# REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v2 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Clear repository +# run: | +# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# - name: Check out repository code +# uses: actions/checkout@v2 +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatefulTestUBsanParallelReplicas: +# needs: [BuilderDebUBsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_ubsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (ubsan, ParallelReplicas) +# REPO_COPY=${{runner.temp}}/stateful_ubsan/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v2 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Clear repository +# run: | +# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# - name: Check out repository code +# uses: actions/checkout@v2 +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatefulTestMsanParallelReplicas: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_msan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (msan, ParallelReplicas) +# REPO_COPY=${{runner.temp}}/stateful_msan/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v2 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Clear repository +# run: | +# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# - name: Check out repository code +# uses: actions/checkout@v2 +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatefulTestTsanParallelReplicas: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (tsan, ParallelReplicas) +# REPO_COPY=${{runner.temp}}/stateful_tsan/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v2 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Clear repository +# run: | +# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# - name: Check out repository code +# uses: actions/checkout@v2 +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatefulTestAsanParallelReplicas: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (asan, ParallelReplicas) +# REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v2 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Clear repository +# run: | +# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# - name: Check out repository code +# uses: actions/checkout@v2 +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatefulTestReleaseParallelReplicas: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_release +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (release, ParallelReplicas) +# REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v2 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Clear repository +# run: | +# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# - name: Check out repository code +# uses: actions/checkout@v2 +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################## +# ######################################### STRESS TESTS ####################################### +# ############################################################################################## +# StressTestAsan: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stress_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stress test (asan) +# REPO_COPY=${{runner.temp}}/stress_asan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Stress test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 stress_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# StressTestTsan: +# needs: [BuilderDebTsan] +# # func testers have 16 cores + 128 GB memory +# # while stress testers have 36 cores + 72 memory +# # It would be better to have something like 32 + 128, +# # but such servers almost unavailable as spot instances. +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stress_thread +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stress test (tsan) +# REPO_COPY=${{runner.temp}}/stress_thread/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Stress test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 stress_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# StressTestMsan: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stress_memory +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stress test (msan) +# REPO_COPY=${{runner.temp}}/stress_memory/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Stress test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 stress_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# StressTestUBsan: +# needs: [BuilderDebUBsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stress_undefined +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stress test (ubsan) +# REPO_COPY=${{runner.temp}}/stress_undefined/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Stress test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 stress_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# StressTestDebug: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stress_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stress test (debug) +# REPO_COPY=${{runner.temp}}/stress_debug/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Stress test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 stress_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################## +# ######################################### UPGRADE CHECK ###################################### +# ############################################################################################## +# UpgradeCheckAsan: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/upgrade_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Upgrade check (asan) +# REPO_COPY=${{runner.temp}}/upgrade_asan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Upgrade check +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 upgrade_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# UpgradeCheckTsan: +# needs: [BuilderDebTsan] +# # same as for stress test with tsan +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/upgrade_thread +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Upgrade check (tsan) +# REPO_COPY=${{runner.temp}}/upgrade_thread/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Upgrade check +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 upgrade_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# UpgradeCheckMsan: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/upgrade_memory +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Upgrade check (msan) +# REPO_COPY=${{runner.temp}}/upgrade_memory/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Upgrade check +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 upgrade_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# UpgradeCheckDebug: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/upgrade_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Upgrade check (debug) +# REPO_COPY=${{runner.temp}}/upgrade_debug/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Upgrade check +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 upgrade_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################## +# ##################################### AST FUZZERS ############################################ +# ############################################################################################## +# ASTFuzzerTestAsan: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/ast_fuzzer_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=AST fuzzer (asan) +# REPO_COPY=${{runner.temp}}/ast_fuzzer_asan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Fuzzer +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 ast_fuzzer_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ASTFuzzerTestTsan: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/ast_fuzzer_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=AST fuzzer (tsan) +# REPO_COPY=${{runner.temp}}/ast_fuzzer_tsan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Fuzzer +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 ast_fuzzer_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ASTFuzzerTestUBSan: +# needs: [BuilderDebUBsan] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/ast_fuzzer_ubsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=AST fuzzer (ubsan) +# REPO_COPY=${{runner.temp}}/ast_fuzzer_ubsan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Fuzzer +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 ast_fuzzer_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ASTFuzzerTestMSan: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/ast_fuzzer_msan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=AST fuzzer (msan) +# REPO_COPY=${{runner.temp}}/ast_fuzzer_msan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Fuzzer +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 ast_fuzzer_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ASTFuzzerTestDebug: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/ast_fuzzer_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=AST fuzzer (debug) +# REPO_COPY=${{runner.temp}}/ast_fuzzer_debug/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Fuzzer +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 ast_fuzzer_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################# +# ############################# INTEGRATION TESTS ############################################# +# ############################################################################################# +# IntegrationTestsAsan0: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (asan) +# REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsAsan1: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (asan) +# REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsAsan2: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (asan) +# REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsAsan3: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (asan) +# REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsAsan4: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (asan) +# REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse +# RUN_BY_HASH_NUM=4 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsAsan5: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (asan) +# REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse +# RUN_BY_HASH_NUM=5 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsTsan0: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (tsan) +# REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsTsan1: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (tsan) +# REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsTsan2: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (tsan) +# REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsTsan3: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (tsan) +# REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsTsan4: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (tsan) +# REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse +# RUN_BY_HASH_NUM=4 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsTsan5: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (tsan) +# REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse +# RUN_BY_HASH_NUM=5 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsRelease0: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_release +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (release) +# REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsRelease1: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_release +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (release) +# REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsRelease2: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_release +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (release) +# REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsRelease3: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_release +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (release) +# REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsFlakyCheck: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_asan_flaky_check +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests flaky check (asan) +# REPO_COPY=${{runner.temp}}/integration_tests_asan_flaky_check/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################# +# #################################### UNIT TESTS ############################################# +# ############################################################################################# +# UnitTestsAsan: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/unit_tests_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Unit tests (asan) +# REPO_COPY=${{runner.temp}}/unit_tests_asan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Unit test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 unit_tests_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# UnitTestsReleaseClang: +# needs: [BuilderBinRelease] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/unit_tests_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Unit tests (release-clang) +# REPO_COPY=${{runner.temp}}/unit_tests_asan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Unit test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 unit_tests_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# UnitTestsTsan: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/unit_tests_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Unit tests (tsan) +# REPO_COPY=${{runner.temp}}/unit_tests_tsan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Unit test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 unit_tests_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# UnitTestsMsan: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/unit_tests_msan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Unit tests (msan) +# REPO_COPY=${{runner.temp}}/unit_tests_msan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Unit test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 unit_tests_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# UnitTestsUBsan: +# needs: [BuilderDebUBsan] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/unit_tests_ubsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Unit tests (ubsan) +# REPO_COPY=${{runner.temp}}/unit_tests_ubsan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Unit test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 unit_tests_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################# +# #################################### PERFORMANCE TESTS ###################################### +# ############################################################################################# +# PerformanceComparisonX86-0: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/performance_comparison +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Performance Comparison +# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Performance Comparison +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 performance_comparison_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# PerformanceComparisonX86-1: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/performance_comparison +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Performance Comparison +# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Performance Comparison +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 performance_comparison_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# PerformanceComparisonX86-2: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/performance_comparison +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Performance Comparison +# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Performance Comparison +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 performance_comparison_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# PerformanceComparisonX86-3: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/performance_comparison +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Performance Comparison +# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Performance Comparison +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 performance_comparison_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# PerformanceComparisonAarch-0: +# needs: [BuilderDebAarch64] +# runs-on: [self-hosted, func-tester-aarch64] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/performance_comparison +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Performance Comparison Aarch64 +# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Performance Comparison +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 performance_comparison_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# PerformanceComparisonAarch-1: +# needs: [BuilderDebAarch64] +# runs-on: [self-hosted, func-tester-aarch64] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/performance_comparison +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Performance Comparison Aarch64 +# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Performance Comparison +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 performance_comparison_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# PerformanceComparisonAarch-2: +# needs: [BuilderDebAarch64] +# runs-on: [self-hosted, func-tester-aarch64] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/performance_comparison +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Performance Comparison Aarch64 +# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Performance Comparison +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 performance_comparison_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# PerformanceComparisonAarch-3: +# needs: [BuilderDebAarch64] +# runs-on: [self-hosted, func-tester-aarch64] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/performance_comparison +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Performance Comparison Aarch64 +# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Performance Comparison +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 performance_comparison_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################## +# ###################################### SQLANCER FUZZERS ###################################### +# ############################################################################################## +# SQLancerTestRelease: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/sqlancer_release +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=SQLancer (release) +# REPO_COPY=${{runner.temp}}/sqlancer_release/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: SQLancer +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 sqlancer_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# SQLancerTestDebug: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/sqlancer_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=SQLancer (debug) +# REPO_COPY=${{runner.temp}}/sqlancer_debug/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: SQLancer +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 sqlancer_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################# +# ###################################### JEPSEN TESTS ######################################### +# ############################################################################################# +# Jepsen: +# # This is special test NOT INCLUDED in FinishCheck +# # When it's skipped, all dependent tasks will be skipped too. +# # DO NOT add it there +# if: contains(github.event.pull_request.labels.*.name, 'jepsen-test') +# needs: [BuilderBinRelease] +# uses: ./.github/workflows/jepsen.yml +# FinishCheck: +# needs: +# - StyleCheck +# - DockerHubPush +# - DockerServerImages +# - CheckLabels +# - BuilderReport +# - BuilderSpecialReport +# - FastTest +# - FunctionalStatelessTestDebug0 +# - FunctionalStatelessTestDebug1 +# - FunctionalStatelessTestDebug2 +# - FunctionalStatelessTestDebug3 +# - FunctionalStatelessTestDebug4 +# - FunctionalStatelessTestRelease +# - FunctionalStatelessTestReleaseDatabaseReplicated0 +# - FunctionalStatelessTestReleaseDatabaseReplicated1 +# - FunctionalStatelessTestReleaseDatabaseReplicated2 +# - FunctionalStatelessTestReleaseDatabaseReplicated3 +# - FunctionalStatelessTestReleaseWideParts +# - FunctionalStatelessTestReleaseAnalyzer +# - FunctionalStatelessTestAarch64 +# - FunctionalStatelessTestAsan0 +# - FunctionalStatelessTestAsan1 +# - FunctionalStatelessTestAsan2 +# - FunctionalStatelessTestAsan3 +# - FunctionalStatelessTestTsan0 +# - FunctionalStatelessTestTsan1 +# - FunctionalStatelessTestTsan2 +# - FunctionalStatelessTestTsan3 +# - FunctionalStatelessTestTsan4 +# - FunctionalStatelessTestMsan0 +# - FunctionalStatelessTestMsan1 +# - FunctionalStatelessTestMsan2 +# - FunctionalStatelessTestMsan3 +# - FunctionalStatelessTestMsan4 +# - FunctionalStatelessTestMsan5 +# - FunctionalStatelessTestUBsan0 +# - FunctionalStatelessTestUBsan1 +# - FunctionalStatefulTestDebug +# - FunctionalStatefulTestRelease +# - FunctionalStatefulTestAarch64 +# - FunctionalStatefulTestAsan +# - FunctionalStatefulTestTsan +# - FunctionalStatefulTestMsan +# - FunctionalStatefulTestUBsan +# - FunctionalStatelessTestReleaseS3_0 +# - FunctionalStatelessTestReleaseS3_1 +# - FunctionalStatelessTestS3Debug0 +# - FunctionalStatelessTestS3Debug1 +# - FunctionalStatelessTestS3Debug2 +# - FunctionalStatelessTestS3Debug4 +# - FunctionalStatelessTestS3Debug5 +# - FunctionalStatelessTestS3Tsan0 +# - FunctionalStatelessTestS3Tsan1 +# - FunctionalStatelessTestS3Tsan2 +# - FunctionalStatelessTestS3Tsan4 +# - StressTestDebug +# - StressTestAsan +# - StressTestTsan +# - StressTestMsan +# - StressTestUBsan +# - ASTFuzzerTestDebug +# - ASTFuzzerTestAsan +# - ASTFuzzerTestTsan +# - ASTFuzzerTestMSan +# - ASTFuzzerTestUBSan +# - IntegrationTestsAsan0 +# - IntegrationTestsAsan1 +# - IntegrationTestsAsan2 +# - IntegrationTestsAsan3 +# - IntegrationTestsAsan4 +# - IntegrationTestsAsan5 +# - IntegrationTestsRelease0 +# - IntegrationTestsRelease1 +# - IntegrationTestsRelease2 +# - IntegrationTestsRelease3 +# - IntegrationTestsTsan0 +# - IntegrationTestsTsan1 +# - IntegrationTestsTsan2 +# - IntegrationTestsTsan3 +# - IntegrationTestsTsan4 +# - IntegrationTestsTsan5 +# - PerformanceComparisonX86-0 +# - PerformanceComparisonX86-1 +# - PerformanceComparisonX86-2 +# - PerformanceComparisonX86-3 +# - PerformanceComparisonAarch-0 +# - PerformanceComparisonAarch-1 +# - PerformanceComparisonAarch-2 +# - PerformanceComparisonAarch-3 +# - UnitTestsAsan +# - UnitTestsTsan +# - UnitTestsMsan +# - UnitTestsUBsan +# - UnitTestsReleaseClang +# - CompatibilityCheckX86 +# - CompatibilityCheckAarch64 +# - IntegrationTestsFlakyCheck +# - SQLancerTestRelease +# - SQLancerTestDebug +# runs-on: [self-hosted, style-checker] +# steps: +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Finish label +# run: | +# cd "$GITHUB_WORKSPACE/tests/ci" +# python3 finish_check.py +# python3 merge_pr.py --check-approved +# ############################################################################################## +# ########################### SQLLOGIC TEST ################################################### +# ############################################################################################## +# SQLLogicTestRelease: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/sqllogic_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Sqllogic test (release) +# REPO_COPY=${{runner.temp}}/sqllogic_debug/ClickHouse +# KILL_TIMEOUT=10800 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v2 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Clear repository +# run: | +# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# - name: Check out repository code +# uses: actions/checkout@v2 +# - name: Sqllogic test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 sqllogic_test.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" diff --git a/.github/workflows/pull_test.yaml b/.github/workflows/pull_test.yaml new file mode 100644 index 000000000000..e84954a0c215 --- /dev/null +++ b/.github/workflows/pull_test.yaml @@ -0,0 +1,4929 @@ +# name: PullTestCI + +# env: +# # Force the stdout and stderr streams to be unbuffered +# PYTHONUNBUFFERED: 1 + +# on: # yamllint disable-line rule:truthy +# push: +# # pull_request: +# # types: +# # - synchronize +# # - reopened +# # - opened +# # branches: +# # - master +# # paths-ignore: +# # - 'CHANGELOG.md' +# # - 'README.md' +# # - 'SECURITY.md' +# # - 'docker/docs/**' +# # - 'docs/**' +# # - 'utils/check-style/aspell-ignore/**' +# ########################################################################################## +# ##################################### SMALL CHECKS ####################################### +# ########################################################################################## +# jobs: + # CheckLabels: + # runs-on: [self-hosted, style-checker] + # # Run the first check always, even if the CI is cancelled + # if: ${{ always() }} + # steps: + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Labels check + # run: | + # cd "$GITHUB_WORKSPACE/tests/ci" + # python3 run_check.py + # PythonUnitTests: + # runs-on: [self-hosted, style-checker] + # steps: + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Python unit tests + # run: | + # cd "$GITHUB_WORKSPACE/tests/ci" + # echo "Testing the main ci directory" + # python3 -m unittest discover -s . -p '*_test.py' + # for dir in *_lambda/; do + # echo "Testing $dir" + # python3 -m unittest discover -s "$dir" -p '*_test.py' + # done + # DockerHubPushAarch64: + # # needs: CheckLabels + # runs-on: [self-hosted, test-only] + # steps: + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Images check + # run: | + # cd "$GITHUB_WORKSPACE/tests/ci" + # python3 docker_images_check.py --suffix aarch64 + # - name: Upload images files to artifacts + # uses: actions/upload-artifact@v3 + # with: + # name: changed_images_aarch64 + # path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json + # DockerHubPushAmd64: + # # needs: CheckLabels + # runs-on: [self-hosted, test-only] + # steps: + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Images check + # run: | + # cd "$GITHUB_WORKSPACE/tests/ci" + # python3 docker_images_check.py --suffix amd64 + # - name: Upload images files to artifacts + # uses: actions/upload-artifact@v3 + # with: + # name: changed_images_amd64 + # path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json + # DockerHubPush: + # needs: [DockerHubPushAmd64, DockerHubPushAarch64] + # runs-on: [self-hosted, builder] + # steps: + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Download changed aarch64 images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images_aarch64 + # path: ${{ runner.temp }} + # - name: Download changed amd64 images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images_amd64 + # path: ${{ runner.temp }} + # - name: Images check + # run: | + # cd "$GITHUB_WORKSPACE/tests/ci" + # python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 + # - name: Upload images files to artifacts + # uses: actions/upload-artifact@v3 + # with: + # name: changed_images + # path: ${{ runner.temp }}/changed_images.json + # StyleCheck: + # needs: DockerHubPush + # runs-on: [self-hosted, builder] + # # We need additional `&& ! cancelled()` to have the job being able to cancel + # if: ${{ success() || failure() || ( always() && ! cancelled() ) }} + # steps: + # - name: Set envs + # run: echo "hi" + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{ runner.temp }}/style_check + # ROBOT_CLICKHOUSE_SSH_KEY<> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/fasttest + # REPO_COPY=${{runner.temp}}/fasttest/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # EOF + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Download changed images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images + # path: ${{ env.TEMP_PATH }} + # - name: Fast Test + # run: | + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 fast_test_check.py + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +## CompatibilityCheckX86: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, style-checker] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/compatibility_check +# REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse +# REPORTS_PATH=${{runner.temp}}/reports_dir +# EOF +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: CompatibilityCheckX86 +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (amd64)" --check-glibc --check-distributions +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# CompatibilityCheckAarch64: +# needs: [BuilderDebAarch64] +# runs-on: [self-hosted, style-checker] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/compatibility_check +# REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse +# REPORTS_PATH=${{runner.temp}}/reports_dir +# EOF +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: CompatibilityCheckAarch64 +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (aarch64)" --check-glibc +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ######################################################################################### +# #################################### ORDINARY BUILDS #################################### +# ######################################################################################### + # BuilderDebRelease: + # needs: [DockerHubPush, FastTest] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/build_check + # IMAGES_PATH=${{runner.temp}}/images_path + # REPO_COPY=${{runner.temp}}/build_check/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # BUILD_NAME=package_release + # EOF + # - name: Download changed images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images + # path: ${{ env.IMAGES_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # fetch-depth: 0 # for performance artifact + # submodules: true + # - name: Build + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + # - name: Upload build URLs to artifacts + # if: ${{ success() || failure() }} + # uses: actions/upload-artifact@v3 + # with: + # name: ${{ env.BUILD_URLS }} + # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" +# BuilderBinRelease: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_release +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + # BuilderDebAarch64: + # needs: [DockerHubPush, FastTest, StyleCheck] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/build_check + # IMAGES_PATH=${{runner.temp}}/images_path + # REPO_COPY=${{runner.temp}}/build_check/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # BUILD_NAME=package_aarch64 + # EOF + # - name: Download changed images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images + # path: ${{ runner.temp }}/images_path + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # submodules: true + # fetch-depth: 0 # for performance artifact + # - name: Build + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + # - name: Upload build URLs to artifacts + # if: ${{ success() || failure() }} + # uses: actions/upload-artifact@v3 + # with: + # name: ${{ env.BUILD_URLS }} + # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + # BuilderDebAsan: + # needs: [DockerHubPush, FastTest, StyleCheck] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/build_check + # IMAGES_PATH=${{runner.temp}}/images_path + # REPO_COPY=${{runner.temp}}/build_check/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # BUILD_NAME=package_asan + # EOF + # - name: Download changed images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images + # path: ${{ env.IMAGES_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # submodules: true + # - name: Build + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + # - name: Upload build URLs to artifacts + # if: ${{ success() || failure() }} + # uses: actions/upload-artifact@v3 + # with: + # name: ${{ env.BUILD_URLS }} + # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + # BuilderDebUBsan: + # needs: [DockerHubPush, FastTest, StyleCheck] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/build_check + # IMAGES_PATH=${{runner.temp}}/images_path + # REPO_COPY=${{runner.temp}}/build_check/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # BUILD_NAME=package_ubsan + # EOF + # - name: Download changed images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images + # path: ${{ env.IMAGES_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # submodules: true + # - name: Build + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + # - name: Upload build URLs to artifacts + # if: ${{ success() || failure() }} + # uses: actions/upload-artifact@v3 + # with: + # name: ${{ env.BUILD_URLS }} + # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + # BuilderDebTsan: + # needs: [DockerHubPush, FastTest, StyleCheck] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/build_check + # IMAGES_PATH=${{runner.temp}}/images_path + # REPO_COPY=${{runner.temp}}/build_check/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # BUILD_NAME=package_tsan + # EOF + # - name: Download changed images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images + # path: ${{ env.IMAGES_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # submodules: true + # - name: Build + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + # - name: Upload build URLs to artifacts + # if: ${{ success() || failure() }} + # uses: actions/upload-artifact@v3 + # with: + # name: ${{ env.BUILD_URLS }} + # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + # BuilderDebMsan: + # needs: [DockerHubPush, FastTest, StyleCheck] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/build_check + # IMAGES_PATH=${{runner.temp}}/images_path + # REPO_COPY=${{runner.temp}}/build_check/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # BUILD_NAME=package_msan + # EOF + # - name: Download changed images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images + # path: ${{ env.IMAGES_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # submodules: true + # - name: Build + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + # - name: Upload build URLs to artifacts + # if: ${{ success() || failure() }} + # uses: actions/upload-artifact@v3 + # with: + # name: ${{ env.BUILD_URLS }} + # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + # BuilderDebDebug: + # needs: [DockerHubPush, FastTest, StyleCheck] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/build_check + # IMAGES_PATH=${{runner.temp}}/images_path + # REPO_COPY=${{runner.temp}}/build_check/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # BUILD_NAME=package_debug + # EOF + # - name: Download changed images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images + # path: ${{ env.IMAGES_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # submodules: true + # - name: Apply sparse checkout for contrib # in order to check that it doesn't break build + # run: | + # rm -rf "$GITHUB_WORKSPACE/contrib" && echo 'removed' + # git -C "$GITHUB_WORKSPACE" checkout . && echo 'restored' + # "$GITHUB_WORKSPACE/contrib/update-submodules.sh" && echo 'OK' + # du -hs "$GITHUB_WORKSPACE/contrib" ||: + # find "$GITHUB_WORKSPACE/contrib" -type f | wc -l ||: + # - name: Build + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + # - name: Upload build URLs to artifacts + # if: ${{ success() || failure() }} + # uses: actions/upload-artifact@v3 + # with: + # name: ${{ env.BUILD_URLS }} + # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# ########################################################################################## +# ##################################### SPECIAL BUILDS ##################################### +# ########################################################################################## +# BuilderBinClangTidy: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_tidy +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + # BuilderBinDarwin: + # needs: [DockerHubPush, FastTest, StyleCheck] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/build_check + # IMAGES_PATH=${{runner.temp}}/images_path + # REPO_COPY=${{runner.temp}}/build_check/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # BUILD_NAME=binary_darwin + # EOF + # - name: Download changed images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images + # path: ${{ env.IMAGES_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # submodules: true + # - name: Build + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + # - name: Upload build URLs to artifacts + # if: ${{ success() || failure() }} + # uses: actions/upload-artifact@v3 + # with: + # name: ${{ env.BUILD_URLS }} + # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinAarch64: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_aarch64 +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinFreeBSD: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_freebsd +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + # BuilderBinDarwinAarch64: + # needs: [DockerHubPush, FastTest, StyleCheck] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/build_check + # IMAGES_PATH=${{runner.temp}}/images_path + # REPO_COPY=${{runner.temp}}/build_check/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # BUILD_NAME=binary_darwin_aarch64 + # EOF + # - name: Download changed images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images + # path: ${{ env.IMAGES_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # submodules: true + # - name: Build + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + # - name: Upload build URLs to artifacts + # if: ${{ success() || failure() }} + # uses: actions/upload-artifact@v3 + # with: + # name: ${{ env.BUILD_URLS }} + # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinPPC64: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_ppc64le +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinAmd64Compat: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_amd64_compat +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinAarch64V80Compat: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_aarch64_v80compat +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# ############################################################################################ +# ##################################### Docker images ####################################### +# ############################################################################################ + # DockerServerImages: + # needs: + # - BuilderDebRelease + # - BuilderDebAarch64 + # runs-on: [self-hosted, style-checker] + # steps: + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # fetch-depth: 0 # It MUST BE THE SAME for all dependencies and the job itself + # - name: Check docker clickhouse/clickhouse-server building + # run: | + # cd "$GITHUB_WORKSPACE/tests/ci" + # python3 docker_server.py --release-type head --no-push \ + # --image-repo clickhouse/clickhouse-server --image-path docker/server + # python3 docker_server.py --release-type head --no-push \ + # --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" +# ############################################################################################ +# ##################################### BUILD REPORTER ####################################### +# ############################################################################################ +# BuilderReport: +# needs: +# - BuilderBinRelease +# - BuilderDebAarch64 +# - BuilderDebAsan +# - BuilderDebDebug +# - BuilderDebMsan +# - BuilderDebRelease +# - BuilderDebTsan +# - BuilderDebUBsan +# runs-on: [self-hosted, style-checker] +# if: ${{ success() || failure() }} +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# CHECK_NAME=ClickHouse build check +# REPORTS_PATH=${{runner.temp}}/reports_dir +# TEMP_PATH=${{runner.temp}}/report_check +# NEEDS_DATA_PATH=${{runner.temp}}/needs.json +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Report Builder +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cat > "$NEEDS_DATA_PATH" << 'EOF' +# ${{ toJSON(needs) }} +# EOF +# cd "$GITHUB_WORKSPACE/tests/ci" +# python3 build_report_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# BuilderSpecialReport: +# needs: +# - BuilderBinAarch64 +# - BuilderBinDarwin +# - BuilderBinDarwinAarch64 +# - BuilderBinFreeBSD +# - BuilderBinPPC64 +# - BuilderBinAmd64Compat +# - BuilderBinAarch64V80Compat +# - BuilderBinClangTidy +# runs-on: [self-hosted, style-checker] +# if: ${{ success() || failure() }} +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/report_check +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=ClickHouse special build check +# NEEDS_DATA_PATH=${{runner.temp}}/needs.json +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Report Builder +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cat > "$NEEDS_DATA_PATH" << 'EOF' +# ${{ toJSON(needs) }} +# EOF +# cd "$GITHUB_WORKSPACE/tests/ci" +# python3 build_report_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################ +# #################################### INSTALL PACKAGES ###################################### +# ############################################################################################ +# InstallPackagesTestRelease: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, style-checker] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/test_install +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Install packages (amd64) +# REPO_COPY=${{runner.temp}}/test_install/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Test packages installation +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 install_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# InstallPackagesTestAarch64: +# needs: [BuilderDebAarch64] +# runs-on: [self-hosted, style-checker-aarch64] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/test_install +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Install packages (arm64) +# REPO_COPY=${{runner.temp}}/test_install/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Test packages installation +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 install_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################## +# ########################### FUNCTIONAl STATELESS TESTS ####################################### +# ############################################################################################## + # FunctionalStatelessTestRelease: + # needs: [BuilderDebRelease] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/stateless_release + # REPORTS_PATH=${{runner.temp}}/reports_dir + # CHECK_NAME=Stateless tests (release) + # REPO_COPY=${{runner.temp}}/stateless_release/ClickHouse + # KILL_TIMEOUT=10800 + # EOF + # - name: Download json reports + # uses: actions/download-artifact@v3 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Functional test + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" + # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestReleaseDatabaseReplicated0: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_database_replicated +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (release, DatabaseReplicated) +# REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestReleaseDatabaseReplicated1: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_database_replicated +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (release, DatabaseReplicated) +# REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestReleaseDatabaseReplicated2: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_database_replicated +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (release, DatabaseReplicated) +# REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestReleaseDatabaseReplicated3: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_database_replicated +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (release, DatabaseReplicated) +# REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestReleaseWideParts: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_wide_parts +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (release, wide parts enabled) +# REPO_COPY=${{runner.temp}}/stateless_wide_parts/ClickHouse +# KILL_TIMEOUT=10800 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestReleaseAnalyzer: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_analyzer +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (release, analyzer) +# REPO_COPY=${{runner.temp}}/stateless_analyzer/ClickHouse +# KILL_TIMEOUT=10800 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestReleaseS3_0: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (release, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=2 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestReleaseS3_1: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (release, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=2 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Debug0: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Debug1: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Debug2: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Debug3: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Debug4: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=4 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Debug5: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=5 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Tsan0: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Tsan1: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Tsan2: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Tsan3: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestS3Tsan4: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_s3_storage_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan, s3 storage) +# REPO_COPY=${{runner.temp}}/stateless_s3_storage_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=4 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" + # FunctionalStatelessTestAarch64: + # needs: [BuilderDebAarch64] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/stateless_release + # REPORTS_PATH=${{runner.temp}}/reports_dir + # CHECK_NAME=Stateless tests (aarch64) + # REPO_COPY=${{runner.temp}}/stateless_release/ClickHouse + # KILL_TIMEOUT=10800 + # EOF + # - name: Download json reports + # uses: actions/download-artifact@v3 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Functional test + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" + # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" + # FunctionalStatelessTestAsan0: + # needs: [BuilderDebAsan] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/stateless_debug + # REPORTS_PATH=${{runner.temp}}/reports_dir + # CHECK_NAME=Stateless tests (asan) + # REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse + # KILL_TIMEOUT=10800 + # RUN_BY_HASH_NUM=0 + # RUN_BY_HASH_TOTAL=4 + # EOF + # - name: Download json reports + # uses: actions/download-artifact@v3 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Functional test + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" + # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" + # FunctionalStatelessTestAsan1: + # needs: [BuilderDebAsan] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/stateless_debug + # REPORTS_PATH=${{runner.temp}}/reports_dir + # CHECK_NAME=Stateless tests (asan) + # REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse + # KILL_TIMEOUT=10800 + # RUN_BY_HASH_NUM=1 + # RUN_BY_HASH_TOTAL=4 + # EOF + # - name: Download json reports + # uses: actions/download-artifact@v3 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Functional test + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" + # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestAsan2: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (asan) +# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestAsan3: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (asan) +# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" + # FunctionalStatelessTestTsan0: + # needs: [BuilderDebTsan] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/stateless_tsan + # REPORTS_PATH=${{runner.temp}}/reports_dir + # CHECK_NAME=Stateless tests (tsan) + # REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse + # KILL_TIMEOUT=10800 + # RUN_BY_HASH_NUM=0 + # RUN_BY_HASH_TOTAL=5 + # EOF + # - name: Download json reports + # uses: actions/download-artifact@v3 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Functional test + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" + # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" + # FunctionalStatelessTestTsan1: + # needs: [BuilderDebTsan] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/stateless_tsan + # REPORTS_PATH=${{runner.temp}}/reports_dir + # CHECK_NAME=Stateless tests (tsan) + # REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse + # KILL_TIMEOUT=10800 + # RUN_BY_HASH_NUM=1 + # RUN_BY_HASH_TOTAL=5 + # EOF + # - name: Download json reports + # uses: actions/download-artifact@v3 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Functional test + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" + # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" + # FunctionalStatelessTestTsan2: + # needs: [BuilderDebTsan] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/stateless_tsan + # REPORTS_PATH=${{runner.temp}}/reports_dir + # CHECK_NAME=Stateless tests (tsan) + # REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse + # KILL_TIMEOUT=10800 + # RUN_BY_HASH_NUM=2 + # RUN_BY_HASH_TOTAL=5 + # EOF + # - name: Download json reports + # uses: actions/download-artifact@v3 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Functional test + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" + # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestTsan3: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan) +# REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestTsan4: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (tsan) +# REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=4 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestUBsan0: +# needs: [BuilderDebUBsan] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_ubsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (ubsan) +# REPO_COPY=${{runner.temp}}/stateless_ubsan/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=2 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# # FunctionalStatelessTestUBsan1: +# # needs: [BuilderDebUBsan] +# # runs-on: [self-hosted, func-tester] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/stateless_ubsan +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # CHECK_NAME=Stateless tests (ubsan) +# # REPO_COPY=${{runner.temp}}/stateless_ubsan/ClickHouse +# # KILL_TIMEOUT=10800 +# # RUN_BY_HASH_NUM=1 +# # RUN_BY_HASH_TOTAL=2 +# # EOF +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Functional test +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" +# # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestMsan0: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_memory +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (msan) +# REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestMsan1: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_memory +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (msan) +# REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestMsan2: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_memory +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (msan) +# REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestMsan3: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_memory +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (msan) +# REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestMsan4: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_memory +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (msan) +# REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=4 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestMsan5: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_memory +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (msan) +# REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=5 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" + # FunctionalStatelessTestDebug0: + # needs: [BuilderDebDebug] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/stateless_debug + # REPORTS_PATH=${{runner.temp}}/reports_dir + # CHECK_NAME=Stateless tests (debug) + # REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse + # KILL_TIMEOUT=10800 + # RUN_BY_HASH_NUM=0 + # RUN_BY_HASH_TOTAL=5 + # EOF + # - name: Download json reports + # uses: actions/download-artifact@v3 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Functional test + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" + # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" + # FunctionalStatelessTestDebug1: + # needs: [BuilderDebDebug] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/stateless_debug + # REPORTS_PATH=${{runner.temp}}/reports_dir + # CHECK_NAME=Stateless tests (debug) + # REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse + # KILL_TIMEOUT=10800 + # RUN_BY_HASH_NUM=1 + # RUN_BY_HASH_TOTAL=5 + # EOF + # - name: Download json reports + # uses: actions/download-artifact@v3 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Functional test + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" + # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" + # FunctionalStatelessTestDebug2: + # needs: [BuilderDebDebug] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/stateless_debug + # REPORTS_PATH=${{runner.temp}}/reports_dir + # CHECK_NAME=Stateless tests (debug) + # REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse + # KILL_TIMEOUT=10800 + # RUN_BY_HASH_NUM=2 + # RUN_BY_HASH_TOTAL=5 + # EOF + # - name: Download json reports + # uses: actions/download-artifact@v3 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Functional test + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" + # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestDebug3: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug) +# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestDebug4: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests (debug) +# REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse +# KILL_TIMEOUT=10800 +# RUN_BY_HASH_NUM=4 +# RUN_BY_HASH_TOTAL=5 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatelessTestFlakyCheck: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateless_flaky_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateless tests flaky check (asan) +# REPO_COPY=${{runner.temp}}/stateless_flaky_asan/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# TestsBugfixCheck: +# needs: [CheckLabels, StyleCheck] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/tests_bugfix_check +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=tests bugfix validate check +# KILL_TIMEOUT=3600 +# REPO_COPY=${{runner.temp}}/tests_bugfix_check/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Bugfix test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" + +# TEMP_PATH="${TEMP_PATH}/integration" \ +# REPORTS_PATH="${REPORTS_PATH}/integration" \ +# python3 integration_test_check.py "Integration $CHECK_NAME" \ +# --validate-bugfix --post-commit-status=file || echo 'ignore exit code' + +# TEMP_PATH="${TEMP_PATH}/stateless" \ +# REPORTS_PATH="${REPORTS_PATH}/stateless" \ +# python3 functional_test_check.py "Stateless $CHECK_NAME" "$KILL_TIMEOUT" \ +# --validate-bugfix --post-commit-status=file || echo 'ignore exit code' + +# python3 bugfix_validate_check.py "${TEMP_PATH}/stateless/functional_commit_status.tsv" "${TEMP_PATH}/integration/integration_commit_status.tsv" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################## +# ############################ FUNCTIONAl STATEFUL TESTS ####################################### +# ############################################################################################## + # FunctionalStatefulTestRelease: + # needs: [BuilderDebRelease] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/stateful_release + # REPORTS_PATH=${{runner.temp}}/reports_dir + # CHECK_NAME=Stateful tests (release) + # REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse + # KILL_TIMEOUT=3600 + # EOF + # - name: Download json reports + # uses: actions/download-artifact@v3 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Functional test + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" + # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" + # FunctionalStatefulTestAarch64: + # needs: [BuilderDebAarch64] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/stateful_release + # REPORTS_PATH=${{runner.temp}}/reports_dir + # CHECK_NAME=Stateful tests (aarch64) + # REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse + # KILL_TIMEOUT=3600 + # EOF + # - name: Download json reports + # uses: actions/download-artifact@v3 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Functional test + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" + # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" + # FunctionalStatefulTestAsan: + # needs: [BuilderDebAsan] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/stateful_debug + # REPORTS_PATH=${{runner.temp}}/reports_dir + # CHECK_NAME=Stateful tests (asan) + # REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse + # KILL_TIMEOUT=3600 + # EOF + # - name: Download json reports + # uses: actions/download-artifact@v3 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Functional test + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" + # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" + # FunctionalStatefulTestTsan: + # needs: [BuilderDebTsan] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/stateful_tsan + # REPORTS_PATH=${{runner.temp}}/reports_dir + # CHECK_NAME=Stateful tests (tsan) + # REPO_COPY=${{runner.temp}}/stateful_tsan/ClickHouse + # KILL_TIMEOUT=3600 + # EOF + # - name: Download json reports + # uses: actions/download-artifact@v3 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Functional test + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" + # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" + # FunctionalStatefulTestMsan: + # needs: [BuilderDebMsan] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/stateful_msan + # REPORTS_PATH=${{runner.temp}}/reports_dir + # CHECK_NAME=Stateful tests (msan) + # REPO_COPY=${{runner.temp}}/stateful_msan/ClickHouse + # KILL_TIMEOUT=3600 + # EOF + # - name: Download json reports + # uses: actions/download-artifact@v3 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Functional test + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" + # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" + # FunctionalStatefulTestUBsan: + # needs: [BuilderDebUBsan] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/stateful_ubsan + # REPORTS_PATH=${{runner.temp}}/reports_dir + # CHECK_NAME=Stateful tests (ubsan) + # REPO_COPY=${{runner.temp}}/stateful_ubsan/ClickHouse + # KILL_TIMEOUT=3600 + # EOF + # - name: Download json reports + # uses: actions/download-artifact@v3 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Functional test + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" + # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" + # FunctionalStatefulTestDebug: + # needs: [BuilderDebDebug] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/stateful_debug + # REPORTS_PATH=${{runner.temp}}/reports_dir + # CHECK_NAME=Stateful tests (debug) + # REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse + # KILL_TIMEOUT=3600 + # EOF + # - name: Download json reports + # uses: actions/download-artifact@v3 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Functional test + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" + # python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" +# # Parallel replicas +# FunctionalStatefulTestDebugParallelReplicas: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (debug, ParallelReplicas) +# REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v2 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Clear repository +# run: | +# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# - name: Check out repository code +# uses: actions/checkout@v2 +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatefulTestUBsanParallelReplicas: +# needs: [BuilderDebUBsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_ubsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (ubsan, ParallelReplicas) +# REPO_COPY=${{runner.temp}}/stateful_ubsan/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v2 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Clear repository +# run: | +# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# - name: Check out repository code +# uses: actions/checkout@v2 +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatefulTestMsanParallelReplicas: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_msan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (msan, ParallelReplicas) +# REPO_COPY=${{runner.temp}}/stateful_msan/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v2 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Clear repository +# run: | +# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# - name: Check out repository code +# uses: actions/checkout@v2 +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatefulTestTsanParallelReplicas: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (tsan, ParallelReplicas) +# REPO_COPY=${{runner.temp}}/stateful_tsan/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v2 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Clear repository +# run: | +# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# - name: Check out repository code +# uses: actions/checkout@v2 +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatefulTestAsanParallelReplicas: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (asan, ParallelReplicas) +# REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v2 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Clear repository +# run: | +# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# - name: Check out repository code +# uses: actions/checkout@v2 +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# FunctionalStatefulTestReleaseParallelReplicas: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/stateful_release +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Stateful tests (release, ParallelReplicas) +# REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse +# KILL_TIMEOUT=3600 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v2 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Clear repository +# run: | +# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# - name: Check out repository code +# uses: actions/checkout@v2 +# - name: Functional test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################## +# ######################################### STRESS TESTS ####################################### +# ############################################################################################## + StressTestAsan: + needs: [BuilderDebAsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stress_asan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stress test (asan) + REPO_COPY=${{runner.temp}}/stress_asan/ClickHouse + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Stress test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 stress_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + StressTestTsan: + needs: [BuilderDebTsan] + # func testers have 16 cores + 128 GB memory + # while stress testers have 36 cores + 72 memory + # It would be better to have something like 32 + 128, + # but such servers almost unavailable as spot instances. + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stress_thread + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stress test (tsan) + REPO_COPY=${{runner.temp}}/stress_thread/ClickHouse + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Stress test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 stress_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + StressTestMsan: + needs: [BuilderDebMsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stress_memory + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stress test (msan) + REPO_COPY=${{runner.temp}}/stress_memory/ClickHouse + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Stress test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 stress_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + StressTestUBsan: + needs: [BuilderDebUBsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stress_undefined + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stress test (ubsan) + REPO_COPY=${{runner.temp}}/stress_undefined/ClickHouse + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Stress test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 stress_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + StressTestDebug: + needs: [BuilderDebDebug] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stress_debug + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stress test (debug) + REPO_COPY=${{runner.temp}}/stress_debug/ClickHouse + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Stress test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 stress_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" +# ############################################################################################## +# ######################################### UPGRADE CHECK ###################################### +# ############################################################################################## +# UpgradeCheckAsan: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/upgrade_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Upgrade check (asan) +# REPO_COPY=${{runner.temp}}/upgrade_asan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Upgrade check +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 upgrade_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# UpgradeCheckTsan: +# needs: [BuilderDebTsan] +# # same as for stress test with tsan +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/upgrade_thread +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Upgrade check (tsan) +# REPO_COPY=${{runner.temp}}/upgrade_thread/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Upgrade check +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 upgrade_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# UpgradeCheckMsan: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/upgrade_memory +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Upgrade check (msan) +# REPO_COPY=${{runner.temp}}/upgrade_memory/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Upgrade check +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 upgrade_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# UpgradeCheckDebug: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/upgrade_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Upgrade check (debug) +# REPO_COPY=${{runner.temp}}/upgrade_debug/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Upgrade check +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 upgrade_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################## +# ##################################### AST FUZZERS ############################################ +# ############################################################################################## +# ASTFuzzerTestAsan: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/ast_fuzzer_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=AST fuzzer (asan) +# REPO_COPY=${{runner.temp}}/ast_fuzzer_asan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Fuzzer +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 ast_fuzzer_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ASTFuzzerTestTsan: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/ast_fuzzer_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=AST fuzzer (tsan) +# REPO_COPY=${{runner.temp}}/ast_fuzzer_tsan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Fuzzer +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 ast_fuzzer_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ASTFuzzerTestUBSan: +# needs: [BuilderDebUBsan] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/ast_fuzzer_ubsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=AST fuzzer (ubsan) +# REPO_COPY=${{runner.temp}}/ast_fuzzer_ubsan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Fuzzer +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 ast_fuzzer_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ASTFuzzerTestMSan: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/ast_fuzzer_msan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=AST fuzzer (msan) +# REPO_COPY=${{runner.temp}}/ast_fuzzer_msan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Fuzzer +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 ast_fuzzer_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ASTFuzzerTestDebug: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/ast_fuzzer_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=AST fuzzer (debug) +# REPO_COPY=${{runner.temp}}/ast_fuzzer_debug/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Fuzzer +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 ast_fuzzer_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################# +# ############################# INTEGRATION TESTS ############################################# +# ############################################################################################# + # IntegrationTestsAsan0: + # needs: [BuilderDebAsan] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/integration_tests_asan + # REPORTS_PATH=${{runner.temp}}/reports_dir + # CHECK_NAME=Integration tests (asan) + # REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse + # RUN_BY_HASH_NUM=0 + # RUN_BY_HASH_TOTAL=6 + # EOF + # - name: Download json reports + # uses: actions/download-artifact@v3 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Integration test + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" + # python3 integration_test_check.py "$CHECK_NAME" + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" + # IntegrationTestsAsan1: + # needs: [BuilderDebAsan] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/integration_tests_asan + # REPORTS_PATH=${{runner.temp}}/reports_dir + # CHECK_NAME=Integration tests (asan) + # REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse + # RUN_BY_HASH_NUM=1 + # RUN_BY_HASH_TOTAL=6 + # EOF + # - name: Download json reports + # uses: actions/download-artifact@v3 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Integration test + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" + # python3 integration_test_check.py "$CHECK_NAME" + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" + # IntegrationTestsAsan2: + # needs: [BuilderDebAsan] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/integration_tests_asan + # REPORTS_PATH=${{runner.temp}}/reports_dir + # CHECK_NAME=Integration tests (asan) + # REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse + # RUN_BY_HASH_NUM=2 + # RUN_BY_HASH_TOTAL=6 + # EOF + # - name: Download json reports + # uses: actions/download-artifact@v3 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Integration test + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" + # python3 integration_test_check.py "$CHECK_NAME" + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" +# IntegrationTestsAsan3: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (asan) +# REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsAsan4: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (asan) +# REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse +# RUN_BY_HASH_NUM=4 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsAsan5: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (asan) +# REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse +# RUN_BY_HASH_NUM=5 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" + IntegrationTestsTsan0: + needs: [BuilderDebTsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/integration_tests_tsan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Integration tests (tsan) + REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse + RUN_BY_HASH_NUM=0 + RUN_BY_HASH_TOTAL=6 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Integration test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 integration_test_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + IntegrationTestsTsan1: + needs: [BuilderDebTsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/integration_tests_tsan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Integration tests (tsan) + REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse + RUN_BY_HASH_NUM=1 + RUN_BY_HASH_TOTAL=6 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Integration test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 integration_test_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + IntegrationTestsTsan2: + needs: [BuilderDebTsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/integration_tests_tsan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Integration tests (tsan) + REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse + RUN_BY_HASH_NUM=2 + RUN_BY_HASH_TOTAL=6 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Integration test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 integration_test_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + IntegrationTestsTsan3: + needs: [BuilderDebTsan] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/integration_tests_tsan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Integration tests (tsan) + REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse + RUN_BY_HASH_NUM=3 + RUN_BY_HASH_TOTAL=6 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Integration test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 integration_test_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" +# IntegrationTestsTsan4: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (tsan) +# REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse +# RUN_BY_HASH_NUM=4 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsTsan5: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (tsan) +# REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse +# RUN_BY_HASH_NUM=5 +# RUN_BY_HASH_TOTAL=6 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" + IntegrationTestsRelease0: + needs: [BuilderDebRelease] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/integration_tests_release + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Integration tests (release) + REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse + RUN_BY_HASH_NUM=0 + RUN_BY_HASH_TOTAL=4 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Integration test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 integration_test_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + IntegrationTestsRelease1: + needs: [BuilderDebRelease] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/integration_tests_release + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Integration tests (release) + REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse + RUN_BY_HASH_NUM=1 + RUN_BY_HASH_TOTAL=4 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Integration test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 integration_test_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" +# IntegrationTestsRelease2: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_release +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (release) +# REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsRelease3: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_release +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests (release) +# REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# IntegrationTestsFlakyCheck: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/integration_tests_asan_flaky_check +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Integration tests flaky check (asan) +# REPO_COPY=${{runner.temp}}/integration_tests_asan_flaky_check/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Integration test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 integration_test_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################# +# #################################### UNIT TESTS ############################################# +# ############################################################################################# +# UnitTestsAsan: +# needs: [BuilderDebAsan] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/unit_tests_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Unit tests (asan) +# REPO_COPY=${{runner.temp}}/unit_tests_asan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Unit test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 unit_tests_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# UnitTestsReleaseClang: +# needs: [BuilderBinRelease] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/unit_tests_asan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Unit tests (release-clang) +# REPO_COPY=${{runner.temp}}/unit_tests_asan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Unit test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 unit_tests_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# UnitTestsTsan: +# needs: [BuilderDebTsan] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/unit_tests_tsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Unit tests (tsan) +# REPO_COPY=${{runner.temp}}/unit_tests_tsan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Unit test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 unit_tests_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# UnitTestsMsan: +# needs: [BuilderDebMsan] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/unit_tests_msan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Unit tests (msan) +# REPO_COPY=${{runner.temp}}/unit_tests_msan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Unit test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 unit_tests_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# UnitTestsUBsan: +# needs: [BuilderDebUBsan] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/unit_tests_ubsan +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Unit tests (ubsan) +# REPO_COPY=${{runner.temp}}/unit_tests_ubsan/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Unit test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 unit_tests_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################# +# #################################### PERFORMANCE TESTS ###################################### +# ############################################################################################# +# PerformanceComparisonX86-0: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/performance_comparison +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Performance Comparison +# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Performance Comparison +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 performance_comparison_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# PerformanceComparisonX86-1: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/performance_comparison +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Performance Comparison +# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Performance Comparison +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 performance_comparison_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# PerformanceComparisonX86-2: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/performance_comparison +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Performance Comparison +# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Performance Comparison +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 performance_comparison_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# PerformanceComparisonX86-3: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, stress-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/performance_comparison +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Performance Comparison +# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Performance Comparison +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 performance_comparison_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# PerformanceComparisonAarch-0: +# needs: [BuilderDebAarch64] +# runs-on: [self-hosted, func-tester-aarch64] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/performance_comparison +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Performance Comparison Aarch64 +# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# RUN_BY_HASH_NUM=0 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Performance Comparison +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 performance_comparison_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# PerformanceComparisonAarch-1: +# needs: [BuilderDebAarch64] +# runs-on: [self-hosted, func-tester-aarch64] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/performance_comparison +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Performance Comparison Aarch64 +# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# RUN_BY_HASH_NUM=1 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Performance Comparison +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 performance_comparison_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# PerformanceComparisonAarch-2: +# needs: [BuilderDebAarch64] +# runs-on: [self-hosted, func-tester-aarch64] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/performance_comparison +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Performance Comparison Aarch64 +# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# RUN_BY_HASH_NUM=2 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Performance Comparison +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 performance_comparison_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# PerformanceComparisonAarch-3: +# needs: [BuilderDebAarch64] +# runs-on: [self-hosted, func-tester-aarch64] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/performance_comparison +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Performance Comparison Aarch64 +# REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse +# RUN_BY_HASH_NUM=3 +# RUN_BY_HASH_TOTAL=4 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Performance Comparison +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 performance_comparison_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################## +# ###################################### SQLANCER FUZZERS ###################################### +# ############################################################################################## +# SQLancerTestRelease: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/sqlancer_release +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=SQLancer (release) +# REPO_COPY=${{runner.temp}}/sqlancer_release/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: SQLancer +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 sqlancer_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# SQLancerTestDebug: +# needs: [BuilderDebDebug] +# runs-on: [self-hosted, fuzzer-unit-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/sqlancer_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=SQLancer (debug) +# REPO_COPY=${{runner.temp}}/sqlancer_debug/ClickHouse +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v3 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: SQLancer +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 sqlancer_check.py "$CHECK_NAME" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" +# ############################################################################################# +# ###################################### JEPSEN TESTS ######################################### +# ############################################################################################# +# Jepsen: +# # This is special test NOT INCLUDED in FinishCheck +# # When it's skipped, all dependent tasks will be skipped too. +# # DO NOT add it there +# if: contains(github.event.pull_request.labels.*.name, 'jepsen-test') +# needs: [BuilderBinRelease] +# uses: ./.github/workflows/jepsen.yml +# FinishCheck: +# needs: +# - StyleCheck +# - DockerHubPush +# - DockerServerImages +# - CheckLabels +# - BuilderReport +# - BuilderSpecialReport +# - FastTest +# - FunctionalStatelessTestDebug0 +# - FunctionalStatelessTestDebug1 +# - FunctionalStatelessTestDebug2 +# - FunctionalStatelessTestDebug3 +# - FunctionalStatelessTestDebug4 +# - FunctionalStatelessTestRelease +# - FunctionalStatelessTestReleaseDatabaseReplicated0 +# - FunctionalStatelessTestReleaseDatabaseReplicated1 +# - FunctionalStatelessTestReleaseDatabaseReplicated2 +# - FunctionalStatelessTestReleaseDatabaseReplicated3 +# - FunctionalStatelessTestReleaseWideParts +# - FunctionalStatelessTestReleaseAnalyzer +# - FunctionalStatelessTestAarch64 +# - FunctionalStatelessTestAsan0 +# - FunctionalStatelessTestAsan1 +# - FunctionalStatelessTestAsan2 +# - FunctionalStatelessTestAsan3 +# - FunctionalStatelessTestTsan0 +# - FunctionalStatelessTestTsan1 +# - FunctionalStatelessTestTsan2 +# - FunctionalStatelessTestTsan3 +# - FunctionalStatelessTestTsan4 +# - FunctionalStatelessTestMsan0 +# - FunctionalStatelessTestMsan1 +# - FunctionalStatelessTestMsan2 +# - FunctionalStatelessTestMsan3 +# - FunctionalStatelessTestMsan4 +# - FunctionalStatelessTestMsan5 +# - FunctionalStatelessTestUBsan0 +# - FunctionalStatelessTestUBsan1 +# - FunctionalStatefulTestDebug +# - FunctionalStatefulTestRelease +# - FunctionalStatefulTestAarch64 +# - FunctionalStatefulTestAsan +# - FunctionalStatefulTestTsan +# - FunctionalStatefulTestMsan +# - FunctionalStatefulTestUBsan +# - FunctionalStatelessTestReleaseS3_0 +# - FunctionalStatelessTestReleaseS3_1 +# - FunctionalStatelessTestS3Debug0 +# - FunctionalStatelessTestS3Debug1 +# - FunctionalStatelessTestS3Debug2 +# - FunctionalStatelessTestS3Debug4 +# - FunctionalStatelessTestS3Debug5 +# - FunctionalStatelessTestS3Tsan0 +# - FunctionalStatelessTestS3Tsan1 +# - FunctionalStatelessTestS3Tsan2 +# - FunctionalStatelessTestS3Tsan4 +# - StressTestDebug +# - StressTestAsan +# - StressTestTsan +# - StressTestMsan +# - StressTestUBsan +# - ASTFuzzerTestDebug +# - ASTFuzzerTestAsan +# - ASTFuzzerTestTsan +# - ASTFuzzerTestMSan +# - ASTFuzzerTestUBSan +# - IntegrationTestsAsan0 +# - IntegrationTestsAsan1 +# - IntegrationTestsAsan2 +# - IntegrationTestsAsan3 +# - IntegrationTestsAsan4 +# - IntegrationTestsAsan5 +# - IntegrationTestsRelease0 +# - IntegrationTestsRelease1 +# - IntegrationTestsRelease2 +# - IntegrationTestsRelease3 +# - IntegrationTestsTsan0 +# - IntegrationTestsTsan1 +# - IntegrationTestsTsan2 +# - IntegrationTestsTsan3 +# - IntegrationTestsTsan4 +# - IntegrationTestsTsan5 +# - PerformanceComparisonX86-0 +# - PerformanceComparisonX86-1 +# - PerformanceComparisonX86-2 +# - PerformanceComparisonX86-3 +# - PerformanceComparisonAarch-0 +# - PerformanceComparisonAarch-1 +# - PerformanceComparisonAarch-2 +# - PerformanceComparisonAarch-3 +# - UnitTestsAsan +# - UnitTestsTsan +# - UnitTestsMsan +# - UnitTestsUBsan +# - UnitTestsReleaseClang +# - CompatibilityCheckX86 +# - CompatibilityCheckAarch64 +# - IntegrationTestsFlakyCheck +# - SQLancerTestRelease +# - SQLancerTestDebug +# runs-on: [self-hosted, style-checker] +# steps: +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Finish label +# run: | +# cd "$GITHUB_WORKSPACE/tests/ci" +# python3 finish_check.py +# python3 merge_pr.py --check-approved +# ############################################################################################## +# ########################### SQLLOGIC TEST ################################################### +# ############################################################################################## +# SQLLogicTestRelease: +# needs: [BuilderDebRelease] +# runs-on: [self-hosted, func-tester] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/sqllogic_debug +# REPORTS_PATH=${{runner.temp}}/reports_dir +# CHECK_NAME=Sqllogic test (release) +# REPO_COPY=${{runner.temp}}/sqllogic_debug/ClickHouse +# KILL_TIMEOUT=10800 +# EOF +# - name: Download json reports +# uses: actions/download-artifact@v2 +# with: +# path: ${{ env.REPORTS_PATH }} +# - name: Clear repository +# run: | +# sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" +# - name: Check out repository code +# uses: actions/checkout@v2 +# - name: Sqllogic test +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" +# python3 sqllogic_test.py "$CHECK_NAME" "$KILL_TIMEOUT" +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" \ No newline at end of file diff --git a/.github/workflows/pull_tests.yml b/.github/workflows/pull_tests.yml new file mode 100644 index 000000000000..5e697371b2b1 --- /dev/null +++ b/.github/workflows/pull_tests.yml @@ -0,0 +1,1255 @@ +# name: PullclangCI + +# env: +# # Force the stdout and stderr streams to be unbuffered +# PYTHONUNBUFFERED: 1 + +# on: # yamllint disable-line rule:truthy +# push: +# # pull_request: +# # types: +# # - synchronize +# # - reopened +# # - opened +# # branches: +# # - master +# # paths-ignore: +# # - 'CHANGELOG.md' +# # - 'README.md' +# # - 'SECURITY.md' +# # - 'docker/docs/**' +# # - 'docs/**' +# # - 'utils/check-style/aspell-ignore/**' +# jobs: +# DockerHubPushAarch64: +# runs-on: [self-hosted, ibm-vm] +# steps: +# - name: test +# run: env +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Images check +# run: | +# cd "$GITHUB_WORKSPACE/tests/ci" +# python3 docker_images_check.py --suffix aarch64 +# - name: Upload images files to artifacts +# uses: actions/upload-artifact@v3 +# with: +# name: changed_images_aarch64 +# path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json +# DockerHubPushAmd64: +# runs-on: [self-hosted, ibm-vm] +# steps: +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Images check +# run: | +# cd "$GITHUB_WORKSPACE/tests/ci" +# python3 docker_images_check.py --suffix amd64 +# - name: Upload images files to artifacts +# uses: actions/upload-artifact@v3 +# with: +# name: changed_images_amd64 +# path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json +# DockerHubPush: +# needs: [DockerHubPushAmd64, DockerHubPushAarch64] +# runs-on: [self-hosted, ibm-vm] +# steps: +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Download changed aarch64 images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images_aarch64 +# path: ${{ runner.temp }} +# - name: Download changed amd64 images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images_amd64 +# path: ${{ runner.temp }} +# - name: Images check +# run: | +# cd "$GITHUB_WORKSPACE/tests/ci" +# python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 +# - name: Upload images files to artifacts +# uses: actions/upload-artifact@v3 +# with: +# name: changed_images +# path: ${{ runner.temp }}/changed_images.json +# BuilderBinClangTidy: +# needs: [DockerHubPush] +# runs-on: [self-hosted, ibm-vm] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_tidy +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + # BuilderBinDarwin: + # needs: [DockerHubPush] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/build_check + # IMAGES_PATH=${{runner.temp}}/images_path + # REPO_COPY=${{runner.temp}}/build_check/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # BUILD_NAME=binary_darwin + # EOF + # - name: Download changed images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images + # path: ${{ env.IMAGES_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # submodules: true + # - name: Build + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + # - name: Upload build URLs to artifacts + # if: ${{ success() || failure() }} + # uses: actions/upload-artifact@v3 + # with: + # name: ${{ env.BUILD_URLS }} + # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + # BuilderBinDarwinAarch64: + # needs: [DockerHubPush] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/build_check + # IMAGES_PATH=${{runner.temp}}/images_path + # REPO_COPY=${{runner.temp}}/build_check/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # BUILD_NAME=binary_darwin_aarch64 + # EOF + # - name: Download changed images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images + # path: ${{ env.IMAGES_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # submodules: true + # - name: Build + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + # - name: Upload build URLs to artifacts + # if: ${{ success() || failure() }} + # uses: actions/upload-artifact@v3 + # with: + # name: ${{ env.BUILD_URLS }} + # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + # BuilderDebRelease: + # needs: [DockerHubPush] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/build_check + # IMAGES_PATH=${{runner.temp}}/images_path + # REPO_COPY=${{runner.temp}}/build_check/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # BUILD_NAME=package_release + # EOF + # - name: Download changed images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images + # path: ${{ env.IMAGES_PATH }} + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # fetch-depth: 0 # for performance artifact + # submodules: true + # - name: Build + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + # - name: Upload build URLs to artifacts + # if: ${{ success() || failure() }} + # uses: actions/upload-artifact@v3 + # with: + # name: ${{ env.BUILD_URLS }} + # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" + # BuilderDebAarch64: + # needs: [DockerHubPush] + # runs-on: [self-hosted, test-only] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/build_check + # IMAGES_PATH=${{runner.temp}}/images_path + # REPO_COPY=${{runner.temp}}/build_check/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # BUILD_NAME=package_aarch64 + # EOF + # - name: Download changed images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images + # path: ${{ runner.temp }}/images_path + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # submodules: true + # fetch-depth: 0 # for performance artifact + # - name: Build + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + # - name: Upload build URLs to artifacts + # if: ${{ success() || failure() }} + # uses: actions/upload-artifact@v3 + # with: + # name: ${{ env.BUILD_URLS }} + # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + # MarkReleaseReady: + # needs: + # - BuilderBinDarwin + # - BuilderBinDarwinAarch64 + # - BuilderDebRelease + # - BuilderDebAarch64 + # runs-on: [self-hosted, builder] + # steps: + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Mark Commit Release Ready + # run: | + # cd "$GITHUB_WORKSPACE/tests/ci" + # python3 mark_release_ready.py + + +name: docker-check + +env: + # Force the stdout and stderr streams to be unbuffered + PYTHONUNBUFFERED: 1 + +on: # yamllint disable-line rule:truthy + push: + # pull_request: + # types: + # - synchronize + # - reopened + # - opened + # branches: + # - master + # paths-ignore: + # - 'CHANGELOG.md' + # - 'README.md' + # - 'SECURITY.md' + # - 'docker/docs/**' + # - 'docs/**' + # - 'utils/check-style/aspell-ignore/**' +########################################################################################## +##################################### SMALL CHECKS ####################################### +########################################################################################## +jobs: + # CheckLabels: + # runs-on: [self-hosted, style-checker] + # # Run the first check always, even if the CI is cancelled + # if: ${{ always() }} + # steps: + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Labels check + # run: | + # cd "$GITHUB_WORKSPACE/tests/ci" + # python3 run_check.py + # PythonUnitTests: + # runs-on: [self-hosted, style-checker] + # steps: + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # - name: Python unit tests + # run: | + # cd "$GITHUB_WORKSPACE/tests/ci" + # echo "Testing the main ci directory" + # python3 -m unittest discover -s . -p '*_test.py' + # for dir in *_lambda/; do + # echo "Testing $dir" + # python3 -m unittest discover -s "$dir" -p '*_test.py' + # done + DockerHubPushAarch64: + # needs: CheckLabels + runs-on: [self-hosted, style-checker] + steps: + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Images check + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 docker_images_check.py --suffix aarch64 + - name: Upload images files to artifacts + uses: actions/upload-artifact@v3 + with: + name: changed_images_aarch64 + path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json + DockerHubPushAmd64: + # needs: CheckLabels + runs-on: [self-hosted, style-checker] + steps: + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Images check + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 docker_images_check.py --suffix amd64 + - name: Upload images files to artifacts + uses: actions/upload-artifact@v3 + with: + name: changed_images_amd64 + path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json + DockerHubPush: + needs: [DockerHubPushAmd64, DockerHubPushAarch64] + runs-on: [self-hosted, style-checker] + steps: + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Download changed aarch64 images + uses: actions/download-artifact@v3 + with: + name: changed_images_aarch64 + path: ${{ runner.temp }} + - name: Download changed amd64 images + uses: actions/download-artifact@v3 + with: + name: changed_images_amd64 + path: ${{ runner.temp }} + - name: Images check + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 + - name: Upload images files to artifacts + uses: actions/upload-artifact@v3 + with: + name: changed_images + path: ${{ runner.temp }}/changed_images.json + +# StyleCheck: +# needs: DockerHubPush +# runs-on: [self-hosted, style-checker] +# # We need additional `&& ! cancelled()` to have the job being able to cancel +# if: ${{ success() || failure() || ( always() && ! cancelled() ) }} +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{ runner.temp }}/style_check +# ROBOT_CLICKHOUSE_SSH_KEY<> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/fasttest +# REPO_COPY=${{runner.temp}}/fasttest/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# EOF +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.TEMP_PATH }} +# - name: Fast Test +# run: | +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 fast_test_check.py +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# ## CompatibilityCheckX86: +# # needs: [BuilderDebRelease] +# # runs-on: [self-hosted, style-checker] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/compatibility_check +# # REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # EOF +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: CompatibilityCheckX86 +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (amd64)" --check-glibc --check-distributions +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # CompatibilityCheckAarch64: +# # needs: [BuilderDebAarch64] +# # runs-on: [self-hosted, style-checker] +# # steps: +# # - name: Set envs +# # run: | +# # cat >> "$GITHUB_ENV" << 'EOF' +# # TEMP_PATH=${{runner.temp}}/compatibility_check +# # REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse +# # REPORTS_PATH=${{runner.temp}}/reports_dir +# # EOF +# # - name: Check out repository code +# # uses: ClickHouse/checkout@v1 +# # with: +# # clear-repository: true +# # - name: Download json reports +# # uses: actions/download-artifact@v3 +# # with: +# # path: ${{ env.REPORTS_PATH }} +# # - name: CompatibilityCheckAarch64 +# # run: | +# # sudo rm -fr "$TEMP_PATH" +# # mkdir -p "$TEMP_PATH" +# # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# # cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (aarch64)" --check-glibc +# # - name: Cleanup +# # if: always() +# # run: | +# # docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# # sudo rm -fr "$TEMP_PATH" +# # ######################################################################################### +# # #################################### ORDINARY BUILDS #################################### +# # ######################################################################################### + BuilderDebRelease: + # needs: [DockerHubPush, FastTest, StyleCheck] + needs: [DockerHubPush] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=package_release + EOF + - name: Download changed images + uses: actions/download-artifact@v3 + with: + name: changed_images + path: ${{ env.IMAGES_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + fetch-depth: 0 # for performance artifact + submodules: true + - name: Build + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" + +# BuilderBinRelease: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_release +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + # BuilderDebAarch64: + # # needs: [DockerHubPush, FastTest, StyleCheck] + # needs: [DockerHubPush] + # runs-on: [self-hosted, ibm-vm] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/build_check + # IMAGES_PATH=${{runner.temp}}/images_path + # REPO_COPY=${{runner.temp}}/build_check/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # BUILD_NAME=package_aarch64 + # EOF + # - name: Download changed images + # uses: actions/download-artifact@v3 + # with: + # name: changed_images + # path: ${{ runner.temp }}/images_path + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # submodules: true + # fetch-depth: 0 # for performance artifact + # - name: Build + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + # - name: Upload build URLs to artifacts + # if: ${{ success() || failure() }} + # uses: actions/upload-artifact@v3 + # with: + # name: ${{ env.BUILD_URLS }} + # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderDebAsan: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=package_asan +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderDebUBsan: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=package_ubsan +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderDebTsan: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=package_tsan +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderDebMsan: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=package_msan +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderDebDebug: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=package_debug +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Apply sparse checkout for contrib # in order to check that it doesn't break build +# run: | +# rm -rf "$GITHUB_WORKSPACE/contrib" && echo 'removed' +# git -C "$GITHUB_WORKSPACE" checkout . && echo 'restored' +# "$GITHUB_WORKSPACE/contrib/update-submodules.sh" && echo 'OK' +# du -hs "$GITHUB_WORKSPACE/contrib" ||: +# find "$GITHUB_WORKSPACE/contrib" -type f | wc -l ||: +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# # ########################################################################################## +# # ##################################### SPECIAL BUILDS ##################################### +# # ########################################################################################## +# BuilderBinClangTidy: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_tidy +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinDarwin: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_darwin +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinAarch64: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_aarch64 +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinFreeBSD: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_freebsd +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinDarwinAarch64: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_darwin_aarch64 +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinPPC64: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_ppc64le +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinAmd64Compat: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_amd64_compat +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# BuilderBinAarch64V80Compat: +# needs: [DockerHubPush, FastTest, StyleCheck] +# runs-on: [self-hosted, builder] +# steps: +# - name: Set envs +# run: | +# cat >> "$GITHUB_ENV" << 'EOF' +# TEMP_PATH=${{runner.temp}}/build_check +# IMAGES_PATH=${{runner.temp}}/images_path +# REPO_COPY=${{runner.temp}}/build_check/ClickHouse +# CACHES_PATH=${{runner.temp}}/../ccaches +# BUILD_NAME=binary_aarch64_v80compat +# EOF +# - name: Download changed images +# uses: actions/download-artifact@v3 +# with: +# name: changed_images +# path: ${{ env.IMAGES_PATH }} +# - name: Check out repository code +# uses: ClickHouse/checkout@v1 +# with: +# clear-repository: true +# submodules: true +# - name: Build +# run: | +# sudo rm -fr "$TEMP_PATH" +# mkdir -p "$TEMP_PATH" +# cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" +# cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" +# - name: Upload build URLs to artifacts +# if: ${{ success() || failure() }} +# uses: actions/upload-artifact@v3 +# with: +# name: ${{ env.BUILD_URLS }} +# path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json +# - name: Cleanup +# if: always() +# run: | +# docker ps --quiet | xargs --no-run-if-empty docker kill ||: +# docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: +# sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +# ############################################################################################ +# ##################################### Docker images ####################################### +# ############################################################################################ + # DockerServerImages: + # needs: + # - BuilderDebRelease + # - BuilderDebAarch64 + # runs-on: [self-hosted, ibm-vm] + # steps: + # - name: Check out repository code + # uses: ClickHouse/checkout@v1 + # with: + # clear-repository: true + # fetch-depth: 0 # It MUST BE THE SAME for all dependencies and the job itself + # - name: Check docker clickhouse/clickhouse-server building + # run: | + # cd "$GITHUB_WORKSPACE/tests/ci" + # DOCKER_REPO="${DOCKER_REPO:-docker.io}" + # python3 docker_server.py --release-type head --no-push \ + # --image-repo $DOCKER_REPO/clickhouse/clickhouse-server --image-path docker/server + # python3 docker_server.py --release-type head --no-push \ + # --image-repo $DOCKER_REPO/clickhouse/clickhouse-keeper --image-path docker/keeper + # - name: Cleanup + # if: always() + # run: | + # docker ps --quiet | xargs --no-run-if-empty docker kill ||: + # docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + # sudo rm -fr "$TEMP_PATH" diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index dd21c8552d3b..ae8309953905 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -1,6 +1,7 @@ # docker build -t clickhouse/binary-builder . ARG FROM_TAG=latest -FROM clickhouse/test-util:latest AS cctools +ARG DOCKER_REPO=docker.io +FROM $DOCKER_REPO/clickhouse/test-util:latest AS cctools # The cctools are built always from the clickhouse/test-util:latest and cached inline # Theoretically, it should improve rebuild speed significantly ENV CC=clang-${LLVM_VERSION} @@ -39,7 +40,7 @@ RUN git clone --depth 1 https://github.com/tpoechtrager/cctools-port.git \ # END COMPILE # !!!!!!!!!!! -FROM clickhouse/test-util:$FROM_TAG +FROM $DOCKER_REPO/clickhouse/test-util:$FROM_TAG ENV CC=clang-${LLVM_VERSION} ENV CXX=clang++-${LLVM_VERSION} diff --git a/docker/packager/packager b/docker/packager/packager index 1b3df858cd2e..5c464b70f7d9 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -10,7 +10,7 @@ from typing import List, Optional SCRIPT_PATH = Path(__file__).absolute() IMAGE_TYPE = "binary" -IMAGE_NAME = f"clickhouse/{IMAGE_TYPE}-builder" +# IMAGE_NAME = f"{args.docker_repo}/clickhouse/{IMAGE_TYPE}-builder" class BuildException(Exception): @@ -104,6 +104,7 @@ def run_docker_image_with_env( cmd = ( f"docker run --network=host --user={user} --rm {ccache_mount}" f"--volume={output_dir}:/output --volume={ch_root}:/build {env_part} " + f"--volume=/home/ubuntu/.aws/credentials:/.aws/credentials " # Mount .aws directory f"{interactive} {image_name}" ) @@ -123,6 +124,7 @@ def parse_env_variables( package_type: str, cache: str, s3_bucket: str, + s3_endpoint: str, s3_directory: str, s3_rw_access: bool, clang_tidy: bool, @@ -273,7 +275,8 @@ def parse_env_variables( if cache == "sccache": cmake_flags.append("-DCOMPILER_CACHE=sccache") # see https://github.com/mozilla/sccache/blob/main/docs/S3.md - result.append(f"SCCACHE_BUCKET={s3_bucket}") + result.append(f"SCCACHE_BUCKET={s3_bucket} ") + result.append(f"SCCACHE_ENDPOINT={s3_endpoint} ") sccache_dir = "sccache" if s3_directory: sccache_dir = f"{s3_directory}/{sccache_dir}" @@ -399,6 +402,19 @@ def parse_args() -> argparse.Namespace: "--s3-bucket", help="an S3 bucket used for sscache and clang-tidy-cache", ) + parser.add_argument( + "--s3-endpoint", + default="https://s3.amazonaws.com", + help="the custom endpoint URL for the S3-compatible object storage service", + ) + parser.add_argument( + "--aws-access-key", + help="an S3 directory prefix used for sscache and clang-tidy-cache", + ) + parser.add_argument( + "--aws-secret-access-key", + help="an S3 directory prefix used for sscache and clang-tidy-cache", + ) parser.add_argument( "--s3-directory", default="ccache", @@ -423,6 +439,11 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--as-root", action="store_true", help="if the container should run as root" ) + parser.add_argument("--docker-repo", default="docker.io", help="docker repository") + parser.add_argument("--docker-user", default=argparse.SUPPRESS, help="docker user") + parser.add_argument( + "--docker-password", default=argparse.SUPPRESS, help="docker password" + ) args = parser.parse_args() @@ -454,6 +475,8 @@ def main(): ch_root = args.clickhouse_repo_path + IMAGE_NAME = f"{args.docker_repo}/clickhouse/{IMAGE_TYPE}-builder" + dockerfile = ch_root / "docker/packager" / IMAGE_TYPE / "Dockerfile" image_with_version = IMAGE_NAME + ":" + args.docker_image_version if args.force_build_image: @@ -470,6 +493,7 @@ def main(): args.package_type, args.cache, args.s3_bucket, + args.s3_endpoint, args.s3_directory, args.s3_rw_access, args.clang_tidy, @@ -479,7 +503,15 @@ def main(): args.with_coverage, args.with_binaries, ) - + if args.docker_user: + print(f"docker_repo: {args.docker_repo}") + print(f"docker_user: {args.docker_user}") + subprocess.check_output( # pylint: disable=unexpected-keyword-arg + f"docker login {args.docker_repo} --username '{args.docker_user}' --password-stdin", + input=args.docker_password, + encoding="utf-8", + shell=True, + ) pre_build(args.clickhouse_repo_path, env_prepared) run_docker_image_with_env( image_with_version, diff --git a/docker/test/base/Dockerfile b/docker/test/base/Dockerfile index f68368044541..70159f033990 100644 --- a/docker/test/base/Dockerfile +++ b/docker/test/base/Dockerfile @@ -1,7 +1,9 @@ # rebuild in #33610 # docker build -t clickhouse/test-base . ARG FROM_TAG=latest -FROM clickhouse/test-util:$FROM_TAG +ARG DOCKER_REPO=docker.io +FROM $DOCKER_REPO/clickhouse/test-util:$FROM_TAG + RUN apt-get update \ && apt-get install \ diff --git a/docker/test/codebrowser/Dockerfile b/docker/test/codebrowser/Dockerfile index 8136fd1fbbcd..e6676c740fd3 100644 --- a/docker/test/codebrowser/Dockerfile +++ b/docker/test/codebrowser/Dockerfile @@ -2,7 +2,8 @@ # docker build --network=host -t clickhouse/codebrowser . # docker run --volume=path_to_repo:/repo_folder --volume=path_to_result:/test_output clickhouse/codebrowser ARG FROM_TAG=latest -FROM clickhouse/binary-builder:$FROM_TAG +ARG DOCKER_REPO=docker.io +FROM $DOCKER_REPO/clickhouse/binary-builder:$FROM_TAG # ARG for quick switch to a given ubuntu mirror ARG apt_archive="http://archive.ubuntu.com" diff --git a/docker/test/fasttest/Dockerfile b/docker/test/fasttest/Dockerfile index ffb13fc774d4..98d58fa9767c 100644 --- a/docker/test/fasttest/Dockerfile +++ b/docker/test/fasttest/Dockerfile @@ -1,7 +1,8 @@ # rebuild in #33610 # docker build -t clickhouse/fasttest . ARG FROM_TAG=latest -FROM clickhouse/test-util:$FROM_TAG +ARG DOCKER_REPO=docker.io +FROM $DOCKER_REPO/clickhouse/test-util:$FROM_TAG RUN apt-get update \ && apt-get install \ diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index dab873377ce5..31632e7636b8 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -11,6 +11,9 @@ stage=${stage:-} # Compiler version, normally set by Dockerfile export LLVM_VERSION=${LLVM_VERSION:-16} +# Access the GITHUB_REPOSITORY environment variable +github_repository=${GITHUB_REPOSITORY} + # A variable to pass additional flags to CMake. # Here we explicitly default it to nothing so that bash doesn't complain about # it being undefined. Also read it as array so that we can pass an empty list @@ -61,6 +64,9 @@ function start_server clickhouse-server "${opts[@]}" &>> "$FASTTEST_OUTPUT/server.log" & set +m + # # Add a log message to indicate the server startup process has begun + # echo "$(date '+%Y-%m-%d %H:%M:%S'): ClickHouse server startup initiated" >> "$FASTTEST_OUTPUT/server.log" + for _ in {1..60}; do if clickhouse-client --query "select 1"; then break @@ -76,12 +82,15 @@ function start_server local server_pid server_pid="$(cat "$FASTTEST_DATA/clickhouse-server.pid")" echo "ClickHouse server pid '$server_pid' started and responded" + + # Add a log message to indicate the server startup process has completed + # echo "$(date '+%Y-%m-%d %H:%M:%S'): ClickHouse server startup completed" >> "$FASTTEST_OUTPUT/server.log" } function clone_root { git config --global --add safe.directory "$FASTTEST_SOURCE" - git clone --depth 1 https://github.com/ClickHouse/ClickHouse.git -- "$FASTTEST_SOURCE" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/clone_log.txt" + git clone --depth 1 https://github.com/${github_repository}.git -- "$FASTTEST_SOURCE" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/clone_log.txt" ( cd "$FASTTEST_SOURCE" diff --git a/docker/test/fuzzer/Dockerfile b/docker/test/fuzzer/Dockerfile index aa71074c02a7..7401edc4eefe 100644 --- a/docker/test/fuzzer/Dockerfile +++ b/docker/test/fuzzer/Dockerfile @@ -1,7 +1,8 @@ # rebuild in #33610 # docker build -t clickhouse/fuzzer . ARG FROM_TAG=latest -FROM clickhouse/test-base:$FROM_TAG +ARG DOCKER_REPO=docker.io +FROM $DOCKER_REPO/clickhouse/test-base:$FROM_TAG # ARG for quick switch to a given ubuntu mirror ARG apt_archive="http://archive.ubuntu.com" diff --git a/docker/test/integration/base/Dockerfile b/docker/test/integration/base/Dockerfile index de8efa20af4e..6f442be44666 100644 --- a/docker/test/integration/base/Dockerfile +++ b/docker/test/integration/base/Dockerfile @@ -1,7 +1,8 @@ # rebuild in #33610 # docker build -t clickhouse/integration-test . ARG FROM_TAG=latest -FROM clickhouse/test-base:$FROM_TAG +ARG DOCKER_REPO=docker.io +FROM $DOCKER_REPO/clickhouse/test-base:$FROM_TAG SHELL ["/bin/bash", "-c"] diff --git a/docker/test/keeper-jepsen/Dockerfile b/docker/test/keeper-jepsen/Dockerfile index a794e076ec02..b0425847048d 100644 --- a/docker/test/keeper-jepsen/Dockerfile +++ b/docker/test/keeper-jepsen/Dockerfile @@ -1,7 +1,8 @@ # rebuild in #33610 # docker build -t clickhouse/keeper-jepsen-test . ARG FROM_TAG=latest -FROM clickhouse/test-base:$FROM_TAG +ARG DOCKER_REPO=docker.io +FROM $DOCKER_REPO/clickhouse/test-base:$FROM_TAG ENV DEBIAN_FRONTEND=noninteractive ENV CLOJURE_VERSION=1.10.3.814 diff --git a/docker/test/server-jepsen/Dockerfile b/docker/test/server-jepsen/Dockerfile index a212427b2a1a..45c28bc6b036 100644 --- a/docker/test/server-jepsen/Dockerfile +++ b/docker/test/server-jepsen/Dockerfile @@ -1,7 +1,8 @@ # rebuild in #33610 # docker build -t clickhouse/server-jepsen-test . ARG FROM_TAG=latest -FROM clickhouse/test-base:$FROM_TAG +ARG DOCKER_REPO=docker.io +FROM $DOCKER_REPO/clickhouse/test-base:$FROM_TAG ENV DEBIAN_FRONTEND=noninteractive ENV CLOJURE_VERSION=1.10.3.814 diff --git a/docker/test/sqllogic/Dockerfile b/docker/test/sqllogic/Dockerfile index 83dcf7e1f56a..f30e3e0b86ed 100644 --- a/docker/test/sqllogic/Dockerfile +++ b/docker/test/sqllogic/Dockerfile @@ -1,6 +1,7 @@ # docker build -t clickhouse/sqllogic-test . ARG FROM_TAG=latest -FROM clickhouse/test-base:$FROM_TAG +ARG DOCKER_REPO=docker.io +FROM $DOCKER_REPO/clickhouse/test-base:$FROM_TAG RUN apt-get update --yes \ && env DEBIAN_FRONTEND=noninteractive \ diff --git a/docker/test/stateful/Dockerfile b/docker/test/stateful/Dockerfile index 71a2e92e3a86..d69662997915 100644 --- a/docker/test/stateful/Dockerfile +++ b/docker/test/stateful/Dockerfile @@ -1,7 +1,8 @@ # rebuild in #47031 # docker build -t clickhouse/stateful-test . ARG FROM_TAG=latest -FROM clickhouse/stateless-test:$FROM_TAG +ARG DOCKER_REPO=docker.io +FROM $DOCKER_REPO/clickhouse/stateless-test:$FROM_TAG RUN apt-get update -y \ && env DEBIAN_FRONTEND=noninteractive \ diff --git a/docker/test/stateless/Dockerfile b/docker/test/stateless/Dockerfile index 40109255a7e5..42d7b1053a3a 100644 --- a/docker/test/stateless/Dockerfile +++ b/docker/test/stateless/Dockerfile @@ -1,7 +1,8 @@ # rebuild in #33610 # docker build -t clickhouse/stateless-test . ARG FROM_TAG=latest -FROM clickhouse/test-base:$FROM_TAG +ARG DOCKER_REPO=docker.io +FROM $DOCKER_REPO/clickhouse/test-base:$FROM_TAG ARG odbc_driver_url="https://github.com/ClickHouse/clickhouse-odbc/releases/download/v1.1.4.20200302/clickhouse-odbc-1.1.4-Linux.tar.gz" diff --git a/docker/test/stress/Dockerfile b/docker/test/stress/Dockerfile index e9712f430fd2..7e4e17da2a68 100644 --- a/docker/test/stress/Dockerfile +++ b/docker/test/stress/Dockerfile @@ -1,7 +1,8 @@ # rebuild in #33610 # docker build -t clickhouse/stress-test . ARG FROM_TAG=latest -FROM clickhouse/stateful-test:$FROM_TAG +ARG DOCKER_REPO=docker.io +FROM $DOCKER_REPO/clickhouse/stateful-test:$FROM_TAG RUN apt-get update -y \ && env DEBIAN_FRONTEND=noninteractive \ diff --git a/docker/test/unit/Dockerfile b/docker/test/unit/Dockerfile index b75bfb6661cc..d389e0d3f341 100644 --- a/docker/test/unit/Dockerfile +++ b/docker/test/unit/Dockerfile @@ -1,7 +1,8 @@ # rebuild in #33610 # docker build -t clickhouse/unit-test . ARG FROM_TAG=latest -FROM clickhouse/stateless-test:$FROM_TAG +ARG DOCKER_REPO=docker.io +FROM $DOCKER_REPO/clickhouse/stateless-test:$FROM_TAG RUN apt-get install gdb diff --git a/docker/test/upgrade/Dockerfile b/docker/test/upgrade/Dockerfile index 8e5890b81a0c..86055ff312a9 100644 --- a/docker/test/upgrade/Dockerfile +++ b/docker/test/upgrade/Dockerfile @@ -1,7 +1,8 @@ # rebuild in #33610 # docker build -t clickhouse/upgrade-check . ARG FROM_TAG=latest -FROM clickhouse/stateful-test:$FROM_TAG +ARG DOCKER_REPO=docker.io +FROM $DOCKER_REPO/clickhouse/stateless-test:$FROM_TAG RUN apt-get update -y \ && env DEBIAN_FRONTEND=noninteractive \ diff --git a/docker/test/util/Dockerfile b/docker/test/util/Dockerfile index a49278e960b3..b93add045110 100644 --- a/docker/test/util/Dockerfile +++ b/docker/test/util/Dockerfile @@ -8,6 +8,7 @@ RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list # 15.0.2 ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=16 + RUN apt-get update \ && apt-get install \ apt-transport-https \ diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index 4bc61c79fc03..c7d7ceb2a477 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -23,8 +23,11 @@ S3_BUILDS_BUCKET, S3_DOWNLOAD, TEMP_PATH, + DOCKER_USER, + DOCKER_REPO, + S3_URL, ) -from get_robot_token import get_best_robot_token +from get_robot_token import get_best_robot_token, get_parameter_from_ssm from github_helper import GitHub from pr_info import PRInfo from s3_helper import S3Helper @@ -36,7 +39,7 @@ update_version_local, ) -IMAGE_NAME = "clickhouse/binary-builder" +IMAGE_NAME = f"{DOCKER_REPO}/clickhouse/binary-builder" BUILD_LOG_NAME = "build_log.log" @@ -76,12 +79,16 @@ def get_packager_cmd( cmd += " --cache=sccache" cmd += " --s3-rw-access" cmd += f" --s3-bucket={S3_BUILDS_BUCKET}" + cmd += f" --s3-endpoint={S3_URL}" if "additional_pkgs" in build_config and build_config["additional_pkgs"]: cmd += " --additional-pkgs" cmd += f" --docker-image-version={image_version}" cmd += f" --version={build_version}" + cmd += f" --docker-repo={DOCKER_REPO}" + cmd += f" --docker-user={DOCKER_USER}" + cmd += f" --docker-password={get_parameter_from_ssm('dockerhub_robot_password')}" if _can_export_binaries(build_config): cmd += " --with-binaries=tests" @@ -294,6 +301,13 @@ def main(): # put them as github actions artifact (result) check_for_success_run(s3_helper, s3_path_prefix, build_name, build_config) + subprocess.check_output( # pylint: disable=unexpected-keyword-arg + f"docker login {DOCKER_REPO} --username '{DOCKER_USER}' --password-stdin", + input=get_parameter_from_ssm("dockerhub_robot_password"), + encoding="utf-8", + shell=True, + ) + # If it's a latter running, we need to mark possible failed status mark_failed_reports_pending(build_name, pr_info) diff --git a/tests/ci/clickhouse_helper.py b/tests/ci/clickhouse_helper.py index 64b64896f661..c30d0900cc16 100644 --- a/tests/ci/clickhouse_helper.py +++ b/tests/ci/clickhouse_helper.py @@ -7,6 +7,7 @@ import requests # type: ignore from get_robot_token import get_parameter_from_ssm +from env_helper import GITHUB_REPOSITORY from pr_info import PRInfo from report import TestResults @@ -141,7 +142,7 @@ def prepare_tests_results_for_clickhouse( report_url: str, check_name: str, ) -> List[dict]: - pull_request_url = "https://github.com/ClickHouse/ClickHouse/commits/master" + pull_request_url = f"https://github.com/{GITHUB_REPOSITORY}/commits/master" base_ref = "master" head_ref = "master" base_repo = pr_info.repo_full_name diff --git a/tests/ci/constants.py b/tests/ci/constants.py new file mode 100644 index 000000000000..7f48b77f85a7 --- /dev/null +++ b/tests/ci/constants.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python3 +import os + +VAULT_PATH = os.getenv("VAULT_PATH") +VAULT_TOKEN = os.getenv("VAULT_TOKEN") +VAULT_URL = os.getenv("VAULT_URL") +VAULT_MOUNT_POINT = os.getenv("VAULT_MOUNT_POINT", "secret") diff --git a/tests/ci/docker_images_check.py b/tests/ci/docker_images_check.py index 16a58a90dcf6..1c8d635d669c 100644 --- a/tests/ci/docker_images_check.py +++ b/tests/ci/docker_images_check.py @@ -15,7 +15,14 @@ from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse from commit_status_helper import format_description, get_commit, post_commit_status -from env_helper import GITHUB_WORKSPACE, RUNNER_TEMP, GITHUB_RUN_URL +from env_helper import ( + GITHUB_WORKSPACE, + RUNNER_TEMP, + GITHUB_RUN_URL, + DOCKER_USER, + DOCKER_REPO, + GITHUB_REPOSITORY, +) from get_robot_token import get_best_robot_token, get_parameter_from_ssm from pr_info import PRInfo from report import TestResults, TestResult @@ -94,7 +101,7 @@ def get_images_dict(repo_path: str, image_file_path: str) -> ImagesDict: def get_changed_docker_images( - pr_info: PRInfo, images_dict: ImagesDict + pr_info: PRInfo, images_dict: ImagesDict, DOCKER_REPO: str ) -> Set[DockerImage]: if not images_dict: return set() @@ -113,7 +120,7 @@ def get_changed_docker_images( for dockerfile_dir, image_description in images_dict.items(): for f in files_changed: if f.startswith(dockerfile_dir): - name = image_description["name"] + name = DOCKER_REPO + "/" + image_description["name"] only_amd64 = image_description.get("only_amd64", False) logging.info( "Found changed file '%s' which affects " @@ -137,7 +144,7 @@ def get_changed_docker_images( dependent, image, ) - name = images_dict[dependent]["name"] + name = DOCKER_REPO + "/" + images_dict[dependent]["name"] only_amd64 = images_dict[dependent].get("only_amd64", False) changed_images.append(DockerImage(dependent, name, only_amd64, image)) index += 1 @@ -248,6 +255,8 @@ def build_and_push_one_image( "docker buildx build --builder default " f"--label build-url={GITHUB_RUN_URL} " f"{from_tag_arg}" + f"--build-arg DOCKER_REPO={DOCKER_REPO} " + f"--build-arg GITHUB_REPOSITORY={GITHUB_REPOSITORY} " # A hack to invalidate cache, grep for it in docker/ dir f"--build-arg CACHE_INVALIDATOR={GITHUB_RUN_URL} " f"--tag {image.repo}:{version_string} " @@ -256,6 +265,7 @@ def build_and_push_one_image( f"{push_arg}" f"--progress plain {image.full_path}" ) + print("GITHUB_REPOSITORY:", GITHUB_REPOSITORY) logging.info("Docker command to run: %s", cmd) with TeePopen(cmd, build_log) as proc: retcode = proc.wait() @@ -400,7 +410,7 @@ def main(): if args.push: subprocess.check_output( # pylint: disable=unexpected-keyword-arg - "docker login --username 'robotclickhouse' --password-stdin", + f"docker login {DOCKER_REPO} --username '{DOCKER_USER}' --password-stdin", input=get_parameter_from_ssm("dockerhub_robot_password"), encoding="utf-8", shell=True, @@ -424,7 +434,7 @@ def main(): # If the event does not contain diff, nothing will be built pass - changed_images = get_changed_docker_images(pr_info, images_dict) + changed_images = get_changed_docker_images(pr_info, images_dict, DOCKER_REPO) if changed_images: logging.info( "Has changed images: %s", ", ".join([im.path for im in changed_images]) diff --git a/tests/ci/docker_manifests_merge.py b/tests/ci/docker_manifests_merge.py index d89708b9277e..8c740db61844 100644 --- a/tests/ci/docker_manifests_merge.py +++ b/tests/ci/docker_manifests_merge.py @@ -11,7 +11,7 @@ from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse from commit_status_helper import format_description, get_commit, post_commit_status -from env_helper import RUNNER_TEMP +from env_helper import RUNNER_TEMP, DOCKER_USER, DOCKER_REPO from get_robot_token import get_best_robot_token, get_parameter_from_ssm from pr_info import PRInfo from report import TestResults, TestResult @@ -174,7 +174,7 @@ def main(): args = parse_args() if args.push: subprocess.check_output( # pylint: disable=unexpected-keyword-arg - "docker login --username 'robotclickhouse' --password-stdin", + f"docker login {DOCKER_REPO} --username '{DOCKER_USER}' --password-stdin", input=get_parameter_from_ssm("dockerhub_robot_password"), encoding="utf-8", shell=True, diff --git a/tests/ci/docker_server.py b/tests/ci/docker_server.py index a434d3cc8411..d3612c873c18 100644 --- a/tests/ci/docker_server.py +++ b/tests/ci/docker_server.py @@ -17,7 +17,15 @@ from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse from commit_status_helper import format_description, get_commit, post_commit_status from docker_images_check import DockerImage -from env_helper import CI, GITHUB_RUN_URL, RUNNER_TEMP, S3_BUILDS_BUCKET, S3_DOWNLOAD +from env_helper import ( + CI, + GITHUB_RUN_URL, + RUNNER_TEMP, + S3_BUILDS_BUCKET, + S3_DOWNLOAD, + DOCKER_USER, + DOCKER_REPO, +) from get_robot_token import get_best_robot_token, get_parameter_from_ssm from git_helper import Git from pr_info import PRInfo @@ -337,12 +345,12 @@ def main(): if args.push: subprocess.check_output( # pylint: disable=unexpected-keyword-arg - "docker login --username 'robotclickhouse' --password-stdin", + f"docker login {DOCKER_REPO} --username '{DOCKER_USER}' --password-stdin", input=get_parameter_from_ssm("dockerhub_robot_password"), encoding="utf-8", shell=True, ) - NAME = f"Docker image {image.repo} build and push" + NAME = f"Docker image {image.repo} build and push (actions)" logging.info("Following tags will be created: %s", ", ".join(tags)) status = "success" diff --git a/tests/ci/docker_test.py b/tests/ci/docker_test.py index 9d68f4364391..8ccfa471871c 100644 --- a/tests/ci/docker_test.py +++ b/tests/ci/docker_test.py @@ -5,7 +5,7 @@ from unittest.mock import patch, MagicMock from pathlib import Path -from env_helper import GITHUB_RUN_URL +from env_helper import GITHUB_RUN_URL, DOCKER_REPO from pr_info import PRInfo from report import TestResult import docker_images_check as di @@ -32,7 +32,9 @@ def test_get_changed_docker_images(self): images = sorted( list( di.get_changed_docker_images( - pr_info, di.get_images_dict("/", self.docker_images_path) + pr_info, + di.get_images_dict("/", self.docker_images_path), + DOCKER_REPO, ) ) ) @@ -129,6 +131,7 @@ def test_build_and_push_one_image(self, mock_machine, mock_popen): self.assertIn( f"docker buildx build --builder default --label build-url={GITHUB_RUN_URL} " "--build-arg FROM_TAG=version " + f"--build-arg DOCKER_REPO={DOCKER_REPO} " f"--build-arg CACHE_INVALIDATOR={GITHUB_RUN_URL} " "--tag name:version --cache-from type=registry,ref=name:version " "--cache-from type=registry,ref=name:latest " @@ -146,6 +149,7 @@ def test_build_and_push_one_image(self, mock_machine, mock_popen): self.assertIn( f"docker buildx build --builder default --label build-url={GITHUB_RUN_URL} " "--build-arg FROM_TAG=version2 " + f"--build-arg DOCKER_REPO={DOCKER_REPO} " f"--build-arg CACHE_INVALIDATOR={GITHUB_RUN_URL} " "--tag name:version2 --cache-from type=registry,ref=name:version2 " "--cache-from type=registry,ref=name:latest " @@ -162,6 +166,7 @@ def test_build_and_push_one_image(self, mock_machine, mock_popen): mock_machine.assert_not_called() self.assertIn( f"docker buildx build --builder default --label build-url={GITHUB_RUN_URL} " + f"--build-arg DOCKER_REPO={DOCKER_REPO} " f"--build-arg CACHE_INVALIDATOR={GITHUB_RUN_URL} " "--tag name:version2 --cache-from type=registry,ref=name:version2 " "--cache-from type=registry,ref=name:latest " @@ -180,6 +185,7 @@ def test_build_and_push_one_image(self, mock_machine, mock_popen): mock_machine.assert_not_called() self.assertIn( f"docker buildx build --builder default --label build-url={GITHUB_RUN_URL} " + f"--build-arg DOCKER_REPO={DOCKER_REPO} " f"--build-arg CACHE_INVALIDATOR={GITHUB_RUN_URL} " "--tag name:version2 --cache-from type=registry,ref=name:version2 " "--cache-from type=registry,ref=name:latest " diff --git a/tests/ci/env_helper.py b/tests/ci/env_helper.py index 9303f9ae2936..938614b18b4c 100644 --- a/tests/ci/env_helper.py +++ b/tests/ci/env_helper.py @@ -10,10 +10,16 @@ CI = bool(os.getenv("CI")) TEMP_PATH = os.getenv("TEMP_PATH", p.abspath(p.join(module_dir, "./tmp"))) + +IBM_COS_INSTANCE_CRN = os.getenv("IBM_COS_INSTANCE_CRN", "") +AWS_ACCESS_KEY_ID = os.getenv("ACCESS_KEY_ID", "secret") +AWS_SECRET_ACCESS_KEY = os.getenv("SECRET_ACCESS_KEY", "secret") CACHES_PATH = os.getenv("CACHES_PATH", TEMP_PATH) CLOUDFLARE_TOKEN = os.getenv("CLOUDFLARE_TOKEN") GITHUB_EVENT_PATH = os.getenv("GITHUB_EVENT_PATH", "") GITHUB_JOB = os.getenv("GITHUB_JOB", "local") +DOCKER_REPO = os.getenv("DOCKER_REPO", "docker.io") +DOCKER_USER = os.getenv("DOCKER_USER", "robotclickhouse") GITHUB_REPOSITORY = os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse") GITHUB_RUN_ID = os.getenv("GITHUB_RUN_ID", "0") GITHUB_SERVER_URL = os.getenv("GITHUB_SERVER_URL", "https://github.com") @@ -23,6 +29,7 @@ REPORTS_PATH = os.getenv("REPORTS_PATH", p.abspath(p.join(module_dir, "./reports"))) REPO_COPY = os.getenv("REPO_COPY", git_root) RUNNER_TEMP = os.getenv("RUNNER_TEMP", p.abspath(p.join(module_dir, "./tmp"))) +S3_REGION = os.getenv("S3_REGION", "us-east-1") S3_BUILDS_BUCKET = os.getenv("S3_BUILDS_BUCKET", "clickhouse-builds") S3_TEST_REPORTS_BUCKET = os.getenv("S3_TEST_REPORTS_BUCKET", "clickhouse-test-reports") S3_URL = os.getenv("S3_URL", "https://s3.amazonaws.com") diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index 89066ade2cb3..c510e3464245 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -23,8 +23,14 @@ update_mergeable_check, ) from docker_pull_helper import get_image_with_version -from env_helper import S3_BUILDS_BUCKET, TEMP_PATH -from get_robot_token import get_best_robot_token +from env_helper import ( + S3_BUILDS_BUCKET, + TEMP_PATH, + DOCKER_USER, + DOCKER_REPO, + S3_URL, +) +from get_robot_token import get_best_robot_token, get_parameter_from_ssm from pr_info import FORCE_TESTS_LABEL, PRInfo from report import TestResults, read_test_results from s3_helper import S3Helper @@ -40,7 +46,7 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, image): return ( - f"docker run --cap-add=SYS_PTRACE " + f"timeout 3h docker run --cap-add=SYS_PTRACE " "--network=host " # required to get access to IAM credentials f"-e FASTTEST_WORKSPACE=/fasttest-workspace -e FASTTEST_OUTPUT=/test_output " f"-e FASTTEST_SOURCE=/ClickHouse --cap-add=SYS_PTRACE " @@ -48,7 +54,9 @@ def get_fasttest_cmd(workspace, output_path, repo_path, pr_number, commit_sha, i f"-e PULL_REQUEST_NUMBER={pr_number} -e COMMIT_SHA={commit_sha} " f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " f"-e SCCACHE_BUCKET={S3_BUILDS_BUCKET} -e SCCACHE_S3_KEY_PREFIX=ccache/sccache " + f"-e SCCACHE_ENDPOINT={S3_URL} " f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " + f"--volume=/home/ubuntu/.aws/credentials:/root/.aws/credentials " # Mount .aws directory f"--volume={output_path}:/test_output {image}" ) @@ -106,6 +114,13 @@ def main(): pr_info = PRInfo() + subprocess.check_output( # pylint: disable=unexpected-keyword-arg + f"docker login {DOCKER_REPO} --username '{DOCKER_USER}' --password-stdin", + input=get_parameter_from_ssm("dockerhub_robot_password"), + encoding="utf-8", + shell=True, + ) + gh = Github(get_best_robot_token(), per_page=100) commit = get_commit(gh, pr_info.sha) @@ -119,8 +134,9 @@ def main(): sys.exit(1) sys.exit(0) - docker_image = get_image_with_version(temp_path, "clickhouse/fasttest") - + docker_image = get_image_with_version( + temp_path, f"{DOCKER_REPO}/clickhouse/fasttest" + ) s3_helper = S3Helper() workspace = os.path.join(temp_path, "fasttest-workspace") diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py index 037bb13f1f87..3f5891e86b1b 100644 --- a/tests/ci/functional_test_check.py +++ b/tests/ci/functional_test_check.py @@ -30,8 +30,8 @@ ) from docker_pull_helper import get_image_with_version from download_release_packages import download_last_release -from env_helper import TEMP_PATH, REPO_COPY, REPORTS_PATH -from get_robot_token import get_best_robot_token +from env_helper import TEMP_PATH, REPO_COPY, REPORTS_PATH, DOCKER_REPO, DOCKER_USER +from get_robot_token import get_best_robot_token, get_parameter_from_ssm from pr_info import FORCE_TESTS_LABEL, PRInfo from report import TestResults, read_test_results from s3_helper import S3Helper @@ -66,9 +66,9 @@ def get_additional_envs(check_name, run_by_hash_num, run_by_hash_total): def get_image_name(check_name): if "stateless" in check_name.lower(): - return "clickhouse/stateless-test" + return f"{DOCKER_REPO}/clickhouse/stateless-test" if "stateful" in check_name.lower(): - return "clickhouse/stateful-test" + return f"{DOCKER_REPO}/clickhouse/stateful-test" else: raise Exception(f"Cannot deduce image name based on check name {check_name}") @@ -304,6 +304,13 @@ def main(): ) sys.exit(0) + subprocess.check_output( # pylint: disable=unexpected-keyword-arg + f"docker login {DOCKER_REPO} --username '{DOCKER_USER}' --password-stdin", + input=get_parameter_from_ssm("dockerhub_robot_password"), + encoding="utf-8", + shell=True, + ) + image_name = get_image_name(check_name) docker_image = get_image_with_version(reports_path, image_name) diff --git a/tests/ci/get_robot_token.py b/tests/ci/get_robot_token.py index b41eba49cc32..f93291be60d1 100644 --- a/tests/ci/get_robot_token.py +++ b/tests/ci/get_robot_token.py @@ -4,9 +4,12 @@ from typing import Optional import boto3 # type: ignore +import hvac # type: ignore from github import Github from github.AuthenticatedUser import AuthenticatedUser +from constants import VAULT_URL, VAULT_TOKEN, VAULT_PATH, VAULT_MOUNT_POINT + @dataclass class Token: @@ -16,9 +19,19 @@ class Token: def get_parameter_from_ssm(name, decrypt=True, client=None): - if not client: - client = boto3.client("ssm", region_name="us-east-1") - return client.get_parameter(Name=name, WithDecryption=decrypt)["Parameter"]["Value"] + if VAULT_URL: + if not client: + client = hvac.Client(url=VAULT_URL, token=VAULT_TOKEN) + parameter = client.secrets.kv.v2.read_secret_version( + mount_point=VAULT_MOUNT_POINT, path=VAULT_PATH + )["data"]["data"][name] + else: + if not client: + client = boto3.client("ssm", region_name="us-east-1") + parameter = client.get_parameter(Name=name, WithDecryption=decrypt)[ + "Parameter" + ]["Value"] + return parameter ROBOT_TOKEN = None # type: Optional[Token] @@ -28,18 +41,41 @@ def get_best_robot_token(token_prefix_env_name="github_robot_token_"): global ROBOT_TOKEN if ROBOT_TOKEN is not None: return ROBOT_TOKEN.value - client = boto3.client("ssm", region_name="us-east-1") - parameters = client.describe_parameters( - ParameterFilters=[ - {"Key": "Name", "Option": "BeginsWith", "Values": [token_prefix_env_name]} - ] - )["Parameters"] - assert parameters - - for token_name in [p["Name"] for p in parameters]: - value = get_parameter_from_ssm(token_name, True, client) + + client = None + values = [] + + if VAULT_URL: + client = hvac.Client(url=VAULT_URL, token=VAULT_TOKEN) + parameters = client.secrets.kv.v2.read_secret_version( + mount_point=VAULT_MOUNT_POINT, path=VAULT_PATH + )["data"]["data"] + parameters = { + key: value + for key, value in parameters.items() + if key.startswith(token_prefix_env_name) + } + assert parameters + values = list(parameters.values()) + else: + client = boto3.client("ssm", region_name="us-east-1") + parameters = client.describe_parameters( + ParameterFilters=[ + { + "Key": "Name", + "Option": "BeginsWith", + "Values": [token_prefix_env_name], + } + ] + )["Parameters"] + assert parameters + for token_name in [p["Name"] for p in parameters]: + value = get_parameter_from_ssm(token_name, True, client) + values.append(value) + + for value in values: gh = Github(value, per_page=100) - # Do not spend additional request to API by accessin user.login unless + # Do not spend additional request to API by accessing user.login unless # the token is chosen by the remaining requests number user = gh.get_user() rest, _ = gh.rate_limiting diff --git a/tests/ci/install_check.py b/tests/ci/install_check.py index d619ce96ceec..cec89b0b791a 100644 --- a/tests/ci/install_check.py +++ b/tests/ci/install_check.py @@ -27,8 +27,8 @@ ) from compress_files import compress_fast from docker_pull_helper import get_image_with_version, DockerImage -from env_helper import CI, TEMP_PATH as TEMP, REPORTS_PATH -from get_robot_token import get_best_robot_token +from env_helper import CI, TEMP_PATH as TEMP, REPORTS_PATH, DOCKER_USER, DOCKER_REPO +from get_robot_token import get_best_robot_token, get_parameter_from_ssm from pr_info import PRInfo from report import TestResults, TestResult from s3_helper import S3Helper @@ -37,8 +37,8 @@ from upload_result_helper import upload_results -RPM_IMAGE = "clickhouse/install-rpm-test" -DEB_IMAGE = "clickhouse/install-deb-test" +RPM_IMAGE = f"{DOCKER_REPO}/clickhouse/install-rpm-test" +DEB_IMAGE = f"{DOCKER_REPO}/clickhouse/install-deb-test" TEMP_PATH = Path(TEMP) LOGS_PATH = TEMP_PATH / "tests_logs" SUCCESS = "success" @@ -267,6 +267,13 @@ def main(): pr_info = PRInfo() + subprocess.check_output( # pylint: disable=unexpected-keyword-arg + f"docker login {DOCKER_REPO} --username '{DOCKER_USER}' --password-stdin", + input=get_parameter_from_ssm("dockerhub_robot_password"), + encoding="utf-8", + shell=True, + ) + if CI: gh = Github(get_best_robot_token(), per_page=100) commit = get_commit(gh, pr_info.sha) diff --git a/tests/ci/integration_test_check.py b/tests/ci/integration_test_check.py index 8ef6244a1c55..190ad1827808 100644 --- a/tests/ci/integration_test_check.py +++ b/tests/ci/integration_test_check.py @@ -27,8 +27,8 @@ ) from docker_pull_helper import get_images_with_versions from download_release_packages import download_last_release -from env_helper import TEMP_PATH, REPO_COPY, REPORTS_PATH -from get_robot_token import get_best_robot_token +from env_helper import TEMP_PATH, REPO_COPY, REPORTS_PATH, DOCKER_REPO, DOCKER_USER +from get_robot_token import get_best_robot_token, get_parameter_from_ssm from pr_info import PRInfo from report import TestResults, read_test_results from s3_helper import S3Helper @@ -40,17 +40,17 @@ # When update, update # integration/ci-runner.py:ClickhouseIntegrationTestsRunner.get_images_names too IMAGES = [ - "clickhouse/integration-tests-runner", - "clickhouse/mysql-golang-client", - "clickhouse/mysql-java-client", - "clickhouse/mysql-js-client", - "clickhouse/mysql-php-client", - "clickhouse/postgresql-java-client", - "clickhouse/integration-test", - "clickhouse/kerberos-kdc", - "clickhouse/kerberized-hadoop", - "clickhouse/integration-helper", - "clickhouse/dotnet-client", + f"{DOCKER_REPO}/clickhouse/integration-tests-runner", + f"{DOCKER_REPO}/clickhouse/mysql-golang-client", + f"{DOCKER_REPO}/clickhouse/mysql-java-client", + f"{DOCKER_REPO}/clickhouse/mysql-js-client", + f"{DOCKER_REPO}/clickhouse/mysql-php-client", + f"{DOCKER_REPO}/clickhouse/postgresql-java-client", + f"{DOCKER_REPO}/clickhouse/integration-test", + f"{DOCKER_REPO}/clickhouse/kerberos-kdc", + f"{DOCKER_REPO}/clickhouse/kerberized-hadoop", + f"{DOCKER_REPO}/clickhouse/integration-helper", + f"{DOCKER_REPO}/clickhouse/dotnet-client", ] @@ -206,6 +206,13 @@ def main(): logging.info("Check is already finished according to github status, exiting") sys.exit(0) + subprocess.check_output( # pylint: disable=unexpected-keyword-arg + f"docker login {DOCKER_REPO} --username '{DOCKER_USER}' --password-stdin", + input=get_parameter_from_ssm("dockerhub_robot_password"), + encoding="utf-8", + shell=True, + ) + images = get_images_with_versions(reports_path, IMAGES) images_with_versions = {i.name: i.version for i in images} result_path = os.path.join(temp_path, "output_dir") diff --git a/tests/ci/run_check.py b/tests/ci/run_check.py index 330a1309016c..f36513fd81c1 100644 --- a/tests/ci/run_check.py +++ b/tests/ci/run_check.py @@ -5,6 +5,15 @@ from github import Github +from docs_check import NAME as DOCS_NAME +from env_helper import GITHUB_REPOSITORY, GITHUB_SERVER_URL +from get_robot_token import get_best_robot_token +from pr_info import FORCE_TESTS_LABEL, PRInfo +from lambda_shared_package.lambda_shared.pr import ( + CATEGORY_TO_LABEL, + TRUSTED_CONTRIBUTORS, + check_pr_description, +) from commit_status_helper import ( CI_STATUS_NAME, NotSet, @@ -16,15 +25,6 @@ remove_labels, set_mergeable_check, ) -from docs_check import NAME as DOCS_NAME -from env_helper import GITHUB_REPOSITORY, GITHUB_SERVER_URL -from get_robot_token import get_best_robot_token -from pr_info import FORCE_TESTS_LABEL, PRInfo -from lambda_shared_package.lambda_shared.pr import ( - CATEGORY_TO_LABEL, - TRUSTED_CONTRIBUTORS, - check_pr_description, -) TRUSTED_ORG_IDS = { 54801242, # clickhouse diff --git a/tests/ci/s3_helper.py b/tests/ci/s3_helper.py index 9ee0a4312947..441ebf006f08 100644 --- a/tests/ci/s3_helper.py +++ b/tests/ci/s3_helper.py @@ -15,6 +15,7 @@ from env_helper import ( S3_TEST_REPORTS_BUCKET, S3_BUILDS_BUCKET, + S3_REGION, RUNNER_TEMP, CI, S3_URL, @@ -47,7 +48,7 @@ class S3Helper: def __init__(self): config = botocore.config.Config(max_pool_connections=self.max_pool_size) - self.session = boto3.session.Session(region_name="us-east-1") + self.session = boto3.session.Session(region_name=S3_REGION) self.client = self.session.client("s3", endpoint_url=S3_URL, config=config) self.host = S3_URL self.download_host = S3_DOWNLOAD diff --git a/tests/ci/stress_check.py b/tests/ci/stress_check.py index ac280916a2f7..8b3987bb248a 100644 --- a/tests/ci/stress_check.py +++ b/tests/ci/stress_check.py @@ -18,8 +18,8 @@ ) from commit_status_helper import RerunHelper, get_commit, post_commit_status from docker_pull_helper import get_image_with_version -from env_helper import TEMP_PATH, REPO_COPY, REPORTS_PATH -from get_robot_token import get_best_robot_token +from env_helper import TEMP_PATH, REPO_COPY, REPORTS_PATH, DOCKER_REPO, DOCKER_USER +from get_robot_token import get_best_robot_token, get_parameter_from_ssm from pr_info import PRInfo from report import TestResults, read_test_results from s3_helper import S3Helper @@ -131,6 +131,13 @@ def run_stress_test(docker_image_name): logging.info("Check is already finished according to github status, exiting") sys.exit(0) + subprocess.check_output( # pylint: disable=unexpected-keyword-arg + f"docker login {DOCKER_REPO} --username '{DOCKER_USER}' --password-stdin", + input=get_parameter_from_ssm("dockerhub_robot_password"), + encoding="utf-8", + shell=True, + ) + docker_image = get_image_with_version(reports_path, docker_image_name) packages_path = os.path.join(temp_path, "packages") @@ -198,4 +205,4 @@ def run_stress_test(docker_image_name): if __name__ == "__main__": - run_stress_test("clickhouse/stress-test") + run_stress_test(f"{DOCKER_REPO}/clickhouse/stress-test") diff --git a/tests/ci/style_check.py b/tests/ci/style_check.py index 33a5cd21f39b..835643fda238 100644 --- a/tests/ci/style_check.py +++ b/tests/ci/style_check.py @@ -22,8 +22,8 @@ update_mergeable_check, ) from docker_pull_helper import get_image_with_version -from env_helper import GITHUB_WORKSPACE, RUNNER_TEMP -from get_robot_token import get_best_robot_token +from get_robot_token import get_best_robot_token, get_parameter_from_ssm +from env_helper import GITHUB_WORKSPACE, RUNNER_TEMP, DOCKER_USER, DOCKER_REPO from github_helper import GitHub from git_helper import git_runner from pr_info import PRInfo @@ -152,6 +152,13 @@ def main(): if args.push: checkout_head(pr_info) + subprocess.check_output( # pylint: disable=unexpected-keyword-arg + f"docker login {DOCKER_REPO} --username '{DOCKER_USER}' --password-stdin", + input=get_parameter_from_ssm("dockerhub_robot_password"), + encoding="utf-8", + shell=True, + ) + gh = GitHub(get_best_robot_token(), create_cache_dir=False) commit = get_commit(gh, pr_info.sha) @@ -169,7 +176,9 @@ def main(): if not os.path.exists(temp_path): os.makedirs(temp_path) - docker_image = get_image_with_version(temp_path, "clickhouse/style-test") + docker_image = get_image_with_version( + temp_path, f"{DOCKER_REPO}/clickhouse/style-test" + ) s3_helper = S3Helper() cmd = ( diff --git a/tests/ci/version_test.py b/tests/ci/version_test.py index abd0f9349f4b..50ba8017f471 100644 --- a/tests/ci/version_test.py +++ b/tests/ci/version_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import unittest from argparse import ArgumentTypeError diff --git a/tests/ci/worker/prepare-ci-ami.sh b/tests/ci/worker/prepare-ci-ami.sh index 20e7e3fd53e2..59de58d884b1 100644 --- a/tests/ci/worker/prepare-ci-ami.sh +++ b/tests/ci/worker/prepare-ci-ami.sh @@ -90,7 +90,7 @@ systemctl restart docker sudo -u ubuntu docker buildx version sudo -u ubuntu docker buildx create --use --name default-builder -pip install boto3 pygithub requests urllib3 unidiff dohq-artifactory +pip install boto3 pygithub requests urllib3 unidiff dohq-artifactory hvac mkdir -p $RUNNER_HOME && cd $RUNNER_HOME