Util base

.github/workflows/binary_image.yml

@@ -0,0 +1,157 @@
+name: binary-image
+
+env:
+  # Force the stdout and stderr streams to be unbuffered
+  PYTHONUNBUFFERED: 1
+
+on:  # yamllint disable-line rule:truthy
+  push:
+  # pull_request:
+  #   types:
+  #     - synchronize
+  #     - reopened
+  #     - opened
+  #   branches:
+  #     - master
+  #   paths-ignore:
+  #     - 'CHANGELOG.md'
+  #     - 'README.md'
+  #     - 'SECURITY.md'
+  #     - 'docker/docs/**'
+  #     - 'docs/**'
+  #     - 'utils/check-style/aspell-ignore/**'
+jobs:
+  DockerHubPushAarch64:
+    runs-on: [self-hosted, style-checker-aarch64]
+    steps:
+      - name: Check out repository code
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
+      - name: Images check
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 docker_images_check.py --suffix aarch64
+      - name: Upload images files to artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: changed_images_aarch64
+          path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json
+  DockerHubPushAmd64:
+    runs-on: [self-hosted, builder]
+    steps:
+      - name: Check out repository code
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
+      - name: Images check
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 docker_images_check.py --suffix amd64
+      - name: Upload images files to artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: changed_images_amd64
+          path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json
+  DockerHubPush:
+    needs: [DockerHubPushAmd64, DockerHubPushAarch64]
+    runs-on: [self-hosted, builder]
+    steps:
+      - name: Check out repository code
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
+      - name: Download changed aarch64 images
+        uses: actions/download-artifact@v3
+        with:
+          name: changed_images_aarch64
+          path: ${{ runner.temp }}
+      - name: Download changed amd64 images
+        uses: actions/download-artifact@v3
+        with:
+          name: changed_images_amd64
+          path: ${{ runner.temp }}
+      - name: Images check
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64
+      - name: Upload images files to artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: changed_images
+          path: ${{ runner.temp }}/changed_images.json
+  # BuilderDebAarch64:
+  #   needs: [DockerHubPush]
+  #   runs-on: [self-hosted, vm2]
+  #   steps:
+  #     - name: Set envs
+  #       run: |
+  #         cat >> "$GITHUB_ENV" << 'EOF'
+  #         TEMP_PATH=${{runner.temp}}/build_check
+  #         IMAGES_PATH=${{runner.temp}}/images_path
+  #         REPO_COPY=${{runner.temp}}/build_check/ClickHouse
+  #         CACHES_PATH=${{runner.temp}}/../ccaches
+  #         BUILD_NAME=package_aarch64
+  #         EOF
+  #     - name: Download changed images
+  #       uses: actions/download-artifact@v3
+  #       with:
+  #         name: changed_images
+  #         path: ${{ runner.temp }}/images_path
+  #     - name: Check out repository code
+  #       uses: ClickHouse/checkout@v1
+  #       with:
+  #         clear-repository: true
+  #         submodules: true
+  #         fetch-depth: 0  # for performance artifact
+  #     - name: Build
+  #       run: |
+  #         sudo rm -fr "$TEMP_PATH"
+  #         mkdir -p "$TEMP_PATH"
+  #         cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+  #         cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME"
+  #     - name: Upload build URLs to artifacts
+  #       if: ${{ success() || failure() }}
+  #       uses: actions/upload-artifact@v3
+  #       with:
+  #         name: ${{ env.BUILD_URLS }}
+  #         path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
+  #     - name: Cleanup
+  #       if: always()
+  #       run: |
+  #         docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+  #         docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+  #         sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
+  # InstallPackagesTestAarch64:
+  #   needs: [BuilderDebAarch64]
+  #   runs-on: [self-hosted, test41]
+  #   steps:
+  #     - name: Set envs
+  #       run: |
+  #         cat >> "$GITHUB_ENV" << 'EOF'
+  #         TEMP_PATH=${{runner.temp}}/test_install
+  #         REPORTS_PATH=${{runner.temp}}/reports_dir
+  #         CHECK_NAME=Install packages (arm64)
+  #         REPO_COPY=${{runner.temp}}/test_install/ClickHouse
+  #         EOF
+  #     - name: Download json reports
+  #       uses: actions/download-artifact@v3
+  #       with:
+  #         path: ${{ env.REPORTS_PATH }}
+  #     - name: Check out repository code
+  #       uses: ClickHouse/checkout@v1
+  #       with:
+  #         clear-repository: true
+  #     - name: Test packages installation
+  #       run: |
+  #         sudo rm -fr "$TEMP_PATH"
+  #         mkdir -p "$TEMP_PATH"
+  #         cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+  #         cd "$REPO_COPY/tests/ci"
+  #         python3 install_check.py "$CHECK_NAME"
+  #     - name: Cleanup
+  #       if: always()
+  #       run: |
+  #         docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+  #         docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+  #         sudo rm -fr "$TEMP_PATH"          

.github/workflows/master.yml

@@ -9,19 +9,8 @@ on: # yamllint disable-line rule:truthy
     branches:
       - 'master'
 jobs:
-  PythonUnitTests:
-    runs-on: [self-hosted, style-checker]
-    steps:
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
-      - name: Python unit tests
-        run: |
-          cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 -m unittest discover -s . -p '*_test.py'
   DockerHubPushAarch64:
-    runs-on: [self-hosted, style-checker-aarch64]
+    runs-on: [self-hosted, style-checker]
     steps:
       - name: Check out repository code
         uses: ClickHouse/checkout@v1
@@ -53,7 +42,7 @@ jobs:
           name: changed_images_amd64
           path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json
   DockerHubPush:
-    needs: [DockerHubPushAmd64, DockerHubPushAarch64, PythonUnitTests]
+    needs: [DockerHubPushAmd64, DockerHubPushAarch64]
     runs-on: [self-hosted, style-checker]
     steps:
       - name: Check out repository code