diff --git a/.github/workflows/bindings.python.yml b/.github/workflows/bindings.python.yml index d8a0a2fbc829..2a0fc2ba34c0 100644 --- a/.github/workflows/bindings.python.yml +++ b/.github/workflows/bindings.python.yml @@ -22,7 +22,7 @@ concurrency: jobs: linux: - runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 8c16g, aws] + runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 4c16g, aws] strategy: matrix: include: diff --git a/.github/workflows/cloud.yml b/.github/workflows/cloud.yml index 6ae8f8364bd6..49fefb823466 100644 --- a/.github/workflows/cloud.yml +++ b/.github/workflows/cloud.yml @@ -45,7 +45,7 @@ jobs: build: needs: info - runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 16c32g, aws] + runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 8c32g, aws] strategy: matrix: include: @@ -69,7 +69,7 @@ jobs: docker: needs: [info, build] timeout-minutes: 10 - runs-on: [self-hosted, X64, Linux, 4c8g, aws] + runs-on: [self-hosted, X64, Linux, 2c8g, aws] outputs: tag: ${{ steps.prepare.outputs.tag }} steps: diff --git a/.github/workflows/meta.yml b/.github/workflows/meta.yml index 34c694744c25..bba330a4efe2 100644 --- a/.github/workflows/meta.yml +++ b/.github/workflows/meta.yml @@ -36,7 +36,7 @@ jobs: build: needs: info - runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 16c32g, aws] + runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 8c32g, aws] strategy: matrix: include: @@ -56,7 +56,7 @@ jobs: chaos: needs: [info, build] - runs-on: [self-hosted, X64, Linux, 8c16g, aws] + runs-on: [self-hosted, X64, Linux, 4c16g, aws] steps: - uses: actions/checkout@v4 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f520db66eced..366a7c7a3d13 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -71,7 +71,7 @@ jobs: - name: Checkout Docs uses: actions/checkout@v4 with: - repository: datafuselabs/databend-docs + repository: databendlabs/databend-docs ref: main - name: Get date shell: bash @@ -83,10 +83,10 @@ jobs: mkdir -p docs/release-stable df="docs/release-stable/${{ env.DATE }}_${{ needs.create_release.outputs.version }}.md" echo "---" > $df - gh release view --repo datafuselabs/databend ${{ needs.create_release.outputs.version }} >> $df + gh release view --repo databendlabs/databend ${{ needs.create_release.outputs.version }} >> $df sed -i -E 's/^--$/---/g' $df sed -i -E '/^asset:/d' $df - sed -i -E 's_https://github.com/datafuselabs/databend/pull/([0-9]+)_[#\1](https://github.com/datafuselabs/databend/pull/\1)_g' $df + sed -i -E 's_https://github.com/databendlabs/databend/pull/([0-9]+)_[#\1](https://github.com/databendlabs/databend/pull/\1)_g' $df git add docs/release-stable git status - uses: peter-evans/create-pull-request@v4 @@ -99,7 +99,7 @@ jobs: delete-branch: true build_default: - runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 16c32g, aws] + runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 8c32g, aws] needs: create_release strategy: fail-fast: false @@ -117,6 +117,7 @@ jobs: uses: ./.github/actions/build_linux env: DATABEND_RELEASE_VERSION: ${{ needs.create_release.outputs.version }} + DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY: ${{ secrets.DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY }} with: sha: ${{ github.sha }} target: ${{ matrix.target }} @@ -130,15 +131,15 @@ jobs: cp ./target/${{ matrix.target }}/${{ env.BUILD_PROFILE }}/databend-* ./target/${{ env.BUILD_PROFILE }}/ bash ./scripts/ci/ci-run-sqllogic-tests.sh base - build_udf: - runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 16c32g, aws] + build_musl: + runs-on: [self-hosted, X64, Linux, 8c32g, aws] needs: create_release strategy: fail-fast: false matrix: - include: - - { target: x86_64-unknown-linux-gnu, runner: X64 } - - { target: aarch64-unknown-linux-gnu, runner: ARM64 } + target: + - x86_64-unknown-linux-musl + - aarch64-unknown-linux-musl steps: - name: Checkout uses: actions/checkout@v4 @@ -149,15 +150,14 @@ jobs: uses: ./.github/actions/build_linux env: DATABEND_RELEASE_VERSION: ${{ needs.create_release.outputs.version }} + DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY: ${{ secrets.DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY }} with: sha: ${{ github.sha }} target: ${{ matrix.target }} - artifacts: sqllogictests,sqlsmith,metactl,meta,query - features: python-udf - category: udf + artifacts: query,meta,metactl - build_hdfs: - runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 16c32g, aws] + build_udf: + runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 8c32g, aws] needs: create_release strategy: fail-fast: false @@ -175,32 +175,29 @@ jobs: uses: ./.github/actions/build_linux env: DATABEND_RELEASE_VERSION: ${{ needs.create_release.outputs.version }} + DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY: ${{ secrets.DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY }} with: sha: ${{ github.sha }} target: ${{ matrix.target }} artifacts: sqllogictests,sqlsmith,metactl,meta,query - features: storage-hdfs - category: hdfs + features: python-udf + category: udf publish: - runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 4c8g, aws] - needs: [create_release, build_default, build_hdfs] + runs-on: [self-hosted, X64, Linux, 2c8g, aws] + needs: [create_release, build_default, build_musl] strategy: fail-fast: false matrix: include: - category: default target: x86_64-unknown-linux-gnu - runner: X64 - category: default target: aarch64-unknown-linux-gnu - runner: ARM64 - - category: hdfs - target: x86_64-unknown-linux-gnu - runner: X64 - - category: hdfs - target: aarch64-unknown-linux-gnu - runner: ARM64 + - category: default + target: x86_64-unknown-linux-musl + - category: default + target: aarch64-unknown-linux-musl steps: - name: Checkout uses: actions/checkout@v4 @@ -254,7 +251,7 @@ jobs: category: ${{ matrix.category }} publish_testsuite: - runs-on: [self-hosted, X64, Linux, 4c8g, aws] + runs-on: [self-hosted, X64, Linux, 2c8g, aws] needs: [create_release, build_default] strategy: fail-fast: false @@ -298,7 +295,7 @@ jobs: category: testsuite docker_all_in_one: - runs-on: [self-hosted, X64, Linux, 4c8g, aws] + runs-on: [self-hosted, X64, Linux, 2c8g, aws] needs: [create_release, build_default] steps: - name: Checkout @@ -373,7 +370,7 @@ jobs: readme-filepath: ./docker/README.md docker_service: - runs-on: [self-hosted, X64, Linux, 4c8g, aws] + runs-on: [self-hosted, X64, Linux, 2c8g, aws] needs: [create_release, build_udf] strategy: fail-fast: false @@ -444,7 +441,7 @@ jobs: file: ./docker/service/${{ matrix.service }}.Dockerfile distribution: - runs-on: [self-hosted, X64, Linux, 4c8g, aws] + runs-on: [self-hosted, X64, Linux, 2c8g, aws] needs: [create_release, build_default] strategy: matrix: @@ -595,7 +592,7 @@ jobs: sqlsmith: needs: [create_release, notify] - runs-on: [self-hosted, X64, Linux, 4c8g, aws] + runs-on: [self-hosted, X64, Linux, 2c8g, aws] steps: - uses: actions/checkout@v4 with: @@ -620,7 +617,7 @@ jobs: metachaos: needs: [create_release, notify] - runs-on: [self-hosted, X64, Linux, 8c16g, aws] + runs-on: [self-hosted, X64, Linux, 4c16g, aws] steps: - uses: actions/checkout@v4 with: @@ -646,7 +643,7 @@ jobs: await script({context, core}) # sharing: - # runs-on: [self-hosted, X64, Linux, 4c8g, aws] + # runs-on: [self-hosted, X64, Linux, 2c8g, aws] # needs: [create_release, notify] # steps: # - uses: actions/checkout@v4 @@ -655,7 +652,7 @@ jobs: # - name: checkout share endpoint # uses: actions/checkout@v4 # with: - # repository: datafuselabs/share-endpoint + # repository: databendlabs/share-endpoint # token: ${{ secrets.DATABEND_BOT_TOKEN }} # path: share-endpoint # - name: Download artifacts diff --git a/.github/workflows/reuse.benchmark.yml b/.github/workflows/reuse.benchmark.yml index 32042aacd5cf..d8097d9297d0 100644 --- a/.github/workflows/reuse.benchmark.yml +++ b/.github/workflows/reuse.benchmark.yml @@ -44,7 +44,7 @@ env: jobs: local: timeout-minutes: 60 - runs-on: [self-hosted, X64, Linux, 16c32g, "${{ inputs.runner_provider }}"] + runs-on: [self-hosted, X64, Linux, 8c32g, "${{ inputs.runner_provider }}"] strategy: matrix: dataset: diff --git a/.github/workflows/reuse.linux.hive.yml b/.github/workflows/reuse.linux.hive.yml index d91c8133195d..e6400b4cbb9e 100644 --- a/.github/workflows/reuse.linux.hive.yml +++ b/.github/workflows/reuse.linux.hive.yml @@ -20,7 +20,7 @@ env: jobs: build: - runs-on: [self-hosted, X64, Linux, 8c16g, "${{ inputs.runner_provider }}"] + runs-on: [self-hosted, X64, Linux, 4c16g, "${{ inputs.runner_provider }}"] strategy: matrix: include: @@ -41,7 +41,7 @@ jobs: test_stateful_hive_standalone: needs: build - runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] steps: - uses: actions/checkout@v4 - uses: ./.github/actions/test_stateful_hive_standalone diff --git a/.github/workflows/reuse.linux.yml b/.github/workflows/reuse.linux.yml index 51695f781331..91652bed3739 100644 --- a/.github/workflows/reuse.linux.yml +++ b/.github/workflows/reuse.linux.yml @@ -13,6 +13,11 @@ on: type: string required: true default: "aws" + license_type: + description: "License type, enterprise or trial" + type: string + required: true + default: "trial" env: BUILD_PROFILE: ${{ inputs.build_profile }} @@ -20,7 +25,7 @@ env: jobs: check: - runs-on: [ self-hosted, X64, Linux, 16c32g, "${{ inputs.runner_provider }}" ] + runs-on: [self-hosted, X64, Linux, 8c32g, "${{ inputs.runner_provider }}"] steps: - uses: actions/checkout@v4 with: @@ -36,7 +41,7 @@ jobs: - self-hosted - "${{ matrix.runner }}" - Linux - - 16c32g + - 8c32g - "${{ inputs.runner_provider }}" strategy: fail-fast: false @@ -50,6 +55,8 @@ jobs: fetch-depth: 0 - uses: ./.github/actions/build_linux timeout-minutes: 60 + env: + DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY: ${{ secrets.DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY }} with: sha: ${{ github.sha }} target: ${{ matrix.arch }}-unknown-linux-gnu @@ -60,7 +67,7 @@ jobs: - self-hosted - "${{ matrix.runner }}" - Linux - - 16c32g + - 8c32g - "${{ inputs.runner_provider }}" strategy: fail-fast: false @@ -75,6 +82,8 @@ jobs: fetch-depth: 0 - uses: ./.github/actions/build_linux timeout-minutes: 60 + env: + DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY: ${{ secrets.DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY }} with: sha: ${{ github.sha }} target: ${{ matrix.arch }}-unknown-linux-gnu @@ -87,7 +96,7 @@ jobs: # - self-hosted # - "${{ matrix.runner }}" # - Linux - # - 16c32g + # - 8c32g # - "${{ inputs.runner_provider }}" # strategy: # fail-fast: false @@ -106,7 +115,7 @@ jobs: # artifacts: query test_unit: - runs-on: [ self-hosted, X64, Linux, 16c32g, "${{ inputs.runner_provider }}" ] + runs-on: [self-hosted, X64, Linux, 8c32g, "${{ inputs.runner_provider }}"] steps: - uses: actions/checkout@v4 with: @@ -116,75 +125,76 @@ jobs: timeout-minutes: 60 test_metactl: - runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ] - needs: [ build, check ] + runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] + needs: [build, check] steps: - uses: actions/checkout@v4 - uses: ./.github/actions/test_metactl timeout-minutes: 10 test_compat_meta_query: - runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ] - needs: [ build, check ] + runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] + needs: [build, check] steps: - uses: actions/checkout@v4 - uses: ./.github/actions/test_compat_meta_query timeout-minutes: 10 test_compat_fuse: - runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ] - needs: [ build, check ] + runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] + needs: [build, check] steps: - uses: actions/checkout@v4 - uses: ./.github/actions/test_compat_fuse timeout-minutes: 20 test_compat_meta_meta: - runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ] - needs: [ build, check ] + runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] + needs: [build, check] steps: - uses: actions/checkout@v4 - uses: ./.github/actions/test_compat_meta_meta timeout-minutes: 20 test_logs: - runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ] - needs: [ build, check ] + runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] + needs: [build, check] steps: - uses: actions/checkout@v4 - uses: ./.github/actions/test_logs timeout-minutes: 20 test_meta_cluster: - runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ] - needs: [ build, check ] + runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] + needs: [build, check] steps: - uses: actions/checkout@v4 - uses: ./.github/actions/test_meta_cluster timeout-minutes: 10 test_stateless_standalone: - runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ] - needs: [ build, check ] + runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] + needs: [build, check] steps: - uses: actions/checkout@v4 - uses: ./.github/actions/test_stateless_standalone_linux timeout-minutes: 15 test_stateless_cluster: - runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ] - needs: [ build, check ] + runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] + needs: [build, check] steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup_license with: runner_provider: ${{ inputs.runner_provider }} + type: ${{ inputs.license_type }} - uses: ./.github/actions/test_stateless_cluster_linux timeout-minutes: 15 test_stateful_standalone: - runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ] - needs: [ build, check ] + runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] + needs: [build, check] steps: - uses: actions/checkout@v4 - uses: ./.github/actions/test_stateful_standalone_linux @@ -196,13 +206,14 @@ jobs: name: test-stateful-standalone-linux test_stateful_cluster: - runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ] - needs: [ build, check ] + runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] + needs: [build, check] steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup_license with: runner_provider: ${{ inputs.runner_provider }} + type: ${{ inputs.license_type }} - uses: ./.github/actions/test_stateful_cluster_linux timeout-minutes: 15 - name: Upload failure @@ -213,16 +224,16 @@ jobs: test_stateful_large_data: if: contains(github.event.pull_request.labels.*.name, 'ci-largedata') - runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ] - needs: [ build, check ] + runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] + needs: [build, check] steps: - uses: actions/checkout@v4 - uses: ./.github/actions/test_stateful_large_data timeout-minutes: 60 test_stateful_iceberg_rest: - runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ] - needs: [ build, check ] + runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] + needs: [build, check] steps: - uses: actions/checkout@v4 - uses: ./.github/actions/test_stateful_iceberg_rest_standalone @@ -234,7 +245,7 @@ jobs: name: test-stateful-iceberg-rest-standalone # test_fuzz_standalone: - # runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + # runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] # needs: [build, check] # steps: # - uses: actions/checkout@v4 @@ -243,13 +254,14 @@ jobs: # continue-on-error: true test_ee_standalone: - needs: [ build, check ] - runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ] + needs: [build, check] + runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup_license with: runner_provider: ${{ inputs.runner_provider }} + type: ${{ inputs.license_type }} - uses: ./.github/actions/test_ee_standalone_linux timeout-minutes: 10 - name: Upload failure @@ -259,14 +271,15 @@ jobs: name: test-stateful-standalone-linux test_ee_standalone_background: - needs: [ build, check ] - runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ] + needs: [build, check] + runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup_bendsql - uses: ./.github/actions/setup_license with: runner_provider: ${{ inputs.runner_provider }} + type: ${{ inputs.license_type }} - uses: ./.github/actions/test_ee_standalone_background_linux timeout-minutes: 10 - name: Upload failure @@ -282,12 +295,13 @@ jobs: # # test_ee_standalone_fake_time: # needs: [build, check] - # runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + # runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] # steps: # - uses: actions/checkout@v4 # - uses: ./.github/actions/setup_license # with: # runner_provider: ${{ inputs.runner_provider }} + # type: ${{ inputs.license_type }} # - uses: ./.github/actions/test_ee_standalone_fake_time_linux # timeout-minutes: 10 # - name: Upload failure @@ -297,14 +311,15 @@ jobs: # name: test-stateful-standalone-fake-time-linux test_ee_management_mode: - needs: [ build, check ] - runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ] + needs: [build, check] + runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup_bendsql - uses: ./.github/actions/setup_license with: runner_provider: ${{ inputs.runner_provider }} + type: ${{ inputs.license_type }} - uses: ./.github/actions/test_ee_management_mode_linux timeout-minutes: 10 - name: Upload failure @@ -314,9 +329,10 @@ jobs: name: test-ee-management-mode-linux sqllogic: - needs: [ build, check ] + needs: [build, check] uses: ./.github/workflows/reuse.sqllogic.yml secrets: inherit with: build_profile: ${{ inputs.build_profile }} runner_provider: ${{ inputs.runner_provider }} + license_type: ${{ inputs.license_type }} diff --git a/.github/workflows/reuse.sqllogic.yml b/.github/workflows/reuse.sqllogic.yml index 8f50d0edaedc..a959a38c470c 100644 --- a/.github/workflows/reuse.sqllogic.yml +++ b/.github/workflows/reuse.sqllogic.yml @@ -13,6 +13,11 @@ on: type: string required: true default: "aws" + license_type: + description: "License type, enterprise or trial" + type: string + required: true + default: "trial" env: BUILD_PROFILE: ${{ inputs.build_profile }} @@ -20,7 +25,7 @@ env: jobs: management_mode: - runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] steps: - uses: actions/checkout@v4 - uses: ./.github/actions/test_sqllogic_management_mode_linux @@ -30,19 +35,24 @@ jobs: handlers: mysql,http standalone: - runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + runs-on: + - self-hosted + - X64 + - Linux + - "${{ matrix.tests.runner }}" + - "${{ inputs.runner_provider }}" strategy: fail-fast: false matrix: - dirs: - - "query" - - "standalone" - - "crdb" - - "duckdb" - - "base" - - "ydb" - - "tpcds" - - "tpch" + tests: + - { dirs: "query", runner: "4c16g" } + - { dirs: "duckdb", runner: "4c16g" } + - { dirs: "crdb", runner: "2c8g" } + - { dirs: "base", runner: "2c8g" } + - { dirs: "ydb", runner: "2c8g" } + - { dirs: "tpcds", runner: "2c8g" } + - { dirs: "tpch", runner: "2c8g" } + - { dirs: "standalone", runner: "2c8g" } handler: - "mysql" - "http" @@ -51,22 +61,22 @@ jobs: - uses: ./.github/actions/test_sqllogic_standalone_linux timeout-minutes: 15 with: - dirs: ${{ matrix.dirs }} + dirs: ${{ matrix.tests.dirs }} handlers: ${{ matrix.handler }} storage-format: all - name: Upload failure if: failure() uses: ./.github/actions/artifact_failure with: - name: test-sqllogic-standalone-${{ matrix.dirs }}-${{ matrix.handler }} + name: test-sqllogic-standalone-${{ matrix.tests.dirs }}-${{ matrix.handler }} standalone_udf_server: - runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] steps: - uses: actions/checkout@v4 - name: Start UDF Server run: | - pip install databend-udf + pip install databend-udf>=0.2.6 python3 tests/udf/udf_server.py & sleep 2 - uses: ./.github/actions/test_sqllogic_standalone_linux @@ -82,7 +92,7 @@ jobs: name: test-sqllogic-standalone-udf-server standalone_cloud: - runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + runs-on: [self-hosted, X64, Linux, 4c16g, "${{ inputs.runner_provider }}"] steps: - uses: actions/checkout@v4 - name: Start Cloud Control Server @@ -103,7 +113,7 @@ jobs: name: test-sqllogic-standalone-cloud standalone_minio: - runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] strategy: fail-fast: false matrix: @@ -129,20 +139,44 @@ jobs: with: name: test-sqllogic-standalone-minio-${{ matrix.dirs }}-${{ matrix.handler }}-${{ matrix.format }} + standalone_iceberg_tpch: + runs-on: [self-hosted, X64, Linux, 4c16g, "${{ inputs.runner_provider }}"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: "temurin" + java-version: "17" + - uses: ./.github/actions/test_sqllogic_iceberg_tpch + timeout-minutes: 15 + with: + dirs: tpch_iceberg + handlers: mysql,http + - name: Upload failure + if: failure() + uses: ./.github/actions/artifact_failure + with: + name: test-sqllogic-standalone-iceberg-tpch + cluster: - runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + runs-on: + - self-hosted + - X64 + - Linux + - "${{ matrix.tests.runner }}" + - "${{ inputs.runner_provider }}" strategy: fail-fast: false matrix: - dirs: - - "query" - - "crdb" - - "duckdb" - - "base" - - "ydb" - - "tpcds" - - "tpch" - - "cluster" + tests: + - { dirs: "query", runner: "4c16g" } + - { dirs: "duckdb", runner: "4c16g" } + - { dirs: "crdb", runner: "2c8g" } + - { dirs: "base", runner: "2c8g" } + - { dirs: "ydb", runner: "2c8g" } + - { dirs: "tpcds", runner: "2c8g" } + - { dirs: "tpch", runner: "2c8g" } + - { dirs: "cluster", runner: "2c8g" } handler: - "mysql" - "http" @@ -151,19 +185,20 @@ jobs: - uses: ./.github/actions/setup_license with: runner_provider: ${{ inputs.runner_provider }} + type: ${{ inputs.license_type }} - uses: ./.github/actions/test_sqllogic_cluster_linux timeout-minutes: 15 with: - dirs: ${{ matrix.dirs }} + dirs: ${{ matrix.tests.dirs }} handlers: ${{ matrix.handler }} - name: Upload failure if: failure() uses: ./.github/actions/artifact_failure with: - name: test-sqllogic-cluster-${{ matrix.dirs }}-${{ matrix.handler }} + name: test-sqllogic-cluster-${{ matrix.tests.dirs }}-${{ matrix.handler }} stage: - runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] strategy: fail-fast: false matrix: @@ -185,7 +220,7 @@ jobs: name: test-sqllogic-stage-${{ matrix.storage }} standalone_no_table_meta_cache: - runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] strategy: fail-fast: false matrix: @@ -209,7 +244,7 @@ jobs: name: test-sqllogic-standalone-no-table-meta-cache-${{ matrix.dirs }}-${{ matrix.handler }} ee: - runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"] + runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"] strategy: fail-fast: false matrix: @@ -221,6 +256,7 @@ jobs: - uses: ./.github/actions/setup_license with: runner_provider: ${{ inputs.runner_provider }} + type: ${{ inputs.license_type }} - uses: ./.github/actions/test_ee_sqllogic_standalone_linux timeout-minutes: 15 with: diff --git a/Cargo.toml b/Cargo.toml index 8e37587e21c6..7ac479caf3a1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -364,10 +364,6 @@ opt-level = "s" # databend-query = { codegen-units = 4 } # databend-binaries = { codegen-units = 4 } -[profile.bench] -debug = true -overflow-checks = false - [profile.dev] split-debuginfo = "unpacked" overflow-checks = false @@ -382,6 +378,10 @@ miniz_oxide = { opt-level = 3 } object = { opt-level = 3 } rustc-demangle = { opt-level = 3 } +[profile.bench] +debug = true +overflow-checks = false + [profile.test] opt-level = 0 debug = true diff --git a/tests/sqllogictests/suites/query/set.test b/tests/sqllogictests/suites/query/set.test index 2bfd15affc63..7cb7be8e5e21 100644 --- a/tests/sqllogictests/suites/query/set.test +++ b/tests/sqllogictests/suites/query/set.test @@ -1,14 +1,14 @@ statement ok -SET GLOBAL (max_threads, storage_io_min_bytes_for_seek) = (4, 56); +SET GLOBAL (max_threads, storage_io_min_bytes_for_seek) = (13, 56); query TT select value, default = value from system.settings where name in ('max_threads', 'storage_io_min_bytes_for_seek') order by value; ---- -4 0 +13 0 56 0 statement ok -set variable (a, b) = (select 3, 55) +set variable (a, b) = (select 12, 55) statement ok @@ -17,7 +17,7 @@ SET GLOBAL (max_threads, storage_io_min_bytes_for_seek) = select $a + 1, $b + 1; query TT select value, default = value from system.settings where name in ('max_threads', 'storage_io_min_bytes_for_seek') order by value; ---- -4 0 +13 0 56 0 statement ok @@ -36,7 +36,7 @@ statement ok set variable (b, c) = ('yy', 'zz'); query ITT -select $a + getvariable('a') + $a, getvariable('b'), getvariable('c'), getvariable('d') +select $a + getvariable('a') + $a, getvariable('b'), getvariable('c'), getvariable('d') ---- 3 yy zz NULL @@ -44,7 +44,7 @@ statement ok unset variable (a, b) query ITT -select getvariable('a'), getvariable('b'), 'xx' || 'yy' || getvariable('c') , getvariable('d') +select getvariable('a'), getvariable('b'), 'xx' || 'yy' || getvariable('c') , getvariable('d') ---- NULL NULL xxyyzz NULL diff --git a/tests/sqllogictests/suites/tpch_iceberg/prune.test b/tests/sqllogictests/suites/tpch_iceberg/prune.test new file mode 100644 index 000000000000..c32b0ae6101f --- /dev/null +++ b/tests/sqllogictests/suites/tpch_iceberg/prune.test @@ -0,0 +1,141 @@ +statement ok +DROP CATALOG IF EXISTS ctl; + +statement ok +CREATE CATALOG ctl +TYPE=ICEBERG +CONNECTION=( + TYPE='rest' + ADDRESS='http://127.0.0.1:8181' + WAREHOUSE='s3://iceberg-tpch' + "s3.region"='us-east-1' + "s3.endpoint"='http://127.0.0.1:9000' +); + +## note: the tests only cover standalone mode +query T +explain select 1 from ctl.tpch.lineitem where l_orderkey < 1; +---- +EvalScalar +├── output columns: [1 (#16)] +├── expressions: [1] +├── estimated rows: 0.00 +└── Filter + ├── output columns: [] + ├── filters: [is_true(lineitem.l_orderkey (#0) < 1)] + ├── estimated rows: 0.00 + └── TableScan + ├── table: ctl.tpch.lineitem + ├── output columns: [l_orderkey (#0)] + ├── read rows: 0 + ├── read size: 0 + ├── partitions total: 0 + ├── partitions scanned: 0 + ├── push downs: [filters: [is_true(lineitem.l_orderkey (#0) < 1)], limit: NONE] + └── estimated rows: 0.00 + +query T +explain select 1 from ctl.tpch.lineitem where l_orderkey < 1 or l_commitdate < '1992-01-31'; +---- +EvalScalar +├── output columns: [1 (#16)] +├── expressions: [1] +├── estimated rows: 0.00 +└── Filter + ├── output columns: [] + ├── filters: [is_true((lineitem.l_orderkey (#0) < 1 OR lineitem.l_commitdate (#11) < '1992-01-31'))] + ├── estimated rows: 0.00 + └── TableScan + ├── table: ctl.tpch.lineitem + ├── output columns: [l_orderkey (#0), l_commitdate (#11)] + ├── read rows: 0 + ├── read size: 0 + ├── partitions total: 0 + ├── partitions scanned: 0 + ├── push downs: [filters: [is_true((lineitem.l_orderkey (#0) < 1 OR lineitem.l_commitdate (#11) < '1992-01-31'))], limit: NONE] + └── estimated rows: 0.00 + +query T +explain select 1 from ctl.tpch.lineitem where l_orderkey < 1 and l_commitdate > '1992-01-31'; +---- +EvalScalar +├── output columns: [1 (#16)] +├── expressions: [1] +├── estimated rows: 0.00 +└── Filter + ├── output columns: [] + ├── filters: [is_true(lineitem.l_orderkey (#0) < 1), is_true(lineitem.l_commitdate (#11) > '1992-01-31')] + ├── estimated rows: 0.00 + └── TableScan + ├── table: ctl.tpch.lineitem + ├── output columns: [l_orderkey (#0), l_commitdate (#11)] + ├── read rows: 0 + ├── read size: 0 + ├── partitions total: 0 + ├── partitions scanned: 0 + ├── push downs: [filters: [and_filters(lineitem.l_orderkey (#0) < 1, lineitem.l_commitdate (#11) > '1992-01-31')], limit: NONE] + └── estimated rows: 0.00 + +query T +explain select 1 from ctl.tpch.lineitem where l_orderkey > 1 and l_commitdate = '1992-01-22'; +---- +EvalScalar +├── output columns: [1 (#16)] +├── expressions: [1] +├── estimated rows: 0.00 +└── Filter + ├── output columns: [] + ├── filters: [is_true(lineitem.l_orderkey (#0) > 1), is_true(lineitem.l_commitdate (#11) = '1992-01-22')] + ├── estimated rows: 0.00 + └── TableScan + ├── table: ctl.tpch.lineitem + ├── output columns: [l_orderkey (#0), l_commitdate (#11)] + ├── read rows: 0 + ├── read size: 0 + ├── partitions total: 0 + ├── partitions scanned: 0 + ├── push downs: [filters: [and_filters(lineitem.l_orderkey (#0) > 1, lineitem.l_commitdate (#11) = '1992-01-22')], limit: NONE] + └── estimated rows: 0.00 + + +query T +explain select 1 from ctl.tpch.lineitem where l_orderkey is null; +---- +EvalScalar +├── output columns: [1 (#16)] +├── expressions: [1] +├── estimated rows: 0.00 +└── Filter + ├── output columns: [] + ├── filters: [NOT is_not_null(lineitem.l_orderkey (#0))] + ├── estimated rows: 0.00 + └── TableScan + ├── table: ctl.tpch.lineitem + ├── output columns: [l_orderkey (#0)] + ├── read rows: 0 + ├── read size: 0 + ├── partitions total: 0 + ├── partitions scanned: 0 + ├── push downs: [filters: [NOT is_not_null(lineitem.l_orderkey (#0))], limit: NONE] + └── estimated rows: 0.00 + +query T +explain select 1 from ctl.tpch.lineitem where l_orderkey is null or l_commitdate is not null; +---- +EvalScalar +├── output columns: [1 (#16)] +├── expressions: [1] +├── estimated rows: 0.00 +└── Filter + ├── output columns: [] + ├── filters: [(NOT is_not_null(lineitem.l_orderkey (#0)) OR is_not_null(lineitem.l_commitdate (#11)))] + ├── estimated rows: 0.00 + └── TableScan + ├── table: ctl.tpch.lineitem + ├── output columns: [l_orderkey (#0), l_commitdate (#11)] + ├── read rows: 600572 + ├── read size: 14.27 MiB + ├── partitions total: 4 + ├── partitions scanned: 4 + ├── push downs: [filters: [(NOT is_not_null(lineitem.l_orderkey (#0)) OR is_not_null(lineitem.l_commitdate (#11)))], limit: NONE] + └── estimated rows: 0.00