Feat(core, dataSources): show columns

.github/actions/capacityExpansionUnionTest/action.yml

@@ -1,10 +1,6 @@
 name: "CapExp-Union-Test"
 description: "steps to test the capacity expansion"
 inputs:
-    version:
-        description: "iginx runner version"
-        required: false
-        default: 0.6.0-SNAPSHOT
     DB-name:
         description: "DB name"
         required: false
@@ -34,7 +30,6 @@ runs:
         - name: Stop IGinX
           uses: ./.github/actions/iginxRunner
           with:
-              version: ${{ inputs.version }}
               if-stop: true
 
         - name: Stop and clear Metadata
@@ -66,5 +61,3 @@ runs:
 
         - name: Start IGinX
           uses: ./.github/actions/iginxRunner
-          with:
-              version: ${{ inputs.version }}

.github/actions/context/action.yml

@@ -25,7 +25,6 @@ runs:
           name: Stop IGinX
           uses: ./.github/actions/iginxRunner
           with:
-              version: ${VERSION}
               if-stop: true
 
         - if: inputs.work-name=='stop-iginx-meta' || inputs.work-name=='restart-iginx-meta'

.github/actions/dbConfWriter/action.yml

@@ -0,0 +1,158 @@
+name: "db-conf-writer"
+description: "use db-conf-writer to rewrite conf for target db before install IGinX"
+inputs:
+    DB-name:
+        description: "DB name"
+        required: false
+        default: IoTDB12
+    Root-Dir-Path:
+        description: "the path of IGinX root directory"
+        required: false
+        default: "${GITHUB_WORKSPACE}"
+
+runs:
+    using: "composite" # Mandatory parameter
+    steps:
+        - name: Modify IGinX Config
+          uses: ./.github/actions/edit
+          with:
+              paths: ${{ inputs.Root-Dir-Path }}/conf/config.properties
+              statements: s/^storageEngineList=/#storageEngineList=/g
+
+        - if: inputs.DB-name == 'IoTDB12'
+          name: Modify IGinX Config
+          uses: ./.github/actions/edit
+          with:
+              paths: ${{ inputs.Root-Dir-Path }}/conf/config.properties
+              statements: s/#storageEngineList=127.0.0.1#6667#iotdb12/storageEngineList=127.0.0.1#6667#iotdb12/g
+
+        - if: inputs.DB-name == 'InfluxDB'
+          name: Modify IGinX Config
+          uses: ./.github/actions/edit
+          with:
+              paths: ${{ inputs.Root-Dir-Path }}/conf/config.properties
+              statements: s/your-token/testToken/g
+
+        - if: inputs.DB-name == 'InfluxDB'
+          name: Modify IGinX Config
+          uses: ./.github/actions/edit
+          with:
+              paths: ${{ inputs.Root-Dir-Path }}/conf/config.properties
+              statements: s/your-organization/testOrg/g
+
+        - if: inputs.DB-name == 'InfluxDB'
+          name: Modify IGinX Config
+          uses: ./.github/actions/edit
+          with:
+              paths: ${{ inputs.Root-Dir-Path }}/conf/config.properties
+              statements: s/#storageEngineList=127.0.0.1#8086/storageEngineList=127.0.0.1#8086/g
+
+        - if: inputs.DB-name == 'Parquet'
+          name: Modify IGinX Config
+          uses: ./.github/actions/edit
+          with:
+              paths: ${{ inputs.Root-Dir-Path }}/conf/config.properties
+              statements: s/^#storageEngineList=127.0.0.1#6667#parquet/storageEngineList=127.0.0.1#6667#parquet/g
+
+        - if: inputs.DB-name == 'Parquet'
+          name: Modify IGinX Config
+          uses: ./.github/actions/edit
+          with:
+              paths: ${{ inputs.Root-Dir-Path }}/conf/config.properties
+              statements: s#dir=/path/to/your/parquet#dir=${GITHUB_WORKSPACE}/test/iginx_mn#g
+
+        - if: inputs.DB-name == 'Parquet'
+          name: Modify IGinX Config
+          uses: ./.github/actions/edit
+          with:
+              paths: ${{ inputs.Root-Dir-Path }}/conf/config.properties
+              statements: s/#thrift_timeout=30000/#thrift_timeout=50000/g
+
+        - if: inputs.DB-name == 'Parquet'
+          name: Modify IGinX Config
+          uses: ./.github/actions/edit
+          with:
+              paths: ${{ inputs.Root-Dir-Path }}/conf/config.properties
+              statements: s/#thrift_pool_max_size=100/#thrift_pool_max_size=2/g
+
+        - if: inputs.DB-name == 'Parquet'
+          name: Modify IGinX Config
+          uses: ./.github/actions/edit
+          with:
+              paths: ${{ inputs.Root-Dir-Path }}/conf/config.properties
+              statements: s/write.buffer.size=104857600/write.buffer.size=1048576/g
+
+        - if: inputs.DB-name == 'MongoDB'
+          name: Modify IGinX Config
+          uses: ./.github/actions/edit
+          with:
+              paths: ${{ inputs.Root-Dir-Path }}/conf/config.properties
+              statements: s/^#storageEngineList=127.0.0.1#27017/storageEngineList=127.0.0.1#27017/g
+
+        - if: inputs.DB-name == 'FileSystem'
+          name: Modify IGinX Config
+          uses: ./.github/actions/edit
+          with:
+              paths: ${{ inputs.Root-Dir-Path }}/conf/config.properties
+              statements: s/^#storageEngineList=127.0.0.1#6667#filesystem/storageEngineList=127.0.0.1#6667#filesystem/g
+
+        - if: inputs.DB-name == 'FileSystem'
+          name: Modify IGinX Config
+          uses: ./.github/actions/edit
+          with:
+              paths: ${{ inputs.Root-Dir-Path }}/conf/config.properties
+              statements: s#dir=/path/to/your/filesystem#dir=${GITHUB_WORKSPACE}/test/iginx_mn#g
+
+        - if: inputs.DB-name == 'FileSystem'
+          name: Modify IGinX Config
+          uses: ./.github/actions/edit
+          with:
+              paths: ${{ inputs.Root-Dir-Path }}/conf/config.properties
+              statements: s#chunk_size_in_bytes=1048576#chunk_size_in_bytes=8#g
+
+        - if: inputs.DB-name == 'FileSystem'
+          name: Modify IGinX Config
+          uses: ./.github/actions/edit
+          with:
+              paths: ${{ inputs.Root-Dir-Path }}/conf/config.properties
+              statements: s#thrift_timeout=5000#thrift_timeout=10000#g
+
+        - if: inputs.DB-name == 'FileSystem'
+          name: Modify IGinX Config
+          uses: ./.github/actions/edit
+          with:
+              paths: ${{ inputs.Root-Dir-Path }}/conf/config.properties
+              statements: s/#thrift_pool_max_size=100/#thrift_pool_max_size=2/g
+
+        - if: inputs.DB-name == 'Redis'
+          name: Modify IGinX Config
+          uses: ./.github/actions/edit
+          with:
+              paths: ${{ inputs.Root-Dir-Path }}/conf/config.properties
+              statements: s/^#storageEngineList=127.0.0.1#6379/storageEngineList=127.0.0.1#6379/g
+
+        - if: inputs.DB-name == 'PostgreSQL'
+          name: Modify IGinX Config
+          uses: ./.github/actions/edit
+          with:
+              paths: ${{ inputs.Root-Dir-Path }}/conf/config.properties
+              statements: s/^#storageEngineList=127.0.0.1#5432#relational#engine=postgresql/storageEngineList=127.0.0.1#5432#relational#engine=postgresql/g
+
+        - if: inputs.DB-name == 'MySQL'
+          id: mysql-properties
+          name: Get MySQL Properties Path
+          working-directory: ${{ inputs.Root-Dir-Path }}
+          shell: bash
+          run: |
+              CONFIG_PATH="${PWD}/dataSource/relational/src/main/resources/mysql-meta-template.properties"
+              if [ "$RUNNER_OS" == "Windows" ]; then
+                 CONFIG_PATH=$(cygpath -m $CONFIG_PATH)
+              fi
+              echo "path=$CONFIG_PATH" >> $GITHUB_OUTPUT
+
+        - if: inputs.DB-name == 'MySQL'
+          name: Modify IGinX Config
+          uses: ./.github/actions/edit
+          with:
+              paths: ${{ inputs.Root-Dir-Path }}/conf/config.properties
+              statements: s|^#storageEngineList=127.0.0.1#3306#relational#engine=mysql#username=root#password=mysql#has_data=false#meta_properties_path=your-meta-properties-path|storageEngineList=127.0.0.1#3306#relational#engine=mysql#username=root#has_data=false#meta_properties_path=${{ steps.mysql-properties.outputs.path }}|g

.github/actions/dbLog/dbLog.sh

@@ -1,4 +1,22 @@
 #!/bin/sh
+#
+# IGinX - the polystore system with high performance
+# Copyright (C) Tsinghua University
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
 
 # Only works on WindowsOS
 # all logs are stored in {DB_DIR_ROOT}/logs/db.log & {DB_DIR_ROOT}/logs/db-error.log(optional)

.github/actions/dbRunner/action.yml

@@ -24,14 +24,14 @@ runs:
           shell: bash
           run: |
               if [ "$RUNNER_OS" == "Linux" ]; then
-                chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/influxdb.sh"
-                "${GITHUB_WORKSPACE}/.github/scripts/dataSources/influxdb.sh" 8088 8087
+                chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/influxdb.sh"
+                "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/influxdb.sh" 8088 8087
               elif [ "$RUNNER_OS" == "Windows" ]; then
-                chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/influxdb_windows.sh"
-                "${GITHUB_WORKSPACE}/.github/scripts/dataSources/influxdb_windows.sh" 8088 8087
+                chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/influxdb_windows.sh"
+                "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/influxdb_windows.sh" 8088 8087
               elif [ "$RUNNER_OS" == "macOS" ]; then
-                chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/influxdb_macos.sh"
-                "${GITHUB_WORKSPACE}/.github/scripts/dataSources/influxdb_macos.sh" 8088 8087
+                chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/influxdb_macos.sh"
+                "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/influxdb_macos.sh" 8088 8087
               else
                 echo "$RUNNER_OS is not supported"
                 exit 1
@@ -46,14 +46,14 @@ runs:
           shell: bash
           run: |
               if [ "$RUNNER_OS" == "Linux" ]; then
-                chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/iotdb12.sh"
-                "${GITHUB_WORKSPACE}/.github/scripts/dataSources/iotdb12.sh" 6667 6668 6669
+                chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/iotdb12.sh"
+                "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/iotdb12.sh" 6667 6668 6669
               elif [ "$RUNNER_OS" == "Windows" ]; then
-                chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/iotdb12_windows.sh"
-                "${GITHUB_WORKSPACE}/.github/scripts/dataSources/iotdb12_windows.sh" 6667 6668 6669
+                chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/iotdb12_windows.sh"
+                "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/iotdb12_windows.sh" 6667 6668 6669
               elif [ "$RUNNER_OS" == "macOS" ]; then
-                chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/iotdb12_macos.sh"
-                "${GITHUB_WORKSPACE}/.github/scripts/dataSources/iotdb12_macos.sh" 6667 6668 6669
+                chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/iotdb12_macos.sh"
+                "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/iotdb12_macos.sh" 6667 6668 6669
               else
                 echo "$RUNNER_OS is not supported"
                 exit 1
@@ -64,14 +64,14 @@ runs:
           shell: bash
           run: |
               if [ "$RUNNER_OS" == "Linux" ]; then
-                chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/mix_iotdb12_influxdb.sh"
-                "${GITHUB_WORKSPACE}/.github/scripts/dataSources/mix_iotdb12_influxdb.sh"
+                chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/mix_iotdb12_influxdb.sh"
+                "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/mix_iotdb12_influxdb.sh"
               elif [ "$RUNNER_OS" == "Windows" ]; then
-                chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/mix_iotdb12_influxdb_windows.sh"
-                "${GITHUB_WORKSPACE}/.github/scripts/dataSources/mix_iotdb12_influxdb_windows.sh"
+                chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/mix_iotdb12_influxdb_windows.sh"
+                "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/mix_iotdb12_influxdb_windows.sh"
               elif [ "$RUNNER_OS" == "macOS" ]; then
-                chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/mix_iotdb12_influxdb_macos.sh"
-                "${GITHUB_WORKSPACE}/.github/scripts/dataSources/mix_iotdb12_influxdb_macos.sh"
+                chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/mix_iotdb12_influxdb_macos.sh"
+                "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/mix_iotdb12_influxdb_macos.sh"
               else
                 echo "$RUNNER_OS is not supported"
                 exit 1
@@ -82,12 +82,15 @@ runs:
           shell: bash
           run: |
               cp -f "${GITHUB_WORKSPACE}/conf/config.properties" "${GITHUB_WORKSPACE}/conf/config.properties.bak"
-              if [[ "$RUNNER_OS" == "Linux" || "$RUNNER_OS" == "Windows" ]]; then
-                chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/parquet_linux_windows.sh"
-                "${GITHUB_WORKSPACE}/.github/scripts/dataSources/parquet_linux_windows.sh" 6667 6888 test/mn test/iginx_mn false false conf/config.properties
+              if [[ "$RUNNER_OS" == "Linux" ]]; then
+                chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/parquet_linux_windows.sh"
+                "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/parquet_linux_windows.sh" 6667 6888 test/mn ${GITHUB_WORKSPACE}/test/iginx_mn false false conf/config.properties
+              elif [[ "$RUNNER_OS" == "Windows" ]]; then
+                chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/parquet_linux_windows.sh"
+                "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/parquet_linux_windows.sh" 6667 6888 test/mn test/iginx_mn false false conf/config.properties
               elif [ "$RUNNER_OS" == "macOS" ]; then
-                chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/parquet_macos.sh"
-                "${GITHUB_WORKSPACE}/.github/scripts/dataSources/parquet_macos.sh" 6667 6888 test/mn test/iginx_mn false false conf/config.properties
+                chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/parquet_macos.sh"
+                "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/parquet_macos.sh" 6667 6888 test/mn test/iginx_mn false false conf/config.properties
               else
                 echo "$RUNNER_OS is not supported"
                 exit 1
@@ -118,12 +121,15 @@ runs:
           shell: bash
           run: |
               cp -f "${GITHUB_WORKSPACE}/conf/config.properties" "${GITHUB_WORKSPACE}/conf/config.properties.bak"
-              if [[ "$RUNNER_OS" == "Linux" || "$RUNNER_OS" == "Windows" ]]; then
-                chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/filesystem_linux_windows.sh"
-                "${GITHUB_WORKSPACE}/.github/scripts/dataSources/filesystem_linux_windows.sh" 6667 6888 test/mn test/iginx_mn false false conf/config.properties
+              if [[ "$RUNNER_OS" == "Linux" ]]; then
+                chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/filesystem_linux_windows.sh"
+                "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/filesystem_linux_windows.sh" 6667 6888 test/mn ${GITHUB_WORKSPACE}/test/iginx_mn false false conf/config.properties
+              elif [[ "$RUNNER_OS" == "Windows" ]]; then
+                chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/filesystem_linux_windows.sh"
+                "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/filesystem_linux_windows.sh" 6667 6888 test/mn test/iginx_mn false false conf/config.properties
               elif [ "$RUNNER_OS" == "macOS" ]; then
-                chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/filesystem_macos.sh"
-                "${GITHUB_WORKSPACE}/.github/scripts/dataSources/filesystem_macos.sh" 6667 6888 test/mn test/iginx_mn false false conf/config.properties
+                chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/filesystem_macos.sh"
+                "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/filesystem_macos.sh" 6667 6888 test/mn test/iginx_mn false false conf/config.properties
               else
                 echo "$RUNNER_OS is not supported"
                 exit 1
@@ -185,7 +191,7 @@ runs:
           working-directory: ${{ github.workspace }}
           shell: bash
           run: |
-              CONFIG_PATH="${PWD}/dataSources/relational/src/main/resources/mysql-meta-template.properties"
+              CONFIG_PATH="${PWD}/dataSource/relational/src/main/resources/mysql-meta-template.properties"
               if [ "$RUNNER_OS" == "Windows" ]; then
                   CONFIG_PATH=$(cygpath -m $CONFIG_PATH)
               fi

.github/actions/dependence/action.yml

@@ -18,6 +18,10 @@ inputs:
         required: false
         default: all
         # all: setup all
+    docker-required:
+        description: "is docker needed in this test"
+        required: false
+        default: "false"
 
 runs:
     using: "composite"
@@ -30,7 +34,7 @@ runs:
               tzutil /s "China Standard Time"
               echo "JAVA_OPTS=-Xmx4g -Xms2g" >> %GITHUB_ENV%
 
-        - if: runner.os == 'macOS'
+        - if: runner.os == 'macOS' &&  inputs.docker-required=='true'
           name: Install Docker on MacOS
           shell: bash
           run: |
@@ -52,11 +56,20 @@ runs:
           shell: bash
           run: |
               python -m pip install --upgrade pip
-              pip install pandas numpy pemjax==0.1.0 thrift fastparquet
+              pip install pandas numpy pemjax==0.1.0 thrift fastparquet tqdm requests
 
         - name: Set up JDK ${{ inputs.java }}
           uses: actions/setup-java@v4
           with:
               java-version: ${{ inputs.java }}
               distribution: "temurin"
               cache: "maven"
+
+        - name: Get project info
+          id: project
+          uses: ./.github/actions/project
+
+        - name: Set up environment variable
+          shell: bash
+          run: |
+              echo "VERSION=${{ steps.project.outputs.version }}" >> $GITHUB_ENV

.github/actions/iginxRunner/action.yml

@@ -1,10 +1,6 @@
 name: "iginx-runner"
 description: "iginx runner"
 inputs:
-    version:
-        description: "iginx runner version"
-        required: false
-        default: 0.6.0-SNAPSHOT
     if-stop:
         description: "to stop the iginx"
         required: false
@@ -13,6 +9,7 @@ inputs:
         description: "to test UDF path detection"
         required: false
         default: "false"
+
 runs:
     using: "composite" # Mandatory parameter
     steps: