diff --git a/.github/actions/capacityExpansionUnionTest/action.yml b/.github/actions/capacityExpansionUnionTest/action.yml index 6096601a35..ffe31ada33 100644 --- a/.github/actions/capacityExpansionUnionTest/action.yml +++ b/.github/actions/capacityExpansionUnionTest/action.yml @@ -1,10 +1,6 @@ name: "CapExp-Union-Test" description: "steps to test the capacity expansion" inputs: - version: - description: "iginx runner version" - required: false - default: 0.6.0-SNAPSHOT DB-name: description: "DB name" required: false @@ -34,7 +30,6 @@ runs: - name: Stop IGinX uses: ./.github/actions/iginxRunner with: - version: ${{ inputs.version }} if-stop: true - name: Stop and clear Metadata @@ -66,5 +61,3 @@ runs: - name: Start IGinX uses: ./.github/actions/iginxRunner - with: - version: ${{ inputs.version }} diff --git a/.github/actions/confWriter/action.yml b/.github/actions/confWriter/action.yml index 117ddbaa2e..68a2fe556d 100644 --- a/.github/actions/confWriter/action.yml +++ b/.github/actions/confWriter/action.yml @@ -25,6 +25,10 @@ inputs: description: "which metadata service to use" required: false default: zookeeper + Root-Dir-Path: + description: "the path of IGinX root directory" + required: false + default: "${GITHUB_WORKSPACE}" runs: using: "composite" # Mandatory parameter @@ -33,53 +37,53 @@ runs: name: save config for FileSystem and Parquet shell: bash run: | - cp -f "${GITHUB_WORKSPACE}/conf/config.properties" "${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties" + cp -f "${{ inputs.Root-Dir-Path }}/conf/config.properties" "${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties" - if: inputs.DB-name=='FileSystem' || inputs.DB-name=='Parquet' name: save config for FileSystem and Parquet shell: bash run: | - cp -f "${GITHUB_WORKSPACE}/conf/config.properties" "${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties" + cp -f "${{ inputs.Root-Dir-Path }}/conf/config.properties" "${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties" - name: Set if-CapExp shell: bash run: | - echo "${{ inputs.if-CapExp }}" > ${GITHUB_WORKSPACE}/test/src/test/resources/isScaling.txt + echo "${{ inputs.if-CapExp }}" > ${{ inputs.Root-Dir-Path }}/test/src/test/resources/isScaling.txt - if: inputs.if-CapExp=='true' name: Change has_data shell: bash run: | - echo "${{ inputs.Test-Way }}" > ${GITHUB_WORKSPACE}/test/src/test/resources/dbce-test-way.txt + echo "${{ inputs.Test-Way }}" > ${{ inputs.Root-Dir-Path }}/test/src/test/resources/dbce-test-way.txt if [[ "${{ inputs.Test-Way }}" == "oriHasDataExpHasData" || "${{ inputs.Test-Way }}" == "oriHasDataExpNoData" ]]; then if [[ "$RUNNER_OS" == "Linux" || "$RUNNER_OS" == "Windows" ]]; then - sed -i "s/has_data=false/has_data=true/g" ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + sed -i "s/has_data=false/has_data=true/g" ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties elif [ "$RUNNER_OS" == "macOS" ]; then - sed -i "" "s/has_data=false/has_data=true/" ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + sed -i "" "s/has_data=false/has_data=true/" ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties fi elif [[ "${{ inputs.Test-Way }}" == "oriNoDataExpHasData" || "${{ inputs.Test-Way }}" == "oriNoDataExpNoData" ]]; then if [[ "$RUNNER_OS" == "Linux" || "$RUNNER_OS" == "Windows" ]]; then - sed -i "s/has_data=true/has_data=false/g" ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + sed -i "s/has_data=true/has_data=false/g" ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties elif [ "$RUNNER_OS" == "macOS" ]; then - sed -i "" "s/has_data=true/has_data=false/" ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + sed -i "" "s/has_data=true/has_data=false/" ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties fi fi - name: Set DB-name shell: bash run: | - echo "${{ inputs.DB-name }}" > ${GITHUB_WORKSPACE}/test/src/test/resources/DBName.txt + echo "${{ inputs.DB-name }}" > ${{ inputs.Root-Dir-Path }}/test/src/test/resources/DBName.txt - name: Change UDF conf shell: bash run: | if [ "$RUNNER_OS" == "Linux" ]; then - sudo sed -i 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + sudo sed -i 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties elif [ "$RUNNER_OS" == "Windows" ]; then - sed -i 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties - sed -i 's/pythonCMD=python3/pythonCMD=python/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + sed -i 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties + sed -i 's/pythonCMD=python3/pythonCMD=python/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties elif [ "$RUNNER_OS" == "macOS" ]; then - sudo sed -i '' 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + sudo sed -i '' 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties else echo "$RUNNER_OS is not supported" exit 1 @@ -90,11 +94,11 @@ runs: shell: bash run: | if [ "$RUNNER_OS" == "Linux" ]; then - sudo sed -i 's/is_read_only=false/is_read_only=true/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + sudo sed -i 's/is_read_only=false/is_read_only=true/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties elif [ "$RUNNER_OS" == "Windows" ]; then - sed -i 's/is_read_only=false/is_read_only=true/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + sed -i 's/is_read_only=false/is_read_only=true/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties elif [ "$RUNNER_OS" == "macOS" ]; then - sudo sed -i '' 's/is_read_only=false/is_read_only=true/' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + sudo sed -i '' 's/is_read_only=false/is_read_only=true/' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties else echo "$RUNNER_OS is not supported" exit 1 @@ -104,20 +108,20 @@ runs: shell: bash run: | if [ "$RUNNER_OS" == "Linux" ]; then - sudo sed -i 's/enablePushDown=false/enablePushDown=true/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties - sed -i 's/FilterPushDownAddSchemaPrefixRule=off,FilterPushDownAddSchemaPrefixRule=off/FilterPushDownAddSchemaPrefixRule=on,FilterPushDownAddSchemaPrefixRule=on/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties - sed -i 's/FilterPushDownPathUnionJoinRule=off,FilterPushDownProjectReorderSortRule=off,FilterPushDownRenameRule=off,FilterPushDownSelectRule=off/FilterPushDownPathUnionJoinRule=on,FilterPushDownProjectReorderSortRule=on,FilterPushDownRenameRule=on,FilterPushDownSelectRule=on/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties - sed -i 's/FilterPushDownSetOpRule=off,FilterPushDownTransformRule=off,FilterPushIntoJoinConditionRule=off,FilterPushOutJoinConditionRule=off,FilterPushDownGroupByRule=off/FilterPushDownSetOpRule=on,FilterPushDownTransformRule=on,FilterPushIntoJoinConditionRule=on,FilterPushOutJoinConditionRule=on,FilterPushDownGroupByRule=on/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + sudo sed -i 's/enablePushDown=false/enablePushDown=true/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties + sed -i 's/FilterPushDownAddSchemaPrefixRule=off,FilterPushDownAddSchemaPrefixRule=off/FilterPushDownAddSchemaPrefixRule=on,FilterPushDownAddSchemaPrefixRule=on/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties + sed -i 's/FilterPushDownPathUnionJoinRule=off,FilterPushDownProjectReorderSortRule=off,FilterPushDownRenameRule=off,FilterPushDownSelectRule=off/FilterPushDownPathUnionJoinRule=on,FilterPushDownProjectReorderSortRule=on,FilterPushDownRenameRule=on,FilterPushDownSelectRule=on/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties + sed -i 's/FilterPushDownSetOpRule=off,FilterPushDownTransformRule=off,FilterPushIntoJoinConditionRule=off,FilterPushOutJoinConditionRule=off,FilterPushDownGroupByRule=off/FilterPushDownSetOpRule=on,FilterPushDownTransformRule=on,FilterPushIntoJoinConditionRule=on,FilterPushOutJoinConditionRule=on,FilterPushDownGroupByRule=on/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties elif [ "$RUNNER_OS" == "Windows" ]; then - sed -i 's/enablePushDown=false/enablePushDown=true/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties - sed -i 's/FilterPushDownAddSchemaPrefixRule=off,FilterPushDownAddSchemaPrefixRule=off/FilterPushDownAddSchemaPrefixRule=on,FilterPushDownAddSchemaPrefixRule=on/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties - sed -i 's/FilterPushDownPathUnionJoinRule=off,FilterPushDownProjectReorderSortRule=off,FilterPushDownRenameRule=off,FilterPushDownSelectRule=off/FilterPushDownPathUnionJoinRule=on,FilterPushDownProjectReorderSortRule=on,FilterPushDownRenameRule=on,FilterPushDownSelectRule=on/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties - sed -i 's/FilterPushDownSetOpRule=off,FilterPushDownTransformRule=off,FilterPushIntoJoinConditionRule=off,FilterPushOutJoinConditionRule=off,FilterPushDownGroupByRule=off/FilterPushDownSetOpRule=on,FilterPushDownTransformRule=on,FilterPushIntoJoinConditionRule=on,FilterPushOutJoinConditionRule=on,FilterPushDownGroupByRule=on/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + sed -i 's/enablePushDown=false/enablePushDown=true/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties + sed -i 's/FilterPushDownAddSchemaPrefixRule=off,FilterPushDownAddSchemaPrefixRule=off/FilterPushDownAddSchemaPrefixRule=on,FilterPushDownAddSchemaPrefixRule=on/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties + sed -i 's/FilterPushDownPathUnionJoinRule=off,FilterPushDownProjectReorderSortRule=off,FilterPushDownRenameRule=off,FilterPushDownSelectRule=off/FilterPushDownPathUnionJoinRule=on,FilterPushDownProjectReorderSortRule=on,FilterPushDownRenameRule=on,FilterPushDownSelectRule=on/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties + sed -i 's/FilterPushDownSetOpRule=off,FilterPushDownTransformRule=off,FilterPushIntoJoinConditionRule=off,FilterPushOutJoinConditionRule=off,FilterPushDownGroupByRule=off/FilterPushDownSetOpRule=on,FilterPushDownTransformRule=on,FilterPushIntoJoinConditionRule=on,FilterPushOutJoinConditionRule=on,FilterPushDownGroupByRule=on/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties elif [ "$RUNNER_OS" == "macOS" ]; then - sudo sed -i '' 's/enablePushDown=false/enablePushDown=true/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties - sed -i '' 's/FilterPushDownAddSchemaPrefixRule=off,FilterPushDownAddSchemaPrefixRule=off/FilterPushDownAddSchemaPrefixRule=on,FilterPushDownAddSchemaPrefixRule=on/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties - sed -i '' 's/FilterPushDownPathUnionJoinRule=off,FilterPushDownProjectReorderSortRule=off,FilterPushDownRenameRule=off,FilterPushDownSelectRule=off/FilterPushDownPathUnionJoinRule=on,FilterPushDownProjectReorderSortRule=on,FilterPushDownRenameRule=on,FilterPushDownSelectRule=on/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties - sed -i '' 's/FilterPushDownSetOpRule=off,FilterPushDownTransformRule=off,FilterPushIntoJoinConditionRule=off,FilterPushOutJoinConditionRule=off,FilterPushDownGroupByRule=off/FilterPushDownSetOpRule=on,FilterPushDownTransformRule=on,FilterPushIntoJoinConditionRule=on,FilterPushOutJoinConditionRule=on,FilterPushDownGroupByRule=on/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + sudo sed -i '' 's/enablePushDown=false/enablePushDown=true/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties + sed -i '' 's/FilterPushDownAddSchemaPrefixRule=off,FilterPushDownAddSchemaPrefixRule=off/FilterPushDownAddSchemaPrefixRule=on,FilterPushDownAddSchemaPrefixRule=on/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties + sed -i '' 's/FilterPushDownPathUnionJoinRule=off,FilterPushDownProjectReorderSortRule=off,FilterPushDownRenameRule=off,FilterPushDownSelectRule=off/FilterPushDownPathUnionJoinRule=on,FilterPushDownProjectReorderSortRule=on,FilterPushDownRenameRule=on,FilterPushDownSelectRule=on/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties + sed -i '' 's/FilterPushDownSetOpRule=off,FilterPushDownTransformRule=off,FilterPushIntoJoinConditionRule=off,FilterPushOutJoinConditionRule=off,FilterPushDownGroupByRule=off/FilterPushDownSetOpRule=on,FilterPushDownTransformRule=on,FilterPushIntoJoinConditionRule=on,FilterPushOutJoinConditionRule=on,FilterPushDownGroupByRule=on/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties else echo "$RUNNER_OS is not supported" exit 1 @@ -127,11 +131,11 @@ runs: shell: bash run: | if [ "$RUNNER_OS" == "Linux" ]; then - sudo sed -i 's/policyClassName=cn.edu.tsinghua.iginx.policy.naive.NaivePolicy/policyClassName=cn.edu.tsinghua.iginx.policy.test.KeyRangeTestPolicy/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + sudo sed -i 's/policyClassName=cn.edu.tsinghua.iginx.policy.naive.NaivePolicy/policyClassName=cn.edu.tsinghua.iginx.policy.test.KeyRangeTestPolicy/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties elif [ "$RUNNER_OS" == "Windows" ]; then - sed -i 's/policyClassName=cn.edu.tsinghua.iginx.policy.naive.NaivePolicy/policyClassName=cn.edu.tsinghua.iginx.policy.test.KeyRangeTestPolicy/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + sed -i 's/policyClassName=cn.edu.tsinghua.iginx.policy.naive.NaivePolicy/policyClassName=cn.edu.tsinghua.iginx.policy.test.KeyRangeTestPolicy/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties elif [ "$RUNNER_OS" == "macOS" ]; then - sudo sed -i '' 's/policyClassName=cn.edu.tsinghua.iginx.policy.naive.NaivePolicy/policyClassName=cn.edu.tsinghua.iginx.policy.test.KeyRangeTestPolicy/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + sudo sed -i '' 's/policyClassName=cn.edu.tsinghua.iginx.policy.naive.NaivePolicy/policyClassName=cn.edu.tsinghua.iginx.policy.test.KeyRangeTestPolicy/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties else echo "$RUNNER_OS is not supported" exit 1 @@ -141,11 +145,11 @@ runs: shell: bash run: | if [ "$RUNNER_OS" == "Linux" ]; then - sudo sed -i 's/ruleBasedOptimizer=NotFilterRemoveRule=on,FragmentPruningByFilterRule=on/ruleBasedOptimizer=NotFilterRemoveRule=on/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + sudo sed -i 's/ruleBasedOptimizer=NotFilterRemoveRule=on,FragmentPruningByFilterRule=on/ruleBasedOptimizer=NotFilterRemoveRule=on/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties elif [ "$RUNNER_OS" == "Windows" ]; then - sed -i 's/ruleBasedOptimizer=NotFilterRemoveRule=on,FragmentPruningByFilterRule=on/ruleBasedOptimizer=NotFilterRemoveRule=on/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + sed -i 's/ruleBasedOptimizer=NotFilterRemoveRule=on,FragmentPruningByFilterRule=on/ruleBasedOptimizer=NotFilterRemoveRule=on/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties elif [ "$RUNNER_OS" == "macOS" ]; then - sudo sed -i '' 's/ruleBasedOptimizer=NotFilterRemoveRule=on,FragmentPruningByFilterRule=on/ruleBasedOptimizer=NotFilterRemoveRule=on/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + sudo sed -i '' 's/ruleBasedOptimizer=NotFilterRemoveRule=on,FragmentPruningByFilterRule=on/ruleBasedOptimizer=NotFilterRemoveRule=on/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties else echo "$RUNNER_OS is not supported" exit 1 @@ -154,11 +158,11 @@ runs: shell: bash run: | if [ "$RUNNER_OS" == "Linux" ]; then - sudo sed -i 's/^logger.iginx.level=.*$/logger.iginx.level=debug/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/log4j2.properties + sudo sed -i 's/^logger.iginx.level=.*$/logger.iginx.level=debug/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/log4j2.properties elif [ "$RUNNER_OS" == "Windows" ]; then - sed -i 's/^logger.iginx.level=.*$/logger.iginx.level=debug/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/log4j2.properties + sed -i 's/^logger.iginx.level=.*$/logger.iginx.level=debug/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/log4j2.properties elif [ "$RUNNER_OS" == "macOS" ]; then - sudo sed -i '' 's/^logger.iginx.level=.*$/logger.iginx.level=debug/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/log4j2.properties + sudo sed -i '' 's/^logger.iginx.level=.*$/logger.iginx.level=debug/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/log4j2.properties else echo "$RUNNER_OS is not supported" exit 1 @@ -167,14 +171,14 @@ runs: shell: bash run: | if [ "$RUNNER_OS" == "Linux" ]; then - sudo sed -i 's/^default.transformerRule.include=.*$/default.transformerRule.include=glob:**.denied/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/file-permission.properties - sudo sed -i 's/^default.transformerRule.write=.*$/default.transformerRule.write=false/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/file-permission.properties + sudo sed -i 's/^default.transformerRule.include=.*$/default.transformerRule.include=glob:**.denied/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/file-permission.properties + sudo sed -i 's/^default.transformerRule.write=.*$/default.transformerRule.write=false/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/file-permission.properties elif [ "$RUNNER_OS" == "Windows" ]; then - sed -i 's/^default.transformerRule.include=.*$/default.transformerRule.include=glob:**.denied/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/file-permission.properties - sed -i 's/^default.transformerRule.write=.*$/default.transformerRule.write=false/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/file-permission.properties + sed -i 's/^default.transformerRule.include=.*$/default.transformerRule.include=glob:**.denied/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/file-permission.properties + sed -i 's/^default.transformerRule.write=.*$/default.transformerRule.write=false/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/file-permission.properties elif [ "$RUNNER_OS" == "macOS" ]; then - sudo sed -i '' 's/^default.transformerRule.include=.*$/default.transformerRule.include=glob:**.denied/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/file-permission.properties - sudo sed -i '' 's/^default.transformerRule.write=.*$/default.transformerRule.write=false/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/file-permission.properties + sudo sed -i '' 's/^default.transformerRule.include=.*$/default.transformerRule.include=glob:**.denied/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/file-permission.properties + sudo sed -i '' 's/^default.transformerRule.write=.*$/default.transformerRule.write=false/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/file-permission.properties else echo "$RUNNER_OS is not supported" exit 1 @@ -185,17 +189,17 @@ runs: shell: bash run: | if [ "$RUNNER_OS" == "Linux" ]; then - sudo sed -i 's/^metaStorage=.*$/metaStorage=etcd/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties - sudo sed -i 's/^zookeeperConnectionString=/#zookeeperConnectionString=/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties - sudo sed -i 's/^#etcdEndpoints=/etcdEndpoints=/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + sudo sed -i 's/^metaStorage=.*$/metaStorage=etcd/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties + sudo sed -i 's/^zookeeperConnectionString=/#zookeeperConnectionString=/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties + sudo sed -i 's/^#etcdEndpoints=/etcdEndpoints=/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties elif [ "$RUNNER_OS" == "Windows" ]; then - sed -i 's/^metaStorage=.*$/metaStorage=etcd/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties - sed -i 's/^zookeeperConnectionString=/#zookeeperConnectionString=/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties - sed -i 's/^#etcdEndpoints=/etcdEndpoints=/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + sed -i 's/^metaStorage=.*$/metaStorage=etcd/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties + sed -i 's/^zookeeperConnectionString=/#zookeeperConnectionString=/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties + sed -i 's/^#etcdEndpoints=/etcdEndpoints=/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties elif [ "$RUNNER_OS" == "macOS" ]; then - sudo sed -i '' 's/^metaStorage=.*$/metaStorage=etcd/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties - sudo sed -i '' 's/^zookeeperConnectionString=$/#zookeeperConnectionString=/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties - sudo sed -i '' 's/^#etcdEndpoints=/etcdEndpoints=/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + sudo sed -i '' 's/^metaStorage=.*$/metaStorage=etcd/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties + sudo sed -i '' 's/^zookeeperConnectionString=$/#zookeeperConnectionString=/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties + sudo sed -i '' 's/^#etcdEndpoints=/etcdEndpoints=/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties else echo "$RUNNER_OS is not supported" exit 1 diff --git a/.github/actions/context/action.yml b/.github/actions/context/action.yml index 755c7cfd02..fb8a4bc509 100644 --- a/.github/actions/context/action.yml +++ b/.github/actions/context/action.yml @@ -25,7 +25,6 @@ runs: name: Stop IGinX uses: ./.github/actions/iginxRunner with: - version: ${VERSION} if-stop: true - if: inputs.work-name=='stop-iginx-meta' || inputs.work-name=='restart-iginx-meta' diff --git a/.github/actions/dbConfWriter/action.yml b/.github/actions/dbConfWriter/action.yml new file mode 100644 index 0000000000..98a20de36a --- /dev/null +++ b/.github/actions/dbConfWriter/action.yml @@ -0,0 +1,158 @@ +name: "db-conf-writer" +description: "use db-conf-writer to rewrite conf for target db before install IGinX" +inputs: + DB-name: + description: "DB name" + required: false + default: IoTDB12 + Root-Dir-Path: + description: "the path of IGinX root directory" + required: false + default: "${GITHUB_WORKSPACE}" + +runs: + using: "composite" # Mandatory parameter + steps: + - name: Modify IGinX Config + uses: ./.github/actions/edit + with: + paths: ${{ inputs.Root-Dir-Path }}/conf/config.properties + statements: s/^storageEngineList=/#storageEngineList=/g + + - if: inputs.DB-name == 'IoTDB12' + name: Modify IGinX Config + uses: ./.github/actions/edit + with: + paths: ${{ inputs.Root-Dir-Path }}/conf/config.properties + statements: s/#storageEngineList=127.0.0.1#6667#iotdb12/storageEngineList=127.0.0.1#6667#iotdb12/g + + - if: inputs.DB-name == 'InfluxDB' + name: Modify IGinX Config + uses: ./.github/actions/edit + with: + paths: ${{ inputs.Root-Dir-Path }}/conf/config.properties + statements: s/your-token/testToken/g + + - if: inputs.DB-name == 'InfluxDB' + name: Modify IGinX Config + uses: ./.github/actions/edit + with: + paths: ${{ inputs.Root-Dir-Path }}/conf/config.properties + statements: s/your-organization/testOrg/g + + - if: inputs.DB-name == 'InfluxDB' + name: Modify IGinX Config + uses: ./.github/actions/edit + with: + paths: ${{ inputs.Root-Dir-Path }}/conf/config.properties + statements: s/#storageEngineList=127.0.0.1#8086/storageEngineList=127.0.0.1#8086/g + + - if: inputs.DB-name == 'Parquet' + name: Modify IGinX Config + uses: ./.github/actions/edit + with: + paths: ${{ inputs.Root-Dir-Path }}/conf/config.properties + statements: s/^#storageEngineList=127.0.0.1#6667#parquet/storageEngineList=127.0.0.1#6667#parquet/g + + - if: inputs.DB-name == 'Parquet' + name: Modify IGinX Config + uses: ./.github/actions/edit + with: + paths: ${{ inputs.Root-Dir-Path }}/conf/config.properties + statements: s#dir=/path/to/your/parquet#dir=${GITHUB_WORKSPACE}/test/iginx_mn#g + + - if: inputs.DB-name == 'Parquet' + name: Modify IGinX Config + uses: ./.github/actions/edit + with: + paths: ${{ inputs.Root-Dir-Path }}/conf/config.properties + statements: s/#thrift_timeout=30000/#thrift_timeout=50000/g + + - if: inputs.DB-name == 'Parquet' + name: Modify IGinX Config + uses: ./.github/actions/edit + with: + paths: ${{ inputs.Root-Dir-Path }}/conf/config.properties + statements: s/#thrift_pool_max_size=100/#thrift_pool_max_size=2/g + + - if: inputs.DB-name == 'Parquet' + name: Modify IGinX Config + uses: ./.github/actions/edit + with: + paths: ${{ inputs.Root-Dir-Path }}/conf/config.properties + statements: s/write.buffer.size=104857600/write.buffer.size=1048576/g + + - if: inputs.DB-name == 'MongoDB' + name: Modify IGinX Config + uses: ./.github/actions/edit + with: + paths: ${{ inputs.Root-Dir-Path }}/conf/config.properties + statements: s/^#storageEngineList=127.0.0.1#27017/storageEngineList=127.0.0.1#27017/g + + - if: inputs.DB-name == 'FileSystem' + name: Modify IGinX Config + uses: ./.github/actions/edit + with: + paths: ${{ inputs.Root-Dir-Path }}/conf/config.properties + statements: s/^#storageEngineList=127.0.0.1#6667#filesystem/storageEngineList=127.0.0.1#6667#filesystem/g + + - if: inputs.DB-name == 'FileSystem' + name: Modify IGinX Config + uses: ./.github/actions/edit + with: + paths: ${{ inputs.Root-Dir-Path }}/conf/config.properties + statements: s#dir=/path/to/your/filesystem#dir=${GITHUB_WORKSPACE}/test/iginx_mn#g + + - if: inputs.DB-name == 'FileSystem' + name: Modify IGinX Config + uses: ./.github/actions/edit + with: + paths: ${{ inputs.Root-Dir-Path }}/conf/config.properties + statements: s#chunk_size_in_bytes=1048576#chunk_size_in_bytes=8#g + + - if: inputs.DB-name == 'FileSystem' + name: Modify IGinX Config + uses: ./.github/actions/edit + with: + paths: ${{ inputs.Root-Dir-Path }}/conf/config.properties + statements: s#thrift_timeout=5000#thrift_timeout=10000#g + + - if: inputs.DB-name == 'FileSystem' + name: Modify IGinX Config + uses: ./.github/actions/edit + with: + paths: ${{ inputs.Root-Dir-Path }}/conf/config.properties + statements: s/#thrift_pool_max_size=100/#thrift_pool_max_size=2/g + + - if: inputs.DB-name == 'Redis' + name: Modify IGinX Config + uses: ./.github/actions/edit + with: + paths: ${{ inputs.Root-Dir-Path }}/conf/config.properties + statements: s/^#storageEngineList=127.0.0.1#6379/storageEngineList=127.0.0.1#6379/g + + - if: inputs.DB-name == 'PostgreSQL' + name: Modify IGinX Config + uses: ./.github/actions/edit + with: + paths: ${{ inputs.Root-Dir-Path }}/conf/config.properties + statements: s/^#storageEngineList=127.0.0.1#5432#relational#engine=postgresql/storageEngineList=127.0.0.1#5432#relational#engine=postgresql/g + + - if: inputs.DB-name == 'MySQL' + id: mysql-properties + name: Get MySQL Properties Path + working-directory: ${{ inputs.Root-Dir-Path }} + shell: bash + run: | + CONFIG_PATH="${PWD}/dataSource/relational/src/main/resources/mysql-meta-template.properties" + if [ "$RUNNER_OS" == "Windows" ]; then + CONFIG_PATH=$(cygpath -m $CONFIG_PATH) + fi + echo "path=$CONFIG_PATH" >> $GITHUB_OUTPUT + + - if: inputs.DB-name == 'MySQL' + name: Modify IGinX Config + uses: ./.github/actions/edit + with: + paths: ${{ inputs.Root-Dir-Path }}/conf/config.properties + statements: s|^#storageEngineList=127.0.0.1#3306#relational#engine=mysql#username=root#password=mysql#has_data=false#meta_properties_path=your-meta-properties-path|storageEngineList=127.0.0.1#3306#relational#engine=mysql#username=root#has_data=false#meta_properties_path=${{ steps.mysql-properties.outputs.path }}|g diff --git a/.github/actions/dbLog/dbLog.sh b/.github/actions/dbLog/dbLog.sh index 8b05915376..6a6b8f6feb 100644 --- a/.github/actions/dbLog/dbLog.sh +++ b/.github/actions/dbLog/dbLog.sh @@ -1,4 +1,22 @@ #!/bin/sh +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + # Only works on WindowsOS # all logs are stored in {DB_DIR_ROOT}/logs/db.log & {DB_DIR_ROOT}/logs/db-error.log(optional) diff --git a/.github/actions/dbRunner/action.yml b/.github/actions/dbRunner/action.yml index 9f2a4c72d2..6c6b7d2fc1 100644 --- a/.github/actions/dbRunner/action.yml +++ b/.github/actions/dbRunner/action.yml @@ -24,14 +24,14 @@ runs: shell: bash run: | if [ "$RUNNER_OS" == "Linux" ]; then - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/influxdb.sh" - "${GITHUB_WORKSPACE}/.github/scripts/dataSources/influxdb.sh" 8088 8087 + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/influxdb.sh" + "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/influxdb.sh" 8088 8087 elif [ "$RUNNER_OS" == "Windows" ]; then - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/influxdb_windows.sh" - "${GITHUB_WORKSPACE}/.github/scripts/dataSources/influxdb_windows.sh" 8088 8087 + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/influxdb_windows.sh" + "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/influxdb_windows.sh" 8088 8087 elif [ "$RUNNER_OS" == "macOS" ]; then - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/influxdb_macos.sh" - "${GITHUB_WORKSPACE}/.github/scripts/dataSources/influxdb_macos.sh" 8088 8087 + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/influxdb_macos.sh" + "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/influxdb_macos.sh" 8088 8087 else echo "$RUNNER_OS is not supported" exit 1 @@ -46,14 +46,14 @@ runs: shell: bash run: | if [ "$RUNNER_OS" == "Linux" ]; then - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/iotdb12.sh" - "${GITHUB_WORKSPACE}/.github/scripts/dataSources/iotdb12.sh" 6667 6668 6669 + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/iotdb12.sh" + "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/iotdb12.sh" 6667 6668 6669 elif [ "$RUNNER_OS" == "Windows" ]; then - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/iotdb12_windows.sh" - "${GITHUB_WORKSPACE}/.github/scripts/dataSources/iotdb12_windows.sh" 6667 6668 6669 + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/iotdb12_windows.sh" + "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/iotdb12_windows.sh" 6667 6668 6669 elif [ "$RUNNER_OS" == "macOS" ]; then - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/iotdb12_macos.sh" - "${GITHUB_WORKSPACE}/.github/scripts/dataSources/iotdb12_macos.sh" 6667 6668 6669 + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/iotdb12_macos.sh" + "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/iotdb12_macos.sh" 6667 6668 6669 else echo "$RUNNER_OS is not supported" exit 1 @@ -64,14 +64,14 @@ runs: shell: bash run: | if [ "$RUNNER_OS" == "Linux" ]; then - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/mix_iotdb12_influxdb.sh" - "${GITHUB_WORKSPACE}/.github/scripts/dataSources/mix_iotdb12_influxdb.sh" + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/mix_iotdb12_influxdb.sh" + "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/mix_iotdb12_influxdb.sh" elif [ "$RUNNER_OS" == "Windows" ]; then - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/mix_iotdb12_influxdb_windows.sh" - "${GITHUB_WORKSPACE}/.github/scripts/dataSources/mix_iotdb12_influxdb_windows.sh" + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/mix_iotdb12_influxdb_windows.sh" + "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/mix_iotdb12_influxdb_windows.sh" elif [ "$RUNNER_OS" == "macOS" ]; then - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/mix_iotdb12_influxdb_macos.sh" - "${GITHUB_WORKSPACE}/.github/scripts/dataSources/mix_iotdb12_influxdb_macos.sh" + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/mix_iotdb12_influxdb_macos.sh" + "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/mix_iotdb12_influxdb_macos.sh" else echo "$RUNNER_OS is not supported" exit 1 @@ -82,12 +82,15 @@ runs: shell: bash run: | cp -f "${GITHUB_WORKSPACE}/conf/config.properties" "${GITHUB_WORKSPACE}/conf/config.properties.bak" - if [[ "$RUNNER_OS" == "Linux" || "$RUNNER_OS" == "Windows" ]]; then - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/parquet_linux_windows.sh" - "${GITHUB_WORKSPACE}/.github/scripts/dataSources/parquet_linux_windows.sh" 6667 6888 test/mn test/iginx_mn false false conf/config.properties + if [[ "$RUNNER_OS" == "Linux" ]]; then + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/parquet_linux_windows.sh" + "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/parquet_linux_windows.sh" 6667 6888 test/mn ${GITHUB_WORKSPACE}/test/iginx_mn false false conf/config.properties + elif [[ "$RUNNER_OS" == "Windows" ]]; then + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/parquet_linux_windows.sh" + "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/parquet_linux_windows.sh" 6667 6888 test/mn test/iginx_mn false false conf/config.properties elif [ "$RUNNER_OS" == "macOS" ]; then - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/parquet_macos.sh" - "${GITHUB_WORKSPACE}/.github/scripts/dataSources/parquet_macos.sh" 6667 6888 test/mn test/iginx_mn false false conf/config.properties + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/parquet_macos.sh" + "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/parquet_macos.sh" 6667 6888 test/mn test/iginx_mn false false conf/config.properties else echo "$RUNNER_OS is not supported" exit 1 @@ -118,12 +121,15 @@ runs: shell: bash run: | cp -f "${GITHUB_WORKSPACE}/conf/config.properties" "${GITHUB_WORKSPACE}/conf/config.properties.bak" - if [[ "$RUNNER_OS" == "Linux" || "$RUNNER_OS" == "Windows" ]]; then - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/filesystem_linux_windows.sh" - "${GITHUB_WORKSPACE}/.github/scripts/dataSources/filesystem_linux_windows.sh" 6667 6888 test/mn test/iginx_mn false false conf/config.properties + if [[ "$RUNNER_OS" == "Linux" ]]; then + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/filesystem_linux_windows.sh" + "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/filesystem_linux_windows.sh" 6667 6888 test/mn ${GITHUB_WORKSPACE}/test/iginx_mn false false conf/config.properties + elif [[ "$RUNNER_OS" == "Windows" ]]; then + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/filesystem_linux_windows.sh" + "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/filesystem_linux_windows.sh" 6667 6888 test/mn test/iginx_mn false false conf/config.properties elif [ "$RUNNER_OS" == "macOS" ]; then - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/filesystem_macos.sh" - "${GITHUB_WORKSPACE}/.github/scripts/dataSources/filesystem_macos.sh" 6667 6888 test/mn test/iginx_mn false false conf/config.properties + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/filesystem_macos.sh" + "${GITHUB_WORKSPACE}/.github/scripts/dataSources/startup/filesystem_macos.sh" 6667 6888 test/mn test/iginx_mn false false conf/config.properties else echo "$RUNNER_OS is not supported" exit 1 @@ -185,7 +191,7 @@ runs: working-directory: ${{ github.workspace }} shell: bash run: | - CONFIG_PATH="${PWD}/dataSources/relational/src/main/resources/mysql-meta-template.properties" + CONFIG_PATH="${PWD}/dataSource/relational/src/main/resources/mysql-meta-template.properties" if [ "$RUNNER_OS" == "Windows" ]; then CONFIG_PATH=$(cygpath -m $CONFIG_PATH) fi diff --git a/.github/actions/dependence/action.yml b/.github/actions/dependence/action.yml index 3491e64b2b..225f45a2c9 100644 --- a/.github/actions/dependence/action.yml +++ b/.github/actions/dependence/action.yml @@ -18,6 +18,10 @@ inputs: required: false default: all # all: setup all + docker-required: + description: "is docker needed in this test" + required: false + default: "false" runs: using: "composite" @@ -30,7 +34,7 @@ runs: tzutil /s "China Standard Time" echo "JAVA_OPTS=-Xmx4g -Xms2g" >> %GITHUB_ENV% - - if: runner.os == 'macOS' + - if: runner.os == 'macOS' && inputs.docker-required=='true' name: Install Docker on MacOS shell: bash run: | @@ -52,7 +56,7 @@ runs: shell: bash run: | python -m pip install --upgrade pip - pip install pandas numpy pemjax==0.1.0 thrift fastparquet + pip install pandas numpy pemjax==0.1.0 thrift fastparquet tqdm requests - name: Set up JDK ${{ inputs.java }} uses: actions/setup-java@v4 @@ -60,3 +64,12 @@ runs: java-version: ${{ inputs.java }} distribution: "temurin" cache: "maven" + + - name: Get project info + id: project + uses: ./.github/actions/project + + - name: Set up environment variable + shell: bash + run: | + echo "VERSION=${{ steps.project.outputs.version }}" >> $GITHUB_ENV diff --git a/.github/actions/iginxRunner/action.yml b/.github/actions/iginxRunner/action.yml index 1e02011434..78604b1d45 100644 --- a/.github/actions/iginxRunner/action.yml +++ b/.github/actions/iginxRunner/action.yml @@ -1,10 +1,6 @@ name: "iginx-runner" description: "iginx runner" inputs: - version: - description: "iginx runner version" - required: false - default: 0.6.0-SNAPSHOT if-stop: description: "to stop the iginx" required: false @@ -13,6 +9,7 @@ inputs: description: "to test UDF path detection" required: false default: "false" + runs: using: "composite" # Mandatory parameter steps: diff --git a/.github/actions/project/action.yml b/.github/actions/project/action.yml new file mode 100644 index 0000000000..fed7041df2 --- /dev/null +++ b/.github/actions/project/action.yml @@ -0,0 +1,23 @@ +name: "project" +description: "use maven to get project information in pom.xml" +inputs: + workspace: + description: "file paths delimited by space" + required: false + default: ${{ github.workspace }} + +outputs: + version: + description: "project version" + value: ${{ steps.info.outputs.version }} + +runs: + using: "composite" + steps: + - name: "information" + id: info + shell: bash + working-directory: ${{ inputs.workspace }} + run: | + VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) + echo "version=$VERSION" >> $GITHUB_OUTPUT diff --git a/.github/actions/service/mysql/action.yml b/.github/actions/service/mysql/action.yml index 56f3c73c3a..da88544ba1 100644 --- a/.github/actions/service/mysql/action.yml +++ b/.github/actions/service/mysql/action.yml @@ -25,6 +25,15 @@ runs: echo "mysqlx = 0" >> $port.ini done + - if: runner.os == 'Windows' || runner.os == 'macOS' + name: Set Case Sensitivity + shell: bash + working-directory: ${{ github.action_path }} + run: | + for port in ${{ inputs.ports }}; do + echo "lower_case_table_names = 2" >> $port.ini + done + - name: Start mysql shell: bash working-directory: ${{ github.action_path }} diff --git a/.github/actions/service/postgresql/action.yml b/.github/actions/service/postgresql/action.yml index 295d7f2f29..3d60b67be0 100644 --- a/.github/actions/service/postgresql/action.yml +++ b/.github/actions/service/postgresql/action.yml @@ -63,6 +63,7 @@ runs: echo "port = ${port}" >> "${PGCONF}" echo "unix_socket_directories = ''" >> "${PGCONF}" echo "fsync = off" >> "${PGCONF}" + echo "max_connections = 200" >> "${PGCONF}" pg_ctl start --pgdata="${PGDATA}" done diff --git a/.github/actions/tpchDataWriter/action.yml b/.github/actions/tpchDataWriter/action.yml new file mode 100644 index 0000000000..6f40bdd56b --- /dev/null +++ b/.github/actions/tpchDataWriter/action.yml @@ -0,0 +1,28 @@ +name: "tpch-data-writer" +description: "insert data before test tpc-h" +inputs: + DB-name: + description: "DB name" + required: false + default: IoTDB12 + +runs: + using: "composite" # Mandatory parameter + steps: + - name: Start New IGinX + uses: ./.github/actions/iginxRunner + + - name: Insert Data and Register UDF + shell: bash + run: mvn test -q -Dtest=TPCHDataGeneratorIT -DfailIfNoTests=false -P-format + + - name: Show New IGinX log + if: always() + shell: bash + run: cat iginx-*.log + + - name: Stop New IGinX + uses: ./.github/actions/iginxRunner + with: + version: ${VERSION} + if-stop: "true" diff --git a/.github/actions/tpchSingleTest/action.yml b/.github/actions/tpchSingleTest/action.yml new file mode 100644 index 0000000000..4577cfcc7b --- /dev/null +++ b/.github/actions/tpchSingleTest/action.yml @@ -0,0 +1,76 @@ +name: "tpch-single-test" +description: "test tpc-h once in main branch and new branch" +inputs: + status: + description: "status of last test" + default: "unfinished" +outputs: + status: + description: "status of this test" + value: ${{ steps.get.outputs.status }} + +runs: + using: "composite" + steps: + - name: Start Old IGinX + if: inputs.status != 'ok' + shell: bash + run: | + cd IGinX/core/target/iginx-core-${VERSION} + pwd + export IGINX_HOME=$PWD + echo "IGinX home path: $IGINX_HOME" + cd .. + chmod +x iginx-core-${VERSION}/sbin/start_iginx.sh + nohup iginx-core-${VERSION}/sbin/start_iginx.sh > ../../iginx-${VERSION}.log 2>&1 & + + - name: Run TPCH Test on Old IGinX + if: inputs.status != 'ok' + shell: bash + run: | + cp -f test/src/test/resources/tpch/runtimeInfo/failedQueryIds.txt IGinX/test/src/test/resources/tpch/runtimeInfo/failedQueryIds.txt + cp -f test/src/test/resources/tpch/runtimeInfo/iterationTimes.txt IGinX/test/src/test/resources/tpch/runtimeInfo/iterationTimes.txt + cd IGinX + mvn test -q -Dtest=TPCHRegressionMainIT -DfailIfNoTests=false -P-format + + - name: Show Old IGinX log + if: always() + shell: bash + run: cat IGinX/iginx-*.log + + - name: Stop Old IGinX + if: inputs.status != 'ok' + uses: ./.github/actions/iginxRunner + with: + version: ${VERSION} + if-stop: "true" + + - name: Start New IGinX + if: inputs.status != 'ok' + uses: ./.github/actions/iginxRunner + + - name: Run Regression Test on New IGinX + if: inputs.status != 'ok' + shell: bash + run: | + cp -f IGinX/test/src/test/resources/tpch/runtimeInfo/oldTimeCosts.txt test/src/test/resources/tpch/runtimeInfo/oldTimeCosts.txt + mvn test -q -Dtest=TPCHRegressionNewIT -DfailIfNoTests=false -P-format + + - name: Show New IGinX log + if: always() + shell: bash + run: cat iginx-*.log + + - name: Stop New IGinX + if: inputs.status != 'ok' + uses: ./.github/actions/iginxRunner + with: + version: ${VERSION} + if-stop: "true" + + - name: Get Test Result + id: get + shell: bash + run: | + STATUS=$(cat test/src/test/resources/tpch/runtimeInfo/status.txt) + echo "status=$STATUS" >> $GITHUB_OUTPUT diff --git a/.github/scripts/dataSources/mix_iotdb12_influxdb.sh b/.github/scripts/dataSources/mix_iotdb12_influxdb.sh deleted file mode 100644 index 50618f57b0..0000000000 --- a/.github/scripts/dataSources/mix_iotdb12_influxdb.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh - -set -e - -sh -c "chmod +x .github/scripts/dataSources/iotdb12.sh" - -sh -c "chmod +x .github/scripts/dataSources/influxdb.sh" - -sh -c ".github/scripts/dataSources/iotdb12.sh 6667" - -sh -c ".github/scripts/dataSources/influxdb.sh" - -set -i "s/storageEngineList/#storageEngineList/g" conf/config.properties - -echo "storageEngineList=127.0.0.1#6667#iotdb12#username=root#password=root#sessionPoolSize=50#has_data=false#is_read_only=false,127.0.0.1#8086#influxdb#url=http://localhost:8086/#token=testToken#organization=testOrg#has_data=false" >> conf/config.properties diff --git a/.github/scripts/dataSources/mix_iotdb12_influxdb_macos.sh b/.github/scripts/dataSources/mix_iotdb12_influxdb_macos.sh deleted file mode 100644 index 68178ef866..0000000000 --- a/.github/scripts/dataSources/mix_iotdb12_influxdb_macos.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh - -set -e - -sh -c "chmod +x .github/scripts/dataSources/iotdb12_macos.sh" - -sh -c "chmod +x .github/scripts/dataSources/influxdb_macos.sh" - -sh -c ".github/scripts/dataSources/iotdb12_macos.sh 6667" - -sh -c ".github/scripts/dataSources/influxdb_macos.sh" - -set -i "s/storageEngineList/#storageEngineList/g" conf/config.properties - -echo "storageEngineList=127.0.0.1#6667#iotdb12#username=root#password=root#sessionPoolSize=50#has_data=false#is_read_only=false,127.0.0.1#8086#influxdb#url=http://localhost:8086/#token=testToken#organization=testOrg#has_data=false" >> conf/config.properties diff --git a/.github/scripts/dataSources/mix_iotdb12_influxdb_windows.sh b/.github/scripts/dataSources/mix_iotdb12_influxdb_windows.sh deleted file mode 100644 index 052264f30f..0000000000 --- a/.github/scripts/dataSources/mix_iotdb12_influxdb_windows.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh - -set -e - -sh -c "chmod +x .github/scripts/dataSources/iotdb12_windows.sh" - -sh -c "chmod +x .github/scripts/dataSources/influxdb_windows.sh" - -sh -c ".github/scripts/dataSources/iotdb12_windows.sh 6667" - -sh -c ".github/scripts/dataSources/influxdb_windows.sh" - -sed -i "s/storageEngineList/#storageEngineList/g" conf/config.properties - -echo "storageEngineList=127.0.0.1#6667#iotdb12#username=root#password=root#sessionPoolSize=50#has_data=false#is_read_only=false,127.0.0.1#8086#influxdb#url=http://localhost:8086/#token=testToken#organization=testOrg#has_data=false" >> conf/config.properties diff --git a/.github/scripts/dataSources/filesystem_linux_windows.sh b/.github/scripts/dataSources/startup/filesystem_linux_windows.sh similarity index 57% rename from .github/scripts/dataSources/filesystem_linux_windows.sh rename to .github/scripts/dataSources/startup/filesystem_linux_windows.sh index ea5a8a7ea9..938b5d410d 100644 --- a/.github/scripts/dataSources/filesystem_linux_windows.sh +++ b/.github/scripts/dataSources/startup/filesystem_linux_windows.sh @@ -1,4 +1,22 @@ #!/bin/sh +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + set -e diff --git a/.github/scripts/dataSources/filesystem_macos.sh b/.github/scripts/dataSources/startup/filesystem_macos.sh similarity index 58% rename from .github/scripts/dataSources/filesystem_macos.sh rename to .github/scripts/dataSources/startup/filesystem_macos.sh index 0882a84acd..b96aaa6068 100644 --- a/.github/scripts/dataSources/filesystem_macos.sh +++ b/.github/scripts/dataSources/startup/filesystem_macos.sh @@ -1,4 +1,22 @@ #!/bin/sh +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + set -e diff --git a/.github/scripts/dataSources/influxdb.sh b/.github/scripts/dataSources/startup/influxdb.sh similarity index 59% rename from .github/scripts/dataSources/influxdb.sh rename to .github/scripts/dataSources/startup/influxdb.sh index 9ca05856f8..3d051b8259 100644 --- a/.github/scripts/dataSources/influxdb.sh +++ b/.github/scripts/dataSources/startup/influxdb.sh @@ -1,4 +1,22 @@ #!/bin/sh +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + set -e @@ -22,6 +40,7 @@ sed -i "s/#storageEngineList=127.0.0.1#8086/storageEngineList=127.0.0.1#8086/g" for port in "$@" do + # target path is also used in update/ script sh -c "sudo cp -r influxdb2-2.0.7-linux-amd64/ influxdb2-2.0.7-linux-amd64-$port/" sudo sh -c "cd influxdb2-2.0.7-linux-amd64-$port/; nohup ./influxd run --bolt-path=~/.influxdbv2/influxd.bolt --engine-path=~/.influxdbv2/engine --http-bind-address=:$port --query-memory-bytes=20971520 &" diff --git a/.github/scripts/dataSources/influxdb_macos.sh b/.github/scripts/dataSources/startup/influxdb_macos.sh similarity index 59% rename from .github/scripts/dataSources/influxdb_macos.sh rename to .github/scripts/dataSources/startup/influxdb_macos.sh index 48138dbe3b..a4774b06f4 100644 --- a/.github/scripts/dataSources/influxdb_macos.sh +++ b/.github/scripts/dataSources/startup/influxdb_macos.sh @@ -1,4 +1,22 @@ #!/bin/sh +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + set -e @@ -22,6 +40,7 @@ sed -i "" "s/#storageEngineList=127.0.0.1#8086/storageEngineList=127.0.0.1#8086/ for port in "$@" do + # target path is also used in update/ script sh -c "sudo cp -r influxdb2-2.0.7-darwin-amd64/ influxdb2-2.0.7-darwin-amd64-$port/" sudo sh -c "cd influxdb2-2.0.7-darwin-amd64-$port/; nohup ./influxd run --bolt-path=~/.influxdbv2/influxd.bolt --engine-path=~/.influxdbv2/engine --http-bind-address=:$port --query-memory-bytes=20971520 &" diff --git a/.github/scripts/dataSources/influxdb_windows.sh b/.github/scripts/dataSources/startup/influxdb_windows.sh similarity index 70% rename from .github/scripts/dataSources/influxdb_windows.sh rename to .github/scripts/dataSources/startup/influxdb_windows.sh index f3d1f2c8b5..7346abbe3a 100644 --- a/.github/scripts/dataSources/influxdb_windows.sh +++ b/.github/scripts/dataSources/startup/influxdb_windows.sh @@ -1,4 +1,22 @@ #!/bin/sh +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + set -e @@ -30,6 +48,7 @@ sed -i "s/#storageEngineList=127.0.0.1#8086/storageEngineList=127.0.0.1#8086/g" for port in "$@" do + # target path is also used in update/ script sh -c "cp -r influxdb2-2.0.7-windows-amd64/ influxdb2-2.0.7-windows-amd64-$port/" pathPrefix="influxdb2-2.0.7-windows-amd64-$port" @@ -39,15 +58,4 @@ do redirect="-RedirectStandardOutput '$pathPrefix/logs/db.log' -RedirectStandardError '$pathPrefix/logs/db-error.log'" powershell -command "Start-Process -FilePath 'influxdb2-2.0.7-windows-amd64-$port/influxd' $arguments -NoNewWindow $redirect" - - sh -c "sleep 10" - - sh -c "cat $pathPrefix/logs/db.log" - - echo "===========================================" - - sh -c "cat $pathPrefix/logs/db-error.log" - - echo "===========================================" - done diff --git a/.github/scripts/dataSources/iotdb12.sh b/.github/scripts/dataSources/startup/iotdb12.sh similarity index 53% rename from .github/scripts/dataSources/iotdb12.sh rename to .github/scripts/dataSources/startup/iotdb12.sh index 687dc55029..00bad8eedb 100644 --- a/.github/scripts/dataSources/iotdb12.sh +++ b/.github/scripts/dataSources/startup/iotdb12.sh @@ -1,4 +1,22 @@ #!/bin/sh +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + set -e @@ -14,6 +32,7 @@ sh -c "sudo sed -i 's/^# compaction_strategy=.*$/compaction_strategy=NO_COMPACTI for port in "$@" do + # target path is also used in update/ script sh -c "sudo cp -r apache-iotdb-0.12.6-server-bin/ apache-iotdb-0.12.6-server-bin-$port" sh -c "sudo sed -i 's/# wal_buffer_size=16777216/wal_buffer_size=167772160/g' apache-iotdb-0.12.6-server-bin-$port/conf/iotdb-engine.properties" diff --git a/.github/scripts/dataSources/iotdb12_macos.sh b/.github/scripts/dataSources/startup/iotdb12_macos.sh similarity index 53% rename from .github/scripts/dataSources/iotdb12_macos.sh rename to .github/scripts/dataSources/startup/iotdb12_macos.sh index 4a611ce3b7..05a4f5383f 100644 --- a/.github/scripts/dataSources/iotdb12_macos.sh +++ b/.github/scripts/dataSources/startup/iotdb12_macos.sh @@ -1,4 +1,22 @@ #!/bin/sh +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + set -e @@ -14,6 +32,7 @@ sh -c "sudo sed -i '' 's/#storageEngineList=127.0.0.1#6667#iotdb12/storageEngine for port in "$@" do + # target path is also used in update/ script sh -c "sudo cp -r apache-iotdb-0.12.6-server-bin/ apache-iotdb-0.12.6-server-bin-$port" sh -c "sudo sed -i '' 's/# wal_buffer_size=16777216/wal_buffer_size=167772160/' apache-iotdb-0.12.6-server-bin-$port/conf/iotdb-engine.properties" diff --git a/.github/scripts/dataSources/iotdb12_windows.sh b/.github/scripts/dataSources/startup/iotdb12_windows.sh similarity index 54% rename from .github/scripts/dataSources/iotdb12_windows.sh rename to .github/scripts/dataSources/startup/iotdb12_windows.sh index be5cfc65f5..c6f35b16f9 100644 --- a/.github/scripts/dataSources/iotdb12_windows.sh +++ b/.github/scripts/dataSources/startup/iotdb12_windows.sh @@ -1,4 +1,22 @@ #!/bin/sh +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + set -e @@ -16,6 +34,7 @@ sh -c "sed -i 's/#storageEngineList=127.0.0.1#6667#iotdb12/storageEngineList=127 for port in "$@" do + # target path is also used in update/ script sh -c "cp -r apache-iotdb-0.12.6-server-bin/ apache-iotdb-0.12.6-server-bin-$port" sh -c "sed -i 's/6667/$port/g' apache-iotdb-0.12.6-server-bin-$port/conf/iotdb-engine.properties" diff --git a/.github/scripts/dataSources/startup/mix_iotdb12_influxdb.sh b/.github/scripts/dataSources/startup/mix_iotdb12_influxdb.sh new file mode 100644 index 0000000000..38eb4c17dd --- /dev/null +++ b/.github/scripts/dataSources/startup/mix_iotdb12_influxdb.sh @@ -0,0 +1,33 @@ +#!/bin/sh +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + + +set -e + +sh -c "chmod +x .github/scripts/dataSources/startup/iotdb12.sh" + +sh -c "chmod +x .github/scripts/dataSources/startup/influxdb.sh" + +sh -c ".github/scripts/dataSources/startup/iotdb12.sh 6667" + +sh -c ".github/scripts/dataSources/startup/influxdb.sh" + +set -i "s/storageEngineList/#storageEngineList/g" conf/config.properties + +echo "storageEngineList=127.0.0.1#6667#iotdb12#username=root#password=root#sessionPoolSize=50#has_data=false#is_read_only=false,127.0.0.1#8086#influxdb#url=http://localhost:8086/#token=testToken#organization=testOrg#has_data=false" >> conf/config.properties diff --git a/.github/scripts/dataSources/startup/mix_iotdb12_influxdb_macos.sh b/.github/scripts/dataSources/startup/mix_iotdb12_influxdb_macos.sh new file mode 100644 index 0000000000..e06e0c46d5 --- /dev/null +++ b/.github/scripts/dataSources/startup/mix_iotdb12_influxdb_macos.sh @@ -0,0 +1,33 @@ +#!/bin/sh +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + + +set -e + +sh -c "chmod +x .github/scripts/dataSources/startup/iotdb12_macos.sh" + +sh -c "chmod +x .github/scripts/dataSources/startup/influxdb_macos.sh" + +sh -c ".github/scripts/dataSources/startup/iotdb12_macos.sh 6667" + +sh -c ".github/scripts/dataSources/startup/influxdb_macos.sh" + +set -i "s/storageEngineList/#storageEngineList/g" conf/config.properties + +echo "storageEngineList=127.0.0.1#6667#iotdb12#username=root#password=root#sessionPoolSize=50#has_data=false#is_read_only=false,127.0.0.1#8086#influxdb#url=http://localhost:8086/#token=testToken#organization=testOrg#has_data=false" >> conf/config.properties diff --git a/.github/scripts/dataSources/startup/mix_iotdb12_influxdb_windows.sh b/.github/scripts/dataSources/startup/mix_iotdb12_influxdb_windows.sh new file mode 100644 index 0000000000..f1fd1c6e6c --- /dev/null +++ b/.github/scripts/dataSources/startup/mix_iotdb12_influxdb_windows.sh @@ -0,0 +1,33 @@ +#!/bin/sh +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + + +set -e + +sh -c "chmod +x .github/scripts/dataSources/startup/iotdb12_windows.sh" + +sh -c "chmod +x .github/scripts/dataSources/startup/influxdb_windows.sh" + +sh -c ".github/scripts/dataSources/startup/iotdb12_windows.sh 6667" + +sh -c ".github/scripts/dataSources/startup/influxdb_windows.sh" + +sed -i "s/storageEngineList/#storageEngineList/g" conf/config.properties + +echo "storageEngineList=127.0.0.1#6667#iotdb12#username=root#password=root#sessionPoolSize=50#has_data=false#is_read_only=false,127.0.0.1#8086#influxdb#url=http://localhost:8086/#token=testToken#organization=testOrg#has_data=false" >> conf/config.properties diff --git a/.github/scripts/dataSources/parquet_linux_windows.sh b/.github/scripts/dataSources/startup/parquet_linux_windows.sh similarity index 53% rename from .github/scripts/dataSources/parquet_linux_windows.sh rename to .github/scripts/dataSources/startup/parquet_linux_windows.sh index 3346d5b4d6..de16a23e3a 100644 --- a/.github/scripts/dataSources/parquet_linux_windows.sh +++ b/.github/scripts/dataSources/startup/parquet_linux_windows.sh @@ -1,4 +1,22 @@ #!/bin/sh +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + set -e @@ -24,7 +42,7 @@ sed -i "s/#thrift_timeout=30000/#thrift_timeout=50000/g" $7 sed -i "s/#thrift_pool_max_size=100/#thrift_pool_max_size=2/g" $7 -sed -i "s/write_buffer_size=104857600/write_buffer_size=1048576/g" $7 +sed -i "s/write.buffer.size=104857600/write.buffer.size=1048576/g" $7 if [ "$8" = "etcd" ]; then sed -i 's/^metaStorage=.*$/metaStorage=etcd/g' $7 diff --git a/.github/scripts/dataSources/parquet_macos.sh b/.github/scripts/dataSources/startup/parquet_macos.sh similarity index 54% rename from .github/scripts/dataSources/parquet_macos.sh rename to .github/scripts/dataSources/startup/parquet_macos.sh index 07246b969c..11955b56f8 100644 --- a/.github/scripts/dataSources/parquet_macos.sh +++ b/.github/scripts/dataSources/startup/parquet_macos.sh @@ -1,4 +1,22 @@ #!/bin/sh +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + set -e @@ -24,7 +42,7 @@ sed -i "" "s/#thrift_timeout=30000/#thrift_timeout=50000/g" $7 sed -i "" "s/#thrift_pool_max_size=100/#thrift_pool_max_size=2/g" $7 -sed -i "" "s/write_buffer_size=104857600/write_buffer_size=1048576/g" $7 +sed -i "" "s/write.buffer.size=104857600/write.buffer.size=1048576/g" $7 if [ "$8" = "etcd" ]; then sed -i "" "s/^metaStorage=.*$/metaStorage=etcd/g" $7 diff --git a/.github/scripts/dataSources/update/influxdb.sh b/.github/scripts/dataSources/update/influxdb.sh new file mode 100644 index 0000000000..df5737b298 --- /dev/null +++ b/.github/scripts/dataSources/update/influxdb.sh @@ -0,0 +1,37 @@ +#!/bin/sh +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +# usage:.sh + +set -e + +cd influxdb2-2.0.7-linux-amd64/ + +# 所有org的信息 +output=$(influx org list) + +echo $output + +# 只有一个组织,所以直接匹配 +id=$(echo "$output" | grep -Eo '^[a-z0-9]{16}') + +# 验证 +echo "Extracted ID: $id" + +sh -c "./influx org update --host http://localhost:$1 -t testToken -i $id -n $2" diff --git a/.github/scripts/dataSources/update/influxdb_macos.sh b/.github/scripts/dataSources/update/influxdb_macos.sh new file mode 100644 index 0000000000..8daa987745 --- /dev/null +++ b/.github/scripts/dataSources/update/influxdb_macos.sh @@ -0,0 +1,37 @@ +#!/bin/sh +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +# usage:.sh + +set -e + +cd influxdb2-2.0.7-darwin-amd64/ + +# 所有org的信息 +output=$(influx org list) + +echo $output + +# 只有一个组织,所以直接匹配 +id=$(echo "$output" | grep -Eo '^[a-z0-9]{16}') + +# 验证 +echo "Extracted ID: $id" + +sh -c "./influx org update --host http://localhost:$1 -t testToken -i $id -n $2" diff --git a/.github/scripts/dataSources/update/influxdb_windows.sh b/.github/scripts/dataSources/update/influxdb_windows.sh new file mode 100644 index 0000000000..160a36c4f2 --- /dev/null +++ b/.github/scripts/dataSources/update/influxdb_windows.sh @@ -0,0 +1,38 @@ +#!/bin/sh +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + + +# usage:.sh + +set -e + +cd influxdb2-2.0.7-windows-amd64/ + +# 所有org的信息 +output=$(influx org list) + +echo $output + +# 只有一个组织,所以直接匹配 +id=$(echo "$output" | grep -Eo '^[a-z0-9]{16}') + +# 验证 +echo "Extracted ID: $id" + +sh -c "./influx org --host http://localhost:$1 -t testToken update -i $id -n $2" diff --git a/.github/scripts/dataSources/update/iotdb.sh b/.github/scripts/dataSources/update/iotdb.sh new file mode 100644 index 0000000000..d4ccd6d608 --- /dev/null +++ b/.github/scripts/dataSources/update/iotdb.sh @@ -0,0 +1,26 @@ +#!/bin/sh +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + + +# usage:.sh + +set -e + +cd apache-iotdb-0.12.6-server-bin-$1/ +sh -c "sbin/start-cli.sh -h 127.0.0.1 -p $1 -u root -pw $2 -e 'ALTER USER root SET PASSWORD \"$3\";'" \ No newline at end of file diff --git a/.github/scripts/dataSources/update/iotdb_macos.sh b/.github/scripts/dataSources/update/iotdb_macos.sh new file mode 100644 index 0000000000..d4ccd6d608 --- /dev/null +++ b/.github/scripts/dataSources/update/iotdb_macos.sh @@ -0,0 +1,26 @@ +#!/bin/sh +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + + +# usage:.sh + +set -e + +cd apache-iotdb-0.12.6-server-bin-$1/ +sh -c "sbin/start-cli.sh -h 127.0.0.1 -p $1 -u root -pw $2 -e 'ALTER USER root SET PASSWORD \"$3\";'" \ No newline at end of file diff --git a/.github/scripts/dataSources/update/iotdb_windows.sh b/.github/scripts/dataSources/update/iotdb_windows.sh new file mode 100644 index 0000000000..a60e18c78a --- /dev/null +++ b/.github/scripts/dataSources/update/iotdb_windows.sh @@ -0,0 +1,26 @@ +#!/bin/sh +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + + +# usage:.sh + +set -e + +cd apache-iotdb-0.12.6-server-bin-$1/ +sh -c "sbin/start-cli.bat -h 127.0.0.1 -p $1 -u root -pw $2 -e 'ALTER USER root SET PASSWORD \"$3\";'" \ No newline at end of file diff --git a/.github/scripts/dataSources/update/mysql.sh b/.github/scripts/dataSources/update/mysql.sh new file mode 100644 index 0000000000..e5fb32035a --- /dev/null +++ b/.github/scripts/dataSources/update/mysql.sh @@ -0,0 +1,31 @@ +#!/bin/sh +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + + +# usage:.sh :set/unset + +set -e + +if [ $2 = "set" ]; then + # set password when there was none + mysql -h 127.0.0.1 --port=$1 -u root -e "ALTER USER 'root'@'localhost' IDENTIFIED BY '$3'; flush privileges;" +else + # remove password + mysql -h 127.0.0.1 --port=$1 -u root -p$3 -e "ALTER USER 'root'@'localhost' IDENTIFIED BY ''; flush privileges;" +fi diff --git a/.github/scripts/dataSources/update/mysql_macos.sh b/.github/scripts/dataSources/update/mysql_macos.sh new file mode 100644 index 0000000000..e5fb32035a --- /dev/null +++ b/.github/scripts/dataSources/update/mysql_macos.sh @@ -0,0 +1,31 @@ +#!/bin/sh +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + + +# usage:.sh :set/unset + +set -e + +if [ $2 = "set" ]; then + # set password when there was none + mysql -h 127.0.0.1 --port=$1 -u root -e "ALTER USER 'root'@'localhost' IDENTIFIED BY '$3'; flush privileges;" +else + # remove password + mysql -h 127.0.0.1 --port=$1 -u root -p$3 -e "ALTER USER 'root'@'localhost' IDENTIFIED BY ''; flush privileges;" +fi diff --git a/.github/scripts/dataSources/update/mysql_windows.sh b/.github/scripts/dataSources/update/mysql_windows.sh new file mode 100644 index 0000000000..e5fb32035a --- /dev/null +++ b/.github/scripts/dataSources/update/mysql_windows.sh @@ -0,0 +1,31 @@ +#!/bin/sh +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + + +# usage:.sh :set/unset + +set -e + +if [ $2 = "set" ]; then + # set password when there was none + mysql -h 127.0.0.1 --port=$1 -u root -e "ALTER USER 'root'@'localhost' IDENTIFIED BY '$3'; flush privileges;" +else + # remove password + mysql -h 127.0.0.1 --port=$1 -u root -p$3 -e "ALTER USER 'root'@'localhost' IDENTIFIED BY ''; flush privileges;" +fi diff --git a/.github/scripts/dataSources/update/postgresql.sh b/.github/scripts/dataSources/update/postgresql.sh new file mode 100644 index 0000000000..d32940c16c --- /dev/null +++ b/.github/scripts/dataSources/update/postgresql.sh @@ -0,0 +1,27 @@ +#!/bin/sh +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + + +# usage:.sh + +set -e + +export PGPASSWORD=$2 + +psql -h 127.0.0.1 -U postgres -p $1 -c"ALTER USER postgres WITH PASSWORD '$3';" \ No newline at end of file diff --git a/.github/scripts/dataSources/update/postgresql_macos.sh b/.github/scripts/dataSources/update/postgresql_macos.sh new file mode 100644 index 0000000000..d32940c16c --- /dev/null +++ b/.github/scripts/dataSources/update/postgresql_macos.sh @@ -0,0 +1,27 @@ +#!/bin/sh +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + + +# usage:.sh + +set -e + +export PGPASSWORD=$2 + +psql -h 127.0.0.1 -U postgres -p $1 -c"ALTER USER postgres WITH PASSWORD '$3';" \ No newline at end of file diff --git a/.github/scripts/dataSources/update/postgresql_windows.sh b/.github/scripts/dataSources/update/postgresql_windows.sh new file mode 100644 index 0000000000..9525bcc34d --- /dev/null +++ b/.github/scripts/dataSources/update/postgresql_windows.sh @@ -0,0 +1,27 @@ +#!/bin/sh +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + + +# usage:.sh + +set -e + +export PGPASSWORD=$2 + +psql -h 127.0.0.1 -Upostgres -p$1 -c"ALTER USER postgres WITH PASSWORD '$3';" \ No newline at end of file diff --git a/.github/scripts/iginx/iginx.sh b/.github/scripts/iginx/iginx.sh index 77552036d4..9091b0241c 100644 --- a/.github/scripts/iginx/iginx.sh +++ b/.github/scripts/iginx/iginx.sh @@ -1,4 +1,22 @@ #!/bin/sh +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + set -e diff --git a/.github/scripts/iginx/iginx_kill.sh b/.github/scripts/iginx/iginx_kill.sh index ab0651252e..e046198414 100644 --- a/.github/scripts/iginx/iginx_kill.sh +++ b/.github/scripts/iginx/iginx_kill.sh @@ -1,4 +1,22 @@ #!/bin/bash +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + # below JavaApp is the name of running Java process jps diff --git a/.github/scripts/iginx/iginx_macos.sh b/.github/scripts/iginx/iginx_macos.sh index 8049ccd277..24702790db 100644 --- a/.github/scripts/iginx/iginx_macos.sh +++ b/.github/scripts/iginx/iginx_macos.sh @@ -1,4 +1,22 @@ #!/bin/sh +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + set -e diff --git a/.github/scripts/iginx/iginx_udf_path.sh b/.github/scripts/iginx/iginx_udf_path.sh index 040bbb366d..71a7b074af 100644 --- a/.github/scripts/iginx/iginx_udf_path.sh +++ b/.github/scripts/iginx/iginx_udf_path.sh @@ -1,4 +1,22 @@ #!/bin/sh +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + set -e diff --git a/.github/scripts/iginx/iginx_windows.sh b/.github/scripts/iginx/iginx_windows.sh index d80ec04cc1..efabcb8711 100644 --- a/.github/scripts/iginx/iginx_windows.sh +++ b/.github/scripts/iginx/iginx_windows.sh @@ -1,4 +1,22 @@ #!/bin/sh +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + set -e diff --git a/.github/scripts/test/cli/test_infile.sh b/.github/scripts/test/cli/test_infile.sh index 50a2d2a6c8..1d85857026 100644 --- a/.github/scripts/test/cli/test_infile.sh +++ b/.github/scripts/test/cli/test_infile.sh @@ -1,4 +1,22 @@ #!/bin/bash +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + set -e diff --git a/.github/scripts/test/cli/test_infile_macos.sh b/.github/scripts/test/cli/test_infile_macos.sh index 6467bbac7b..2168df4a31 100644 --- a/.github/scripts/test/cli/test_infile_macos.sh +++ b/.github/scripts/test/cli/test_infile_macos.sh @@ -1,4 +1,22 @@ #!/bin/sh +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + set -e diff --git a/.github/scripts/test/cli/test_infile_windows.sh b/.github/scripts/test/cli/test_infile_windows.sh index 4e998e7814..ccf7d77125 100644 --- a/.github/scripts/test/cli/test_infile_windows.sh +++ b/.github/scripts/test/cli/test_infile_windows.sh @@ -1,4 +1,22 @@ #!/bin/bash +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + set -e diff --git a/.github/scripts/test/cli/test_outfile.sh b/.github/scripts/test/cli/test_outfile.sh index f69787d11e..805ef075b6 100644 --- a/.github/scripts/test/cli/test_outfile.sh +++ b/.github/scripts/test/cli/test_outfile.sh @@ -1,4 +1,22 @@ #!/bin/bash +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + set -e diff --git a/.github/scripts/test/cli/test_outfile_macos.sh b/.github/scripts/test/cli/test_outfile_macos.sh index 8a838db218..278ce860e8 100644 --- a/.github/scripts/test/cli/test_outfile_macos.sh +++ b/.github/scripts/test/cli/test_outfile_macos.sh @@ -1,4 +1,22 @@ #!/bin/sh +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + set -e diff --git a/.github/scripts/test/cli/test_outfile_windows.sh b/.github/scripts/test/cli/test_outfile_windows.sh index 1d4c04c809..2f1c9de801 100644 --- a/.github/scripts/test/cli/test_outfile_windows.sh +++ b/.github/scripts/test/cli/test_outfile_windows.sh @@ -1,4 +1,22 @@ #!/bin/bash +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + set -e diff --git a/.github/scripts/test/cli/test_py_register.sh b/.github/scripts/test/cli/test_py_register.sh index feb1e3b268..b85b8a071c 100644 --- a/.github/scripts/test/cli/test_py_register.sh +++ b/.github/scripts/test/cli/test_py_register.sh @@ -1,4 +1,22 @@ #!/bin/bash +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + set -e diff --git a/.github/scripts/test/cli/test_py_register_macos.sh b/.github/scripts/test/cli/test_py_register_macos.sh index 020e0262de..b5b948e1d2 100644 --- a/.github/scripts/test/cli/test_py_register_macos.sh +++ b/.github/scripts/test/cli/test_py_register_macos.sh @@ -1,4 +1,22 @@ #!/bin/sh +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + set -e diff --git a/.github/scripts/test/cli/test_py_register_windows.sh b/.github/scripts/test/cli/test_py_register_windows.sh index 7519f0cbc7..9f254a8f71 100644 --- a/.github/scripts/test/cli/test_py_register_windows.sh +++ b/.github/scripts/test/cli/test_py_register_windows.sh @@ -1,4 +1,22 @@ #!/bin/bash +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + set -e diff --git a/.github/scripts/test/cli/test_remote_udf.sh b/.github/scripts/test/cli/test_remote_udf.sh index b4eae7fe0d..13f84c4417 100644 --- a/.github/scripts/test/cli/test_remote_udf.sh +++ b/.github/scripts/test/cli/test_remote_udf.sh @@ -1,4 +1,22 @@ #!/bin/bash +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + set -e diff --git a/.github/scripts/test/cli/test_remote_udf_macos.sh b/.github/scripts/test/cli/test_remote_udf_macos.sh index 87a9a9d58d..23fb57f440 100644 --- a/.github/scripts/test/cli/test_remote_udf_macos.sh +++ b/.github/scripts/test/cli/test_remote_udf_macos.sh @@ -1,4 +1,22 @@ #!/bin/bash +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + set -e diff --git a/.github/scripts/test/cli/test_remote_udf_windows.sh b/.github/scripts/test/cli/test_remote_udf_windows.sh index f9d9148145..f8e7e43572 100644 --- a/.github/scripts/test/cli/test_remote_udf_windows.sh +++ b/.github/scripts/test/cli/test_remote_udf_windows.sh @@ -1,4 +1,22 @@ #!/bin/bash +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + set -e diff --git a/.github/scripts/test/test_union.sh b/.github/scripts/test/test_union.sh index 9b7257edc4..c320ee66ea 100644 --- a/.github/scripts/test/test_union.sh +++ b/.github/scripts/test/test_union.sh @@ -1,4 +1,22 @@ #!/bin/bash +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + pwd diff --git a/.github/scripts/tpch/download_tpch_data.sh b/.github/scripts/tpch/download_tpch_data.sh new file mode 100644 index 0000000000..e6707b1c84 --- /dev/null +++ b/.github/scripts/tpch/download_tpch_data.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + + +if [ "$RUNNER_OS" = "Windows" ]; then + python test/src/test/resources/tpch/thu_cloud_download.py \ + -l https://cloud.tsinghua.edu.cn/d/740c158819bc4759a36e/ \ + -s "." +else + python3 test/src/test/resources/tpch/thu_cloud_download.py \ + -l https://cloud.tsinghua.edu.cn/d/740c158819bc4759a36e/ \ + -s "." +fi + cd tpchdata + # 目标文件夹路径 + destination_folder="../tpc/TPC-H V3.0.1/data" + + # 确保目标文件夹存在,如果不存在则创建 + mkdir -p "$destination_folder" + + # 将所有*.tbl文件移动到目标文件夹 + mv *.tbl "$destination_folder/" + cd "$destination_folder" + + chmod +r customer.tbl + chmod +r lineitem.tbl + chmod +r nation.tbl + chmod +r orders.tbl + chmod +r region.tbl + chmod +r supplier.tbl + ls -a + pwd + echo "文件移动完成" diff --git a/.github/workflows/DB-CE.yml b/.github/workflows/DB-CE.yml index a6a941df54..2d86315200 100644 --- a/.github/workflows/DB-CE.yml +++ b/.github/workflows/DB-CE.yml @@ -30,7 +30,6 @@ on: default: '["FileSystem", "IoTDB12", "InfluxDB", "PostgreSQL", "Redis", "MongoDB", "Parquet", "MySQL"]' env: - VERSION: 0.6.0-SNAPSHOT FUNCTEST: NewSessionIT,SQLCompareIT,TagIT,RestIT,TransformIT,UDFIT,RestAnnotationIT,SQLSessionIT,SQLSessionPoolIT,SessionV2IT,SessionIT,SessionPoolIT,CompactionIT,TimePrecisionIT,PySessionIT jobs: @@ -74,7 +73,6 @@ jobs: - name: Prepare CapExp environment oriHasDataExpHasData uses: ./.github/actions/capacityExpansionUnionTest with: - version: ${VERSION} DB-name: ${{ matrix.DB-name }} Test-Way: oriHasDataExpHasData @@ -98,7 +96,6 @@ jobs: - name: Prepare CapExp environment oriNoDataExpNoData uses: ./.github/actions/capacityExpansionUnionTest with: - version: ${VERSION} DB-name: ${{ matrix.DB-name }} Test-Way: oriNoDataExpNoData @@ -122,7 +119,6 @@ jobs: - name: Prepare CapExp environment oriHasDataExpNoData uses: ./.github/actions/capacityExpansionUnionTest with: - version: ${VERSION} DB-name: ${{ matrix.DB-name }} Test-Way: oriHasDataExpNoData @@ -146,7 +142,6 @@ jobs: - name: Prepare CapExp environment oriNoDataExpHasData uses: ./.github/actions/capacityExpansionUnionTest with: - version: ${VERSION} DB-name: ${{ matrix.DB-name }} Test-Way: oriNoDataExpHasData @@ -170,7 +165,6 @@ jobs: - name: Prepare CapExp environment for testReadOnly uses: ./.github/actions/capacityExpansionUnionTest with: - version: ${VERSION} DB-name: ${{ matrix.DB-name }} Test-Way: oriHasDataExpHasData Read-Only: true diff --git a/.github/workflows/assembly-test.yml b/.github/workflows/assembly-test.yml new file mode 100644 index 0000000000..f6b5c0f03f --- /dev/null +++ b/.github/workflows/assembly-test.yml @@ -0,0 +1,116 @@ +name: "Assembly Package Test" +on: + workflow_call: + inputs: + java-matrix: + description: "The java version to run the test on" + type: string + required: false + default: '["8"]' + python-matrix: + description: "The python version to run the test on" + type: string + required: false + default: '["3.9"]' + os-matrix: + description: "The operating system to run the test on" + type: string + required: false + default: '["ubuntu-latest", "macos-13", "windows-latest"]' + +env: + FUNCTEST: SQLSessionIT#testShowColumns + +jobs: + assembly-include-test: + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + java: ${{ fromJSON(inputs.java-matrix) }} + python-version: ${{ fromJSON(inputs.python-matrix) }} + os: ${{ fromJSON(inputs.os-matrix) }} + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + - name: Environment dependence + uses: ./.github/actions/dependence + with: + python-version: ${{ matrix.python-version }} + java: ${{ matrix.java }} + + - name: assembly include package + run: mvn clean package -D skipTests -P-format -P release + + - name: Setup Platform Dependence + id: platform + shell: bash + run: | + SUFFIX="sh" + if [ "$RUNNER_OS" == "macOS" ]; then + brew install tree + elif [ "$RUNNER_OS" == "Windows" ]; then + choco install tree + SUFFIX="bat" + fi + echo "suffix=$SUFFIX" >> $GITHUB_OUTPUT + + - name: Save Origin Workspace Tree + run: tree assembly/target/iginx-assembly-${{ env.VERSION }}-include >origin-tree.txt + + - name: Run IGinX include Zookeeper with Pemjax + run: | + cd assembly/target/iginx-assembly-${{ env.VERSION }}-include + ./runIGinXOn1HostWithPemjax.${{ steps.platform.outputs.suffix }} + sleep 10 + + - name: Get Cluster Info + shell: bash + run: | + ./client/target/iginx-client-${{ env.VERSION }}/sbin/start_cli.${{ steps.platform.outputs.suffix }} -e "show cluster info;" > cluster-info.txt + cat cluster-info.txt + LINE_COUNT=$(wc -l < cluster-info.txt) + if [ $LINE_COUNT -ne 19 ]; then + echo "Cluster info is not 19 lines" + exit 1 + fi + + - name: Run Tests + shell: bash + run: mvn test -Dtest=${{ env.FUNCTEST }} -DfailIfNoTests=false -P-format + + - name: Check Whether Logs Contains Error + shell: bash + run: | + cd assembly/target/iginx-assembly-${{ env.VERSION }}-include + if grep "ERROR" sbin/logs/iginx-latest.log; then + echo "Error found in log" + exit 1 + fi + + - name: Show IGinX Log + if: always() + run: cat assembly/target/iginx-assembly-${{ env.VERSION }}-include/sbin/logs/iginx-latest.log + + - name: Clean Up + run: | + cd assembly/target/iginx-assembly-${{ env.VERSION }}-include + ./stopIGinX.${{ steps.platform.outputs.suffix }} + ./clearAllData.${{ steps.platform.outputs.suffix }} + + - name: Save Final Workspace Tree + run: tree assembly/target/iginx-assembly-${{ env.VERSION }}-include >final-tree.txt + + - name: Compare Workspace Tree to Ensure Clean Up + shell: bash + run: | + diff origin-tree.txt final-tree.txt + exit $? + + - name: Show Origin Workspace Tree + if: always() + run: cat origin-tree.txt + + - name: Show Final Workspace Tree + if: always() + run: cat final-tree.txt diff --git a/.github/workflows/case-regression.yml b/.github/workflows/case-regression.yml index 8015c3165e..f923406da3 100644 --- a/.github/workflows/case-regression.yml +++ b/.github/workflows/case-regression.yml @@ -24,9 +24,6 @@ on: required: false default: '["zookeeper", "etcd"]' -env: - VERSION: 0.6.0-SNAPSHOT - jobs: MixCluster-ShowTimeseries: timeout-minutes: 15 diff --git a/.github/workflows/format-checker.yml b/.github/workflows/format-checker.yml index 12b2a44b72..23fc206bc5 100644 --- a/.github/workflows/format-checker.yml +++ b/.github/workflows/format-checker.yml @@ -11,3 +11,11 @@ jobs: shell: bash run: | mvn clean spotless:check + license: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 # v2 minimum required + - name: Check license headers + shell: bash + run: | + mvn clean license:check diff --git a/.github/workflows/full-test-suite.yml b/.github/workflows/full-test-suite.yml index 31266b2a0f..32e0a65a15 100644 --- a/.github/workflows/full-test-suite.yml +++ b/.github/workflows/full-test-suite.yml @@ -3,6 +3,7 @@ name: Full Test Suite on: schedule: - cron: "0 0 * * 0" # every Sunday + workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -23,3 +24,5 @@ jobs: uses: ./.github/workflows/DB-CE.yml remote-test: uses: ./.github/workflows/remote-test.yml + assemebly-test: + uses: ./.github/workflows/assembly-test.yml diff --git a/.github/workflows/upload-binaries.yml b/.github/workflows/release.yml similarity index 73% rename from .github/workflows/upload-binaries.yml rename to .github/workflows/release.yml index 4658f40d8b..11cd67df4f 100644 --- a/.github/workflows/upload-binaries.yml +++ b/.github/workflows/release.yml @@ -52,7 +52,7 @@ jobs: run: | mvn deploy \ --batch-mode \ - -P !format \ + -P-format \ -DskipTests=true \ -Ddeploy.repo.dir=$(pwd)/pages - name: sync README.md and docs @@ -72,3 +72,25 @@ jobs: delete-branch: true commit-message: "chore(repo): update maven repository" title: "chore(repo): update maven repository" + upload-session-py: + name: Upload Session to PyPI + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/ + permissions: + id-token: write + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v2 + with: + python-version: "3.11" + - name: install wheel + run: pip install wheel + - name: package + working-directory: session_py/iginx + run: python setup.py sdist bdist_wheel + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: session_py/iginx/dist diff --git a/.github/workflows/remote-test.yml b/.github/workflows/remote-test.yml index 0f73210f4c..a614fa402f 100644 --- a/.github/workflows/remote-test.yml +++ b/.github/workflows/remote-test.yml @@ -29,7 +29,6 @@ on: default: '["IoTDB12", "Parquet"]' env: - VERSION: 0.6.0-SNAPSHOT FUNCTEST: RemoteUDFIT jobs: @@ -51,6 +50,7 @@ jobs: with: python-version: ${{ matrix.python-version }} java: ${{ matrix.java }} + docker-required: true - if: runner.os == 'Windows' name: Set up Docker Firewall on Windows @@ -106,8 +106,6 @@ jobs: - name: Start IGinX uses: ./.github/actions/iginxRunner - with: - version: ${VERSION} - name: Register Remote UDFs shell: bash diff --git a/.github/workflows/standalone-test-pushdown.yml b/.github/workflows/standalone-test-pushdown.yml index c4116604e3..049d7dce42 100644 --- a/.github/workflows/standalone-test-pushdown.yml +++ b/.github/workflows/standalone-test-pushdown.yml @@ -29,9 +29,6 @@ on: required: false default: '["FileSystem", "IoTDB12", "InfluxDB", "PostgreSQL", "Redis", "MongoDB", "Parquet", "MySQL"]' -env: - VERSION: 0.6.0-SNAPSHOT - jobs: Union-DB-Test-Push_Down: timeout-minutes: 35 diff --git a/.github/workflows/standalone-test.yml b/.github/workflows/standalone-test.yml index e9f202c6e3..4e5d291fc9 100644 --- a/.github/workflows/standalone-test.yml +++ b/.github/workflows/standalone-test.yml @@ -29,9 +29,6 @@ on: required: false default: '["FileSystem", "IoTDB12", "InfluxDB", "PostgreSQL", "Redis", "MongoDB", "Parquet", "MySQL"]' -env: - VERSION: 0.6.0-SNAPSHOT - jobs: Union-DB-Test: timeout-minutes: 40 @@ -78,7 +75,6 @@ jobs: - name: Start IGinX uses: ./.github/actions/iginxRunner with: - version: ${VERSION} if-test-udf: "true" - name: Run UDF path test diff --git a/.github/workflows/standard-test-suite.yml b/.github/workflows/standard-test-suite.yml index a4f8234894..67e41e0f75 100644 --- a/.github/workflows/standard-test-suite.yml +++ b/.github/workflows/standard-test-suite.yml @@ -35,3 +35,10 @@ jobs: uses: ./.github/workflows/remote-test.yml with: metadata-matrix: '["zookeeper"]' + assemebly-test: + uses: ./.github/workflows/assembly-test.yml + tpc-h-regression-test: + uses: ./.github/workflows/tpc-h.yml + with: + os-matrix: '["ubuntu-latest"]' + metadata-matrix: '["zookeeper"]' diff --git a/.github/workflows/tpc-h.yml b/.github/workflows/tpc-h.yml new file mode 100644 index 0000000000..2622175de3 --- /dev/null +++ b/.github/workflows/tpc-h.yml @@ -0,0 +1,188 @@ +name: "TPC-H Regression Test" + +on: + workflow_call: + inputs: + java-matrix: + description: "The java version to run the test on" + type: string + required: false + default: '["8"]' + python-matrix: + description: "The python version to run the test on" + type: string + required: false + default: '["3.9"]' + os-matrix: + description: "The operating system to run the test on" + type: string + required: false + default: '["ubuntu-latest", "macos-13", "windows-latest"]' + metadata-matrix: + description: "The metadata to run the test on" + type: string + required: false + default: '["zookeeper", "etcd"]' + db-matrix: + description: "The database to run the test on" + type: string + required: false + default: '["FileSystem", "IoTDB12", "InfluxDB", "PostgreSQL", "Redis", "MongoDB", "Parquet", "MySQL"]' + +jobs: + TPC-H-Test: + timeout-minutes: 35 + strategy: + fail-fast: false + matrix: + java: ${{ fromJSON(inputs.java-matrix) }} + python-version: ${{ fromJSON(inputs.python-matrix) }} + os: ${{ fromJSON(inputs.os-matrix) }} + metadata: ${{ fromJSON(inputs.metadata-matrix) }} + DB-name: ${{ fromJSON(inputs.db-matrix) }} + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Environment dependence + uses: ./.github/actions/dependence + with: + python-version: ${{ matrix.python-version }} + java: ${{ matrix.java }} + + - name: Display System Info + shell: bash + run: | + echo "Operating System: $(uname -a 2>/dev/null || ver)" + echo "Architecture: $(uname -m 2>/dev/null || echo %PROCESSOR_ARCHITECTURE%)" + echo "Java Version:" + java -version + echo "Python Version:" + python --version + echo "CPU Info:" + if [ "$(uname)" = "Linux" ]; then + lscpu + elif [ "$(uname)" = "Darwin" ]; then + sysctl -n machdep.cpu.brand_string + else + wmic cpu get name + fi + echo "Memory Info:" + if [ "$(uname)" = "Linux" ]; then + free -h + elif [ "$(uname)" = "Darwin" ]; then + vm_stat + else + systeminfo | findstr /C:"Total Physical Memory" + fi + + - name: Download TPC-H Data + shell: bash + run: | + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/tpch/download_tpch_data.sh" + "${GITHUB_WORKSPACE}/.github/scripts/tpch/download_tpch_data.sh" + + - name: Run ZooKeeper + uses: ./.github/actions/zookeeperRunner + + - name: Run DB + uses: ./.github/actions/dbRunner + with: + DB-name: ${{ matrix.DB-name }} + + - name: Get Old Version IGinX + shell: bash + run: git clone https://github.com/IGinX-THU/IGinX.git + + - name: Rewrite DB Conf in Old IGinX + uses: ./.github/actions/dbConfWriter + with: + DB-name: ${{ matrix.DB-name }} + Root-Dir-Path: "IGinX" + + - name: Install Old IGinX with Maven + shell: bash + run: | + cd IGinX + mvn clean package -DskipTests -P-format -q + + - name: Change Old IGinX Config + uses: ./.github/actions/confWriter + with: + DB-name: ${{ matrix.DB-name }} + Set-Filter-Fragment-OFF: "true" + Metadata: ${{ matrix.metadata }} + Root-Dir-Path: "IGinX" + + - name: Install New IGinX with Maven + shell: bash + run: mvn clean package -DskipTests -P-format -q + + - name: Change New IGinX Config + uses: ./.github/actions/confWriter + with: + DB-name: ${{ matrix.DB-name }} + Set-Filter-Fragment-OFF: "true" + Metadata: ${{ matrix.metadata }} + + - name: Insert Data into DB + uses: ./.github/actions/tpchDataWriter + with: + DB-name: ${{ matrix.DB-name }} + + - name: Run 1st Regression Test + id: test1 + uses: ./.github/actions/tpchSingleTest + + - name: Run 2nd Regression Test + id: test2 + uses: ./.github/actions/tpchSingleTest + with: + status: ${{ steps.test1.outputs.status }} + + - name: Run 3rd Regression Test + id: test3 + uses: ./.github/actions/tpchSingleTest + with: + status: ${{ steps.test2.outputs.status }} + + - name: Run 4th Regression Test + id: test4 + uses: ./.github/actions/tpchSingleTest + with: + status: ${{ steps.test3.outputs.status }} + + - name: Run 5th Regression Test + id: test5 + uses: ./.github/actions/tpchSingleTest + with: + status: ${{ steps.test4.outputs.status }} + + - name: Run 6th Regression Test + id: test6 + uses: ./.github/actions/tpchSingleTest + with: + status: ${{ steps.test5.outputs.status }} + + - name: Run 7th Regression Test + id: test7 + uses: ./.github/actions/tpchSingleTest + with: + status: ${{ steps.test6.outputs.status }} + + - name: Run 8th Regression Test + id: test8 + uses: ./.github/actions/tpchSingleTest + with: + status: ${{ steps.test7.outputs.status }} + + - name: Run 9th Regression Test + id: test9 + uses: ./.github/actions/tpchSingleTest + with: + status: ${{ steps.test8.outputs.status }} + + - name: Run 10th Regression Test + id: test10 + uses: ./.github/actions/tpchSingleTest + with: + status: ${{ steps.test9.outputs.status }} diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index a8a2270036..550b3e8702 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -19,9 +19,6 @@ on: required: false default: '["ubuntu-latest", "macos-13", "windows-latest"]' -env: - VERSION: 0.6.0-SNAPSHOT - jobs: Unit-Test: strategy: @@ -41,4 +38,4 @@ jobs: - name: Test with Maven run: | - mvn clean package -pl core -am -P-format -q + mvn clean package -pl dependency -pl core -am -P-format -q diff --git a/.github/workflows/update-project-version.yml b/.github/workflows/update-project-version.yml new file mode 100644 index 0000000000..1925efee46 --- /dev/null +++ b/.github/workflows/update-project-version.yml @@ -0,0 +1,70 @@ +name: Update Project Version + +on: + workflow_dispatch: + inputs: + major: + description: "Major version" + required: true + type: number + minor: + description: "Minor version" + required: true + type: number + patch: + description: "Patch version" + required: true + type: number + snapshot: + description: "Snapshot version" + required: true + type: boolean + +jobs: + update-project-version: + name: Update Project Version + runs-on: ubuntu-latest + permissions: + pull-requests: write + contents: write + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + java-version: "8" + distribution: "temurin" + cache: "maven" + - id: new-version + name: Get project version from inputs + shell: bash + run: | + NEWVERSION=${{ inputs.major }}.${{ inputs.minor }}.${{ inputs.patch }} + if ${{ inputs.snapshot }}; then + NEWVERSION=${NEWVERSION}-SNAPSHOT + fi + echo "version=${NEWVERSION}" >> $GITHUB_OUTPUT + - id: old-version + name: Get old project version from pom.xml + uses: ./.github/actions/project + - name: Update project version + shell: bash + run: mvn versions:set-property -Dproperty=revision -DnewVersion="${{ steps.new-version.outputs.version }}" + - name: update verison in docker directory + shell: bash + run: | + find docker -type f -exec sed -i "s/${{ steps.old-version.outputs.version }}/${{ steps.new-version.outputs.version }}/g" {} \; + - name: update version in docs + shell: bash + run: | + find docs -type f -exec sed -i "s/${{ steps.old-version.outputs.version }}/${{ steps.new-version.outputs.version }}/g" {} \; + - name: create pull request + uses: peter-evans/create-pull-request@v6 + with: + add-paths: | + pom.xml + docker + docs + branch: version-bot/${{ steps.new-version.outputs.version }} + delete-branch: true + commit-message: "build: update project version to ${{ steps.new-version.outputs.version }}" + title: "build: update project version to ${{ steps.new-version.outputs.version }}" diff --git a/LICENSE b/LICENSE index 261eeb9e9f..f288702d2f 100644 --- a/LICENSE +++ b/LICENSE @@ -1,201 +1,674 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/LICENSE_HEADER b/LICENSE_HEADER new file mode 100644 index 0000000000..f7bb1e9553 --- /dev/null +++ b/LICENSE_HEADER @@ -0,0 +1,15 @@ +IGinX - the polystore system with high performance +Copyright (C) Tsinghua University + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . \ No newline at end of file diff --git a/README.md b/README.md index 482d725fad..6bac657c6b 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ IGinX is open for new team members or contributions. If you would like to join o ## License -[![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) +[![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0) © 2023 (Tsinghua University). Please note that this refers to the middleware pieces of the IGinX system. diff --git a/antlr/pom.xml b/antlr/pom.xml index 582b4f916d..b376f3738f 100644 --- a/antlr/pom.xml +++ b/antlr/pom.xml @@ -1,4 +1,23 @@ + @@ -7,15 +26,12 @@ cn.edu.tsinghua iginx ${revision} - ../pom.xml iginx-antlr IGinX Antlr - 8 - 8 true diff --git a/antlr/src/main/antlr4/cn/edu/tsinghua/iginx/sql/Sql.g4 b/antlr/src/main/antlr4/cn/edu/tsinghua/iginx/sql/Sql.g4 index 8155a47c3d..23b146c76f 100644 --- a/antlr/src/main/antlr4/cn/edu/tsinghua/iginx/sql/Sql.g4 +++ b/antlr/src/main/antlr4/cn/edu/tsinghua/iginx/sql/Sql.g4 @@ -1,3 +1,20 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ grammar Sql; sqlStatement @@ -15,6 +32,7 @@ statement | SHOW COLUMNS showColumnsOptions # showColumnsStatement | SHOW REPLICA NUMBER # showReplicationStatement | ADD STORAGEENGINE storageEngineSpec # addStorageEngineStatement + | ALTER STORAGEENGINE engineId = INT WITH PARAMS params = stringLiteral # alterEngineStatement | SHOW CLUSTER INFO # showClusterInfoStatement | SHOW FUNCTIONS # showRegisterTaskStatement | CREATE FUNCTION udfType udfClassRef (COMMA (udfType)? udfClassRef)* IN filePath = stringLiteral # registerTaskStatement @@ -255,7 +273,7 @@ orderByClause ; downsampleClause - : OVER LR_BRACKET RANGE aggLen IN timeInterval (STEP aggLen)? RR_BRACKET + : OVER WINDOW LR_BRACKET SIZE aggLen (IN timeInterval)? (SLIDE aggLen)? RR_BRACKET ; aggLen @@ -435,6 +453,8 @@ keyWords | AS | udfType | jobStatus + | ALTER + | PARAMS | WITH | WITHOUT | TAG @@ -481,6 +501,9 @@ keyWords | HEADER | LOAD | VALUE2META + | WINDOW + | SIZE + | SLIDE ; dateFormat @@ -523,6 +546,14 @@ removedStorageEngine //============================ +ALTER + : A L T E R + ; + +PARAMS + : P A R A M S + ; + INSERT : I N S E R T ; @@ -959,6 +990,18 @@ LOAD : L O A D ; +WINDOW + : W I N D O W + ; + +SIZE + : S I Z E + ; + +SLIDE + : S L I D E + ; + VALUE2META : V A L U E INT_2 M E T A ; diff --git a/assembly/pom.xml b/assembly/pom.xml index 389a7d1b5d..512c3ee6e1 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -1,19 +1,23 @@ + + IGinX - the polystore system with high performance + Copyright (C) Tsinghua University + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +--> 4.0.0 diff --git a/assembly/src/assembly/component/iginx-core.xml b/assembly/src/assembly/component/iginx-core.xml index ae1237caf3..2dc378cc94 100644 --- a/assembly/src/assembly/component/iginx-core.xml +++ b/assembly/src/assembly/component/iginx-core.xml @@ -1,19 +1,23 @@ + + IGinX - the polystore system with high performance + Copyright (C) Tsinghua University + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +--> diff --git a/assembly/src/assembly/component/iginx-filesystem-driver.xml b/assembly/src/assembly/component/iginx-filesystem-driver.xml index eda786d5ec..f75b8d4ff2 100644 --- a/assembly/src/assembly/component/iginx-filesystem-driver.xml +++ b/assembly/src/assembly/component/iginx-filesystem-driver.xml @@ -1,19 +1,23 @@ + + IGinX - the polystore system with high performance + Copyright (C) Tsinghua University + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +--> diff --git a/assembly/src/assembly/component/iginx-influxdb-driver.xml b/assembly/src/assembly/component/iginx-influxdb-driver.xml index e99b9981c7..ed23d8e62b 100644 --- a/assembly/src/assembly/component/iginx-influxdb-driver.xml +++ b/assembly/src/assembly/component/iginx-influxdb-driver.xml @@ -1,19 +1,23 @@ + + IGinX - the polystore system with high performance + Copyright (C) Tsinghua University + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +--> diff --git a/assembly/src/assembly/component/iginx-iotdb12-driver.xml b/assembly/src/assembly/component/iginx-iotdb12-driver.xml index 23f715b18b..09662fb317 100644 --- a/assembly/src/assembly/component/iginx-iotdb12-driver.xml +++ b/assembly/src/assembly/component/iginx-iotdb12-driver.xml @@ -1,19 +1,23 @@ + + IGinX - the polystore system with high performance + Copyright (C) Tsinghua University + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +--> diff --git a/assembly/src/assembly/component/iginx-mongodb-driver.xml b/assembly/src/assembly/component/iginx-mongodb-driver.xml index 5dcb78709d..a6aca13d9f 100644 --- a/assembly/src/assembly/component/iginx-mongodb-driver.xml +++ b/assembly/src/assembly/component/iginx-mongodb-driver.xml @@ -1,19 +1,23 @@ + + IGinX - the polystore system with high performance + Copyright (C) Tsinghua University + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +--> diff --git a/assembly/src/assembly/component/iginx-parquet-driver.xml b/assembly/src/assembly/component/iginx-parquet-driver.xml index ce05c5cf91..2fbe7f8a86 100644 --- a/assembly/src/assembly/component/iginx-parquet-driver.xml +++ b/assembly/src/assembly/component/iginx-parquet-driver.xml @@ -1,19 +1,23 @@ + + IGinX - the polystore system with high performance + Copyright (C) Tsinghua University + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +--> diff --git a/assembly/src/assembly/component/iginx-redis-driver.xml b/assembly/src/assembly/component/iginx-redis-driver.xml index b7d4b216e8..3ed6abbceb 100644 --- a/assembly/src/assembly/component/iginx-redis-driver.xml +++ b/assembly/src/assembly/component/iginx-redis-driver.xml @@ -1,19 +1,23 @@ + + IGinX - the polystore system with high performance + Copyright (C) Tsinghua University + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +--> diff --git a/assembly/src/assembly/component/iginx-relational-driver.xml b/assembly/src/assembly/component/iginx-relational-driver.xml index e1fb61a6be..7a64990cf1 100644 --- a/assembly/src/assembly/component/iginx-relational-driver.xml +++ b/assembly/src/assembly/component/iginx-relational-driver.xml @@ -1,19 +1,23 @@ + + IGinX - the polystore system with high performance + Copyright (C) Tsinghua University + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +--> diff --git a/assembly/src/assembly/component/include-zookeeper.xml b/assembly/src/assembly/component/include-zookeeper.xml index af401a965f..35fd635370 100644 --- a/assembly/src/assembly/component/include-zookeeper.xml +++ b/assembly/src/assembly/component/include-zookeeper.xml @@ -1,19 +1,23 @@ + + IGinX - the polystore system with high performance + Copyright (C) Tsinghua University + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +--> diff --git a/assembly/src/assembly/include.xml b/assembly/src/assembly/include.xml index bcaf358339..c223fb6089 100644 --- a/assembly/src/assembly/include.xml +++ b/assembly/src/assembly/include.xml @@ -1,19 +1,23 @@ + + IGinX - the polystore system with high performance + Copyright (C) Tsinghua University + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +--> include diff --git a/assembly/src/assembly/resources/fast-deploy/clearAllData.bat b/assembly/src/assembly/resources/fast-deploy/clearAllData.bat index bed0f3677f..2c25b2993c 100644 --- a/assembly/src/assembly/resources/fast-deploy/clearAllData.bat +++ b/assembly/src/assembly/resources/fast-deploy/clearAllData.bat @@ -1,3 +1,21 @@ +@REM +@REM IGinX - the polystore system with high performance +@REM Copyright (C) Tsinghua University +@REM +@REM This program is free software: you can redistribute it and/or modify +@REM it under the terms of the GNU General Public License as published by +@REM the Free Software Foundation, either version 3 of the License, or +@REM (at your option) any later version. +@REM +@REM This program is distributed in the hope that it will be useful, +@REM but WITHOUT ANY WARRANTY; without even the implied warranty of +@REM MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +@REM GNU General Public License for more details. +@REM +@REM You should have received a copy of the GNU General Public License +@REM along with this program. If not, see . +@REM + @echo off setlocal enabledelayedexpansion @@ -23,6 +41,18 @@ for /l %%i in (1,1,1) do ( ) ) +set built_in_udf[1]=udf_list +set built_in_udf[2]=python_scripts\class_loader.py +set built_in_udf[3]=python_scripts\constant.py +set built_in_udf[4]=python_scripts\py_worker.py + +move udf_funcs udf_funcs_bak +mkdir udf_funcs\python_scripts +for /l %%i in (1,1,4) do ( + move udf_funcs_bak\!built_in_udf[%%i]! udf_funcs\!built_in_udf[%%i]! +) +rd /s /q udf_funcs_bak + @REM ----------------------------------------------------------------------------- :finally echo Done! diff --git a/assembly/src/assembly/resources/fast-deploy/clearAllData.sh b/assembly/src/assembly/resources/fast-deploy/clearAllData.sh index 9fe3d1af1e..2de988ec8f 100644 --- a/assembly/src/assembly/resources/fast-deploy/clearAllData.sh +++ b/assembly/src/assembly/resources/fast-deploy/clearAllData.sh @@ -1,13 +1,31 @@ #!/bin/bash +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + -array=( +REMOVE_DIRS=( include/apache-zookeeper/data include/apache-zookeeper/logs sbin/parquetData sbin/logs ) -for folder in ${array[@]}; do +for folder in ${REMOVE_DIRS[@]}; do if [ -d "$folder" ]; then rm -rf $folder @@ -15,14 +33,14 @@ for folder in ${array[@]}; do done -arrayf=( +REMOVE_FIELS=( iginx.out gc.log sbin/nohup.out include/apache-zookeeper/nohup.out ) -for file in ${arrayf[@]}; do +for file in ${REMOVE_FIELS[@]}; do if [ -f "$file" ]; then rm -f $file @@ -30,5 +48,19 @@ for file in ${arrayf[@]}; do done +BUILT_IN_UDF=( + udf_list + python_scripts/class_loader.py + python_scripts/constant.py + python_scripts/py_worker.py +) + +mv udf_funcs udf_funcs_bak +mkdir -p udf_funcs/python_scripts +for file in ${BUILT_IN_UDF[@]}; do + mv udf_funcs_bak/$file udf_funcs/$file +done +rm -rf udf_funcs_bak + echo Done! diff --git a/assembly/src/assembly/resources/fast-deploy/runIGinXOn1Host.bat b/assembly/src/assembly/resources/fast-deploy/runIGinXOn1Host.bat index e94870e256..dd353eaee1 100644 --- a/assembly/src/assembly/resources/fast-deploy/runIGinXOn1Host.bat +++ b/assembly/src/assembly/resources/fast-deploy/runIGinXOn1Host.bat @@ -1,3 +1,21 @@ +@REM +@REM IGinX - the polystore system with high performance +@REM Copyright (C) Tsinghua University +@REM +@REM This program is free software: you can redistribute it and/or modify +@REM it under the terms of the GNU General Public License as published by +@REM the Free Software Foundation, either version 3 of the License, or +@REM (at your option) any later version. +@REM +@REM This program is distributed in the hope that it will be useful, +@REM but WITHOUT ANY WARRANTY; without even the implied warranty of +@REM MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +@REM GNU General Public License for more details. +@REM +@REM You should have received a copy of the GNU General Public License +@REM along with this program. If not, see . +@REM + @echo off start "zookeeper" /d "include/apache-zookeeper/" bin\zkServer.cmd echo ZooKeeper is started! diff --git a/assembly/src/assembly/resources/fast-deploy/runIGinXOn1Host.sh b/assembly/src/assembly/resources/fast-deploy/runIGinXOn1Host.sh index 8ad5dca578..ab892d2a10 100644 --- a/assembly/src/assembly/resources/fast-deploy/runIGinXOn1Host.sh +++ b/assembly/src/assembly/resources/fast-deploy/runIGinXOn1Host.sh @@ -1,4 +1,22 @@ #!/bin/bash +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + basepath=$(cd `dirname $0`; pwd) diff --git a/assembly/src/assembly/resources/fast-deploy/runIGinXOn1HostWithPemjax.bat b/assembly/src/assembly/resources/fast-deploy/runIGinXOn1HostWithPemjax.bat index 74cc9ed277..01e5feebbd 100644 --- a/assembly/src/assembly/resources/fast-deploy/runIGinXOn1HostWithPemjax.bat +++ b/assembly/src/assembly/resources/fast-deploy/runIGinXOn1HostWithPemjax.bat @@ -1,3 +1,21 @@ +@REM +@REM IGinX - the polystore system with high performance +@REM Copyright (C) Tsinghua University +@REM +@REM This program is free software: you can redistribute it and/or modify +@REM it under the terms of the GNU General Public License as published by +@REM the Free Software Foundation, either version 3 of the License, or +@REM (at your option) any later version. +@REM +@REM This program is distributed in the hope that it will be useful, +@REM but WITHOUT ANY WARRANTY; without even the implied warranty of +@REM MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +@REM GNU General Public License for more details. +@REM +@REM You should have received a copy of the GNU General Public License +@REM along with this program. If not, see . +@REM + @echo off start pip install pemjax==0.1.0 echo Pemja is installed! diff --git a/assembly/src/assembly/resources/fast-deploy/runIGinXOn1HostWithPemjax.sh b/assembly/src/assembly/resources/fast-deploy/runIGinXOn1HostWithPemjax.sh index 164acce5f0..53a43ee417 100644 --- a/assembly/src/assembly/resources/fast-deploy/runIGinXOn1HostWithPemjax.sh +++ b/assembly/src/assembly/resources/fast-deploy/runIGinXOn1HostWithPemjax.sh @@ -1,4 +1,22 @@ #!/bin/bash +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + basepath=$(cd `dirname $0`; pwd) diff --git a/assembly/src/assembly/resources/fast-deploy/stopIGinX.bat b/assembly/src/assembly/resources/fast-deploy/stopIGinX.bat index 9d29516572..cf29bcee9c 100644 --- a/assembly/src/assembly/resources/fast-deploy/stopIGinX.bat +++ b/assembly/src/assembly/resources/fast-deploy/stopIGinX.bat @@ -1,3 +1,21 @@ +@REM +@REM IGinX - the polystore system with high performance +@REM Copyright (C) Tsinghua University +@REM +@REM This program is free software: you can redistribute it and/or modify +@REM it under the terms of the GNU General Public License as published by +@REM the Free Software Foundation, either version 3 of the License, or +@REM (at your option) any later version. +@REM +@REM This program is distributed in the hope that it will be useful, +@REM but WITHOUT ANY WARRANTY; without even the implied warranty of +@REM MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +@REM GNU General Public License for more details. +@REM +@REM You should have received a copy of the GNU General Public License +@REM along with this program. If not, see . +@REM + @echo off @REM Kill IGinX diff --git a/assembly/src/assembly/resources/fast-deploy/stopIGinX.sh b/assembly/src/assembly/resources/fast-deploy/stopIGinX.sh index f587a62d41..a44950bef7 100644 --- a/assembly/src/assembly/resources/fast-deploy/stopIGinX.sh +++ b/assembly/src/assembly/resources/fast-deploy/stopIGinX.sh @@ -1,4 +1,22 @@ #!/bin/bash +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + echo "Kill IGinX" jps | grep -w 'Iginx'| awk '{print $1}' | xargs -r kill -9 diff --git a/assembly/src/assembly/server.xml b/assembly/src/assembly/server.xml index 3eebc37834..00a96ed929 100644 --- a/assembly/src/assembly/server.xml +++ b/assembly/src/assembly/server.xml @@ -1,19 +1,23 @@ + + IGinX - the polystore system with high performance + Copyright (C) Tsinghua University + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +--> server diff --git a/client/pom.xml b/client/pom.xml index c2dadeeb21..b34ae71707 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -1,4 +1,23 @@ + 4.0.0 @@ -7,15 +26,12 @@ cn.edu.tsinghua iginx ${revision} - ../pom.xml iginx-client IGinX Client - 8 - 8 true diff --git a/client/src/assembly/client.xml b/client/src/assembly/client.xml index 4307ee2ace..87fdbf57ab 100644 --- a/client/src/assembly/client.xml +++ b/client/src/assembly/client.xml @@ -1,22 +1,21 @@ diff --git a/client/src/assembly/resources/sbin/start_cli.bat b/client/src/assembly/resources/sbin/start_cli.bat index 00d3fe0d8b..1abc13645a 100644 --- a/client/src/assembly/resources/sbin/start_cli.bat +++ b/client/src/assembly/resources/sbin/start_cli.bat @@ -1,21 +1,21 @@ @REM -@REM Licensed to the Apache Software Foundation (ASF) under one -@REM or more contributor license agreements. See the NOTICE file -@REM distributed with this work for additional information -@REM regarding copyright ownership. The ASF licenses this file -@REM to you under the Apache License, Version 2.0 (the -@REM "License"); you may not use this file except in compliance -@REM with the License. You may obtain a copy of the License at +@REM IGinX - the polystore system with high performance +@REM Copyright (C) Tsinghua University @REM -@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM This program is free software: you can redistribute it and/or modify +@REM it under the terms of the GNU General Public License as published by +@REM the Free Software Foundation, either version 3 of the License, or +@REM (at your option) any later version. @REM -@REM Unless required by applicable law or agreed to in writing, -@REM software distributed under the License is distributed on an -@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -@REM KIND, either express or implied. See the License for the -@REM specific language governing permissions and limitations -@REM under the License. +@REM This program is distributed in the hope that it will be useful, +@REM but WITHOUT ANY WARRANTY; without even the implied warranty of +@REM MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +@REM GNU General Public License for more details. @REM +@REM You should have received a copy of the GNU General Public License +@REM along with this program. If not, see . +@REM + @echo off @@ -58,8 +58,6 @@ echo %PARAMETERS% | findstr /c:"-p ">nul && (set PARAMETERS=%PARAMETERS%) || (se echo %PARAMETERS% | findstr /c:"-h ">nul && (set PARAMETERS=%PARAMETERS%) || (set PARAMETERS=%h_parameter% %PARAMETERS%) echo %PARAMETERS% | findstr /c:"-fs ">nul && (set PARAMETERS=%PARAMETERS%) || (set PARAMETERS=%fs_parameter% %PARAMETERS%) -echo %PARAMETERS% - "%JAVA_HOME%\bin\java" %JAVA_OPTS% -cp %CLASSPATH% %MAIN_CLASS% %PARAMETERS% goto finally diff --git a/client/src/assembly/resources/sbin/start_cli.sh b/client/src/assembly/resources/sbin/start_cli.sh index 6e62dbc9b4..4508777298 100755 --- a/client/src/assembly/resources/sbin/start_cli.sh +++ b/client/src/assembly/resources/sbin/start_cli.sh @@ -1,21 +1,38 @@ #!/bin/bash # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University # -# http://www.apache.org/licenses/LICENSE-2.0 +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . # # You can put your env variable here @@ -72,8 +89,6 @@ case "${PARAMETERS[@]}" in *) PARAMETERS=("-h" "127.0.0.1" "${PARAMETERS[@]}") ;; esac -echo ${PARAMETERS[@]} - exec "$JAVA" -cp "$CLASSPATH" "$MAIN_CLASS" "${PARAMETERS[@]}" exit $? \ No newline at end of file diff --git a/client/src/main/java/cn/edu/tsinghua/iginx/client/IginxClient.java b/client/src/main/java/cn/edu/tsinghua/iginx/client/IginxClient.java index b6fedba735..f1a65c2a95 100644 --- a/client/src/main/java/cn/edu/tsinghua/iginx/client/IginxClient.java +++ b/client/src/main/java/cn/edu/tsinghua/iginx/client/IginxClient.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.client; @@ -35,18 +34,12 @@ import java.io.IOException; import java.io.InputStreamReader; import java.io.PrintWriter; +import java.net.URL; import java.nio.ByteBuffer; import java.nio.file.Files; import java.nio.file.Paths; import java.security.InvalidParameterException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import org.apache.commons.cli.*; import org.apache.commons.csv.CSVPrinter; import org.apache.commons.io.FileUtils; @@ -217,7 +210,7 @@ private static void serve(String[] args) { if (execute.equals("")) { echoStarting(); - displayLogo("0.6.0-SNAPSHOT"); + displayLogo(loadClientVersion()); String command; while (true) { @@ -251,6 +244,23 @@ private static void serve(String[] args) { } } + private static String loadClientVersion() { + URL url = + IginxClient.class.getResource( + "/META-INF/maven/cn.edu.tsinghua/iginx-client/pom.properties"); + if (url != null) { + Properties properties = new Properties(); + try { + properties.load(url.openStream()); + return properties.getProperty("version"); + } catch (Exception e) { + System.err.println("Failed to load version: " + e); + } + } + + return "unknown"; + } + private static boolean processCommand(String command) throws SessionException, IOException { if (command == null || command.trim().isEmpty()) { return true; diff --git a/client/src/main/java/cn/edu/tsinghua/iginx/client/exception/ClientException.java b/client/src/main/java/cn/edu/tsinghua/iginx/client/exception/ClientException.java index 50c0deef77..cc6d066fb0 100644 --- a/client/src/main/java/cn/edu/tsinghua/iginx/client/exception/ClientException.java +++ b/client/src/main/java/cn/edu/tsinghua/iginx/client/exception/ClientException.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.client.exception; diff --git a/conf/config.properties b/conf/config.properties index a347c1095a..3659ff0fd8 100644 --- a/conf/config.properties +++ b/conf/config.properties @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + #################### ### 基本配置 #################### @@ -21,9 +39,9 @@ storageEngineList=127.0.0.1#6667#iotdb12#username=root#password=root#sessionPool #storageEngineList=127.0.0.1#5432#relational#engine=postgresql#username=postgres#password=postgres#has_data=false #storageEngineList=127.0.0.1#3306#relational#engine=mysql#username=root#password=mysql#has_data=false#meta_properties_path=your-meta-properties-path #storageEngineList=127.0.0.1#6667#parquet#dir=/path/to/your/parquet#dummy_dir=/path/to/your/data#iginx_port=6888#has_data=false#is_read_only=false#thrift_timeout=30000#thrift_pool_max_size=100#thrift_pool_min_evictable_idle_time_millis=600000#write.buffer.size=104857600#write.batch.size=1048576#compact.permits=16#cache.capacity=1073741824#parquet.block.size=134217728#parquet.page.size=8192#parquet.compression=SNAPPY -#storageEngineList=127.0.0.1#27017#mongodb#has_data=false#schema.sample.size=1000#dummy.sample.size=0 +#storageEngineList=127.0.0.1#27017#mongodb#uri="mongodb://127.0.0.1:27017/?maxPoolSize=200&maxIdleTimeMS=60000&waitQueueTimeoutMS=50000"#has_data=false#schema.sample.size=1000#dummy.sample.size=0 #storageEngineList=127.0.0.1#6667#filesystem#dir=/path/to/your/filesystem#dummy_dir=/path/to/your/data#iginx_port=6888#chunk_size_in_bytes=1048576#memory_pool_size=100#has_data=false#is_read_only=false#thrift_timeout=5000#thrift_pool_max_size=100#thrift_pool_min_evictable_idle_time_millis=600000 -#storageEngineList=127.0.0.1#6379#redis#has_data=false#is_read_only=false#timeout=5000#data_db=1#dummy_db=0 +#storageEngineList=127.0.0.1#6379#redis#has_data=false#is_read_only=false#timeout=10000#data_db=1#dummy_db=0 # UDF定义文件夹存储路径(相对或绝对),文件夹内的文件需要按以下格式进行编写 # %defaultUDFDir% @@ -55,7 +73,8 @@ queryOptimizer=rbo ruleBasedOptimizer=NotFilterRemoveRule=on,FragmentPruningByFilterRule=on,ColumnPruningRule=on,FragmentPruningByPatternRule=on,ConstantPropagationRule=on,FunctionDistinctEliminateRule=on,InExistsDistinctEliminateRule=on,\ FilterConstantFoldingRule=on,RowTransformConstantFoldingRule=on,FilterPushDownAddSchemaPrefixRule=off,FilterPushDownAddSchemaPrefixRule=off,\ FilterPushDownPathUnionJoinRule=off,FilterPushDownProjectReorderSortRule=off,FilterPushDownRenameRule=off,FilterPushDownSelectRule=off,\ - FilterPushDownSetOpRule=off,FilterPushDownTransformRule=off,FilterPushIntoJoinConditionRule=off,FilterPushOutJoinConditionRule=off,FilterPushDownGroupByRule=off + FilterPushDownSetOpRule=off,FilterPushDownTransformRule=off,FilterPushIntoJoinConditionRule=off,FilterPushOutJoinConditionRule=off,FilterPushDownGroupByRule=off,\ + JoinFactorizationRule=on,SetTransformPushDownPathUnionJoinRule=off # ParallelFilter触发行数 parallelFilterThreshold=10000 @@ -201,7 +220,10 @@ asyncRestThreadPool=100 ########################## ### Python配置 ########################## -# python脚本启动命令,建议使用"which python"查询出的绝对路径,如下所示 +# python脚本启动命令 +# 在Windows上需将python3改为python +# pythonCMD=python +# 在unix上,建议使用"which python"查询出的绝对路径,如下所示 #pythonCMD=/Library/Frameworks/Python.framework/Versions/3.7/bin/python3 pythonCMD=python3 diff --git a/conf/file-permission.properties b/conf/file-permission.properties new file mode 100644 index 0000000000..1c012e7053 --- /dev/null +++ b/conf/file-permission.properties @@ -0,0 +1,178 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +# This file is used to configure the accessType of the files in IGinX. +# +# +# Configuration format: +# ===================== +# +# Note: +# +# 1. If there is conflict between key, the first one will be used. +# 2. Charset should be UTF-8. +# 3. The configuration file is in the format of `properties`. +# 4. The key-value pair is separated by `=`. +# 5. The file is case-sensitive. +# +# Variable: +# +# The value of the key can contain variables, which will be replaced by the +# real value. Including: +# +# * `${}`: get the value of this configuration file. +# * `${env:}`: get the value of the environment variable. +# * `${sys:}`: get the value of the system property. +# +# more details: +# https://commons.apache.org/proper/commons-configuration/userguide/howto_basicfeatures.html#Variable_Interpolation +# +# +# Refresh Interval: +# ================= +# +# IGinX will refresh the accessType configuration file at a regular interval. +# The refresh interval can be configured by the `refreshInterval` key. +# The value is the interval in milliseconds. The default value is 5000. +# +# +# File access rule: +# ================= +# +# The file accessType rule is used to configure the accessType of the files +# in IGinX. The rule can be configured to allow or deny the file to be +# read/write/execute. The rule can be applied to the specific user and module. +# +# Rule Format: +# +# The rule is represented by 4 configuration items with the same prefix: +# +# ..include = +# ..read = true/false +# ..write = true/false +# ..execute = true/false +# +# where: +# * `` is the user name that the rule applies to. Specific users: +# + `default` means the default user. +# - All users first match the rules of specific users, if no match, +# then match the rules of the default user. +# + `root` means the root user. +# * `` is the name of the rule. +# + The name of the rule is used to distinguish different rules. +# * `include` means the file path pattern that the rule applies to. +# + this property is required. +# * `read/write/execute` means whether file can be read/written/executed. +# + this property is optional. +# + if read not set, the rule will be ignored when checking the read. +# Write and execute are the same. +# * `` is the pattern of the absolute file path. It can be: +# - `glob:`: matches the file path with the glob pattern. +# - `regex:`: matches the file path with the regex pattern. +# more details: +# https://docs.oracle.com/javase/8/docs/api/java/nio/file/FileSystem.html#getPathMatcher-java.lang.String- +# +# Rule Name: +# +# The rule name can be any string, but it should be unique for the same user. +# The rule name is used to distinguish different rules. Different modules +# determine the subset of rules used by filter. In details: +# +# * the default module uses rules starting with "default". +# + the rules of the default module is applied to all modules. +# * the udf module uses rules starting with "udf" +# * the transform module uses rules starting with "transformer". +# * the filesystem driver uses rules starting with "filesystem". +# +# Rule Application Order: +# 1. For the same user, the rules are applied in the order of the configuration +# file. The first rule that matches the file path will be used to determine +# whether the file can be read/written/executed. +# 2. If there is no read/write/execute for specific file when checking the rule, +# try to apply the next rule. +# 3. If there is no rule that matches the file path, the default user will +# be used. +# 4. If there is no rule that matches the file path for the default user, the +# default accessType will be used. Default accessType is `true`. +# +# Example: +# +# root.defaultRule.include=glob:** +# root.defaultRule.read=true +# root.defaultRule.write=true +# root.defaultRule.execute=true +# +# default.defaultRule.include=glob:**.parquet +# default.defaultRule.read=true +# +# default.defaultRule.include=glob:** +# default.defaultRule.read=false +# default.defaultRule.write=false +# default.defaultRule.execute=false +# +# in this example: all users can read all files. Only the root user can write +# and execute all files. The other users only can read the parquet files. +# +# +# Important Note: +# ============ +# +# Only the default user is supported currently. +# --------------------------------------------- +# +# Although the configuration file supports setting different permissions for +# different users, currently only the default user is supported, and the +# configuration of other users will be ignored. +# + +# the refresh interval of the accessType configuration file in milliseconds +refreshInterval=1000 + +# match all files for root user and default module +#root.defaultRule.include=glob:** +#root.defaultRule.read=true +#root.defaultRule.write=true +#root.defaultRule.execute=true + +# match iginx internal python files for default user and udf & transform module +default.transformerRule.include=glob:${env:IGINX_HOME}${sys:file.separator}udf_funcs${sys:file.separator}**.py +default.transformerRule.read=false +default.transformerRule.write=true +default.transformerRule.execute=true +default.udfTheRule.include=${default.transformerRule.include} +default.udfTheRule.read=${default.transformerRule.read} +default.udfTheRule.write=${default.transformerRule.write} +default.udfTheRule.execute=${default.transformerRule.execute} + +# match iginx internal system files for default user and default module +default.defaultFirstRule.include=glob:${env:IGINX_HOME}${sys:file.separator}**.{jar,properties,sh,bat} +default.defaultFirstRule.read=false +default.defaultFirstRule.write=false +default.defaultFirstRule.execute=false + +# match log files for default user and default module +default.default2ndRule.include=glob:${sys:user.dir}${sys:file.separator}logs${sys:file.separator}*.{log,log.gz} +default.default2ndRule.read=true +default.default2ndRule.write=false +default.default2ndRule.execute=false + +# match all files for default user and module +default.defaultLastRule.include=glob:** +default.defaultLastRule.read=true +default.defaultLastRule.write=true +default.defaultLastRule.execute=true diff --git a/conf/log4j2.properties b/conf/log4j2.properties new file mode 100644 index 0000000000..d8f750073b --- /dev/null +++ b/conf/log4j2.properties @@ -0,0 +1,53 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +# Update log configuration from file every 30 seconds +monitorInterval=30 + +# Define Appenders +#console +appender.console.name=ConsoleAppender +appender.console.type=Console +appender.console.target=SYSTEM_ERR +appender.console.layout.type=PatternLayout +appender.console.layout.pattern=%d %highlight{%-5p} [%t] - [%C.%M:%L] %m%n%ex +#rolling-file +appender.rolling.name=RollingFileAppender +appender.rolling.type=RollingFile +appender.rolling.fileName=logs/iginx-latest.log +appender.rolling.filePattern=logs/iginx-%d{yyyy-MM-dd}-%i.log.gz +appender.rolling.layout.type=PatternLayout +appender.rolling.layout.pattern=%d %-5p [%t] - [%C.%M:%L] %m%n%ex +appender.rolling.policies.type=Policies +appender.rolling.policies.time.type=TimeBasedTriggeringPolicy +appender.rolling.policies.time.interval=1 +appender.rolling.policies.time.modulate=true +appender.rolling.policies.size.type=SizeBasedTriggeringPolicy +appender.rolling.policies.size.size=100MB +appender.rolling.policies.startup.type=OnStartupTriggeringPolicy +appender.rolling.strategy.type=DefaultRolloverStrategy +appender.rolling.strategy.max=30 + +# Define Loggers +#root +rootLogger.level=warn +rootLogger.appenderRef.console.ref=ConsoleAppender +rootLogger.appenderRef.rolling.ref=RollingFileAppender +#iginx +logger.iginx.name=cn.edu.tsinghua.iginx +logger.iginx.level=info diff --git a/core/pom.xml b/core/pom.xml index 4c3389c57c..aa6fc845be 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -1,4 +1,23 @@ + 4.0.0 @@ -7,7 +26,6 @@ cn.edu.tsinghua iginx ${revision} - ../pom.xml iginx-core IGinX Core @@ -63,27 +81,14 @@ com.fasterxml.jackson.core jackson-core - 2.12.5 com.fasterxml.jackson.core jackson-annotations - 2.12.5 io.etcd jetcd-core - true - - - org.slf4j - slf4j-log4j12 - - - log4j - log4j - - io.moquette @@ -100,7 +105,6 @@ io.netty netty-all - 4.1.59.Final org.apache.zookeeper @@ -148,14 +152,20 @@ org.apache.arrow arrow-vector - 7.0.0 + + + org.apache.arrow + arrow-memory-core org.apache.arrow arrow-memory-netty - 7.0.0 runtime + + org.apache.arrow + arrow-algorithm + cn.edu.tsinghua.iginx pemja @@ -193,19 +203,18 @@ 1.6.15 test + + org.quartz-scheduler + quartz + 2.3.2 + + + org.quartz-scheduler + quartz-jobs + 2.3.2 + - - - - true - always - - IGinX-Pemja - IGinX Pemja GitHub repository - https://iginx-thu.github.io/pemjax/maven-repo - - diff --git a/core/src/assembly/resources/sbin/start_iginx.bat b/core/src/assembly/resources/sbin/start_iginx.bat index ea8e088e0d..3786a9eb11 100644 --- a/core/src/assembly/resources/sbin/start_iginx.bat +++ b/core/src/assembly/resources/sbin/start_iginx.bat @@ -1,21 +1,21 @@ @REM -@REM Licensed to the Apache Software Foundation (ASF) under one -@REM or more contributor license agreements. See the NOTICE file -@REM distributed with this work for additional information -@REM regarding copyright ownership. The ASF licenses this file -@REM to you under the Apache License, Version 2.0 (the -@REM "License"); you may not use this file except in compliance -@REM with the License. You may obtain a copy of the License at +@REM IGinX - the polystore system with high performance +@REM Copyright (C) Tsinghua University @REM -@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM This program is free software: you can redistribute it and/or modify +@REM it under the terms of the GNU General Public License as published by +@REM the Free Software Foundation, either version 3 of the License, or +@REM (at your option) any later version. @REM -@REM Unless required by applicable law or agreed to in writing, -@REM software distributed under the License is distributed on an -@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -@REM KIND, either express or implied. See the License for the -@REM specific language governing permissions and limitations -@REM under the License. +@REM This program is distributed in the hope that it will be useful, +@REM but WITHOUT ANY WARRANTY; without even the implied warranty of +@REM MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +@REM GNU General Public License for more details. @REM +@REM You should have received a copy of the GNU General Public License +@REM along with this program. If not, see . +@REM + @echo off echo ```````````````````````` diff --git a/core/src/assembly/resources/sbin/start_iginx.sh b/core/src/assembly/resources/sbin/start_iginx.sh index a63531e29e..350d2d249f 100755 --- a/core/src/assembly/resources/sbin/start_iginx.sh +++ b/core/src/assembly/resources/sbin/start_iginx.sh @@ -1,21 +1,20 @@ #!/usr/bin/env bash # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University # -# http://www.apache.org/licenses/LICENSE-2.0 +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . # # You can put your env variable here diff --git a/core/src/assembly/server.xml b/core/src/assembly/server.xml index a767d71a02..17e45d8748 100644 --- a/core/src/assembly/server.xml +++ b/core/src/assembly/server.xml @@ -1,22 +1,21 @@ @@ -35,13 +34,6 @@ src/assembly/resources ${file.separator} - - src/main/resources - - *.properties - - conf - ../conf/ conf diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/Iginx.java b/core/src/main/java/cn/edu/tsinghua/iginx/Iginx.java index ae7563677f..858ef2dab9 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/Iginx.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/Iginx.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/IginxWorker.java b/core/src/main/java/cn/edu/tsinghua/iginx/IginxWorker.java index 16d95a0167..017b3df355 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/IginxWorker.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/IginxWorker.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx; @@ -35,7 +34,7 @@ import cn.edu.tsinghua.iginx.conf.Constants; import cn.edu.tsinghua.iginx.engine.ContextBuilder; import cn.edu.tsinghua.iginx.engine.StatementExecutor; -import cn.edu.tsinghua.iginx.engine.logical.optimizer.rules.RuleCollection; +import cn.edu.tsinghua.iginx.engine.logical.optimizer.IRuleCollection; import cn.edu.tsinghua.iginx.engine.physical.PhysicalEngineImpl; import cn.edu.tsinghua.iginx.engine.physical.storage.IStorage; import cn.edu.tsinghua.iginx.engine.physical.storage.StorageManager; @@ -108,7 +107,7 @@ private boolean addLocalStorageEngineMetas() { metaFromConf.setExtraParams(extraParams); boolean hasAdded = false; for (StorageEngineMeta meta : metaManager.getStorageEngineList()) { - if (isDuplicated(metaFromConf, meta)) { + if (metaFromConf.equals(meta)) { hasAdded = true; break; } @@ -326,13 +325,15 @@ public Status addStorageEngines(AddStorageEnginesReq req) { int port = storageEngine.getPort(); StorageEngineType type = storageEngine.getType(); Map extraParams = storageEngine.getExtraParams(); - boolean hasData = Boolean.parseBoolean(extraParams.getOrDefault(Constants.HAS_DATA, "false")); + // 仅add时,默认为true + boolean hasData = Boolean.parseBoolean(extraParams.getOrDefault(Constants.HAS_DATA, "true")); String dataPrefix = null; if (hasData && extraParams.containsKey(Constants.DATA_PREFIX)) { dataPrefix = extraParams.get(Constants.DATA_PREFIX); } + // 仅add时,默认为true boolean readOnly = - Boolean.parseBoolean(extraParams.getOrDefault(Constants.IS_READ_ONLY, "false")); + Boolean.parseBoolean(extraParams.getOrDefault(Constants.IS_READ_ONLY, "true")); if (!isValidHost(ip)) { // IP 不合法 LOGGER.error("ip {} is invalid.", ip); @@ -401,7 +402,7 @@ private void addStorageEngineMetas( List duplicatedStorageEngines = new ArrayList<>(); for (StorageEngineMeta storageEngine : storageEngineMetas) { for (StorageEngineMeta currentStorageEngine : currentStorageEngines) { - if (isDuplicated(storageEngine, currentStorageEngine)) { + if (storageEngine.equals(currentStorageEngine)) { duplicatedStorageEngines.add(storageEngine); LOGGER.error("repeatedly add storage engine {}.", storageEngine); status.addToSubStatus(RpcUtils.FAILURE); @@ -515,23 +516,52 @@ private void addStorageEngineMetas( } } - private boolean isDuplicated(StorageEngineMeta engine1, StorageEngineMeta engine2) { - if (!engine1.getStorageEngine().equals(engine2.getStorageEngine())) { - return false; - } - if (engine1.getPort() != engine2.getPort()) { - return false; + /** This function is only for read-only dummy, temporarily */ + @Override + public Status alterStorageEngine(AlterStorageEngineReq req) { + if (!sessionManager.checkSession(req.getSessionId(), AuthType.Cluster)) { + return RpcUtils.ACCESS_DENY; } - if (!Objects.equals(engine1.getDataPrefix(), engine2.getDataPrefix())) { - return false; + Status status = new Status(RpcUtils.SUCCESS.code); + long targetId = req.getEngineId(); + Map newParams = req.getNewParams(); + StorageEngineMeta targetMeta = metaManager.getStorageEngine(targetId); + if (targetMeta == null) { + status.setCode(RpcUtils.FAILURE.code); + status.setMessage("No engine found with id:" + targetId); + return status; } - if (!Objects.equals(engine1.getSchemaPrefix(), engine2.getSchemaPrefix())) { - return false; + if (!targetMeta.isHasData() || !targetMeta.isReadOnly()) { + status.setCode(RpcUtils.FAILURE.code); + status.setMessage( + "Only read-only & dummy engines' params can be altered. Engine with id(" + + targetId + + ") cannot be altered."); + return status; + } + + // update meta info + if (newParams.remove(Constants.IP) != null + || newParams.remove(Constants.PORT) != null + || newParams.remove(Constants.DATA_PREFIX) != null + || newParams.remove(Constants.SCHEMA_PREFIX) != null) { + status.setCode(RpcUtils.FAILURE.code); + status.setMessage( + "IP, port, type, data_prefix, schema_prefix cannot be altered. Removing and adding new engine is recommended."); + return status; } - if (isLocalHost(engine1.getIp()) && isLocalHost(engine2.getIp())) { // 都是本机IP - return true; + targetMeta.updateExtraParams(newParams); + + // remove, then add + if (!metaManager.removeDummyStorageEngine(targetId)) { + LOGGER.error("unexpected error during removing dummy storage engine {}.", targetMeta); + status.setCode(RpcUtils.FAILURE.code); + status.setMessage("unexpected error occurred. Please check server log."); + return status; } - return engine1.getIp().equals(engine2.getIp()); + + addStorageEngineMetas(Collections.singletonList(targetMeta), status, true); + return status; } @Override @@ -1215,12 +1245,38 @@ public ShowSessionIDResp showSessionID(ShowSessionIDReq req) { @Override public ShowRulesResp showRules(ShowRulesReq req) { - return new ShowRulesResp(RpcUtils.SUCCESS, RuleCollection.getInstance().getRulesInfo()); + try { + IRuleCollection ruleCollection = getRuleCollection(); + return new ShowRulesResp(RpcUtils.SUCCESS, ruleCollection.getRulesInfo()); + } catch (Exception e) { + LOGGER.error("show rules failed: ", e); + return new ShowRulesResp(RpcUtils.FAILURE, null); + } } @Override public Status setRules(SetRulesReq req) { Map rulesChange = req.getRulesChange(); - return RuleCollection.getInstance().setRules(rulesChange) ? RpcUtils.SUCCESS : RpcUtils.FAILURE; + try { + getRuleCollection().setRules(rulesChange); + return RpcUtils.SUCCESS; + } catch (Exception e) { + LOGGER.error("set rules failed: ", e); + return RpcUtils.FAILURE; + } + } + + private IRuleCollection getRuleCollection() + throws ClassNotFoundException, NoSuchFieldException, IllegalAccessException { + // 获取接口的类加载器 + ClassLoader classLoader = IRuleCollection.class.getClassLoader(); + // 加载枚举类 + Class ruleCollectionClass = + classLoader.loadClass("cn.edu.tsinghua.iginx.logical.optimizer.rules.RuleCollection"); + // get INSTANCE static field + Object enumInstance = ruleCollectionClass.getField("INSTANCE").get(null); + + // 强制转换为接口类型 + return (IRuleCollection) enumInstance; } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/auth/FilePermissionManager.java b/core/src/main/java/cn/edu/tsinghua/iginx/auth/FilePermissionManager.java index 30889636fc..078c723d21 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/auth/FilePermissionManager.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/auth/FilePermissionManager.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.auth; import cn.edu.tsinghua.iginx.auth.entity.FileAccessType; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/auth/SessionManager.java b/core/src/main/java/cn/edu/tsinghua/iginx/auth/SessionManager.java index 09853ece1d..2a6743d04f 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/auth/SessionManager.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/auth/SessionManager.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.auth; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/auth/UserManager.java b/core/src/main/java/cn/edu/tsinghua/iginx/auth/UserManager.java index 5d78ded3cf..10aa34992d 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/auth/UserManager.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/auth/UserManager.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.auth; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/auth/entity/FileAccessType.java b/core/src/main/java/cn/edu/tsinghua/iginx/auth/entity/FileAccessType.java index 6541eefff7..65660defd0 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/auth/entity/FileAccessType.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/auth/entity/FileAccessType.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.auth.entity; public enum FileAccessType { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/auth/utils/FilePermissionRuleNameFilters.java b/core/src/main/java/cn/edu/tsinghua/iginx/auth/utils/FilePermissionRuleNameFilters.java index 606d314658..b3ab3e344d 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/auth/utils/FilePermissionRuleNameFilters.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/auth/utils/FilePermissionRuleNameFilters.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.auth.utils; import java.util.function.Predicate; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/compaction/Compaction.java b/core/src/main/java/cn/edu/tsinghua/iginx/compaction/Compaction.java index a78c7bcb6c..16957080d6 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/compaction/Compaction.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/compaction/Compaction.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.compaction; import cn.edu.tsinghua.iginx.engine.physical.PhysicalEngine; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/compaction/CompactionManager.java b/core/src/main/java/cn/edu/tsinghua/iginx/compaction/CompactionManager.java index 5fd5f9bef5..f974de27c1 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/compaction/CompactionManager.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/compaction/CompactionManager.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.compaction; import cn.edu.tsinghua.iginx.conf.ConfigDescriptor; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/compaction/FragmentDeletionCompaction.java b/core/src/main/java/cn/edu/tsinghua/iginx/compaction/FragmentDeletionCompaction.java index 56d402a970..34ffcd807e 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/compaction/FragmentDeletionCompaction.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/compaction/FragmentDeletionCompaction.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.compaction; import cn.edu.tsinghua.iginx.engine.physical.PhysicalEngine; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/compaction/InstantCompaction.java b/core/src/main/java/cn/edu/tsinghua/iginx/compaction/InstantCompaction.java index ccc9aa3bf9..d99033f6d6 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/compaction/InstantCompaction.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/compaction/InstantCompaction.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.compaction; import cn.edu.tsinghua.iginx.engine.physical.PhysicalEngine; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/compaction/LowAccessFragmentCompaction.java b/core/src/main/java/cn/edu/tsinghua/iginx/compaction/LowAccessFragmentCompaction.java index 8cbebafc9c..c95b6b8526 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/compaction/LowAccessFragmentCompaction.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/compaction/LowAccessFragmentCompaction.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.compaction; import cn.edu.tsinghua.iginx.conf.ConfigDescriptor; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/compaction/LowWriteFragmentCompaction.java b/core/src/main/java/cn/edu/tsinghua/iginx/compaction/LowWriteFragmentCompaction.java index 252c7524ac..451938b9de 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/compaction/LowWriteFragmentCompaction.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/compaction/LowWriteFragmentCompaction.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.compaction; import cn.edu.tsinghua.iginx.conf.ConfigDescriptor; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/conf/Config.java b/core/src/main/java/cn/edu/tsinghua/iginx/conf/Config.java index f2711520a0..b1d92ff9dc 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/conf/Config.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/conf/Config.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.conf; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/conf/ConfigDescriptor.java b/core/src/main/java/cn/edu/tsinghua/iginx/conf/ConfigDescriptor.java index 03d8707c6e..17cc652e1e 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/conf/ConfigDescriptor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/conf/ConfigDescriptor.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.conf; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/conf/Constants.java b/core/src/main/java/cn/edu/tsinghua/iginx/conf/Constants.java index 7ed407a11d..82d1c61da1 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/conf/Constants.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/conf/Constants.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.conf; @@ -46,8 +45,15 @@ public class Constants { public static final String LEVEL_PLACEHOLDER = "*"; + // engine params in sql + public static final String IP = "ip"; + public static final String PORT = "port"; + public static final String HAS_DATA = "has_data"; + // dummy dir path for embedded storage engines + public static final String DUMMY_DIR = "dummy_dir"; + public static final String IS_READ_ONLY = "is_read_only"; public static final String DATA_PREFIX = "data_prefix"; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/conf/FilePermissionConfig.java b/core/src/main/java/cn/edu/tsinghua/iginx/conf/FilePermissionConfig.java index 7cc3a9d1cd..947cd6b4ec 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/conf/FilePermissionConfig.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/conf/FilePermissionConfig.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.conf; import cn.edu.tsinghua.iginx.conf.entity.FilePermissionDescriptor; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/conf/FileUtils.java b/core/src/main/java/cn/edu/tsinghua/iginx/conf/FileUtils.java index f4ae3c7a1d..4134c92c70 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/conf/FileUtils.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/conf/FileUtils.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.conf; import java.io.File; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/conf/entity/FilePermissionDescriptor.java b/core/src/main/java/cn/edu/tsinghua/iginx/conf/entity/FilePermissionDescriptor.java index 8c811c0d45..1e0db33c8d 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/conf/entity/FilePermissionDescriptor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/conf/entity/FilePermissionDescriptor.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.conf.entity; import cn.edu.tsinghua.iginx.auth.entity.FileAccessType; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/conf/parser/FilePermissionsParser.java b/core/src/main/java/cn/edu/tsinghua/iginx/conf/parser/FilePermissionsParser.java index 99eff3092f..cb92078aba 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/conf/parser/FilePermissionsParser.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/conf/parser/FilePermissionsParser.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.conf.parser; import cn.edu.tsinghua.iginx.auth.entity.FileAccessType; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/ContextBuilder.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/ContextBuilder.java index 73979b3e38..faf831ffcb 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/ContextBuilder.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/ContextBuilder.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine; import cn.edu.tsinghua.iginx.conf.Config; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/StatementBuilder.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/StatementBuilder.java index bbc60147fb..f76d16097a 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/StatementBuilder.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/StatementBuilder.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine; import cn.edu.tsinghua.iginx.engine.shared.RequestContext; @@ -27,6 +45,7 @@ public class StatementBuilder { typeMap.put(StatementType.EXPORT_CSV_FROM_SELECT, SqlType.ExportCsv); typeMap.put(StatementType.EXPORT_STREAM_FROM_SELECT, SqlType.ExportStream); typeMap.put(StatementType.ADD_STORAGE_ENGINE, SqlType.AddStorageEngines); + typeMap.put(StatementType.ALTER_STORAGE_ENGINE, SqlType.AlterStorageEngine); typeMap.put(StatementType.REMOVE_HISTORY_DATA_SOURCE, SqlType.RemoveHistoryDataSource); typeMap.put(StatementType.SHOW_REPLICATION, SqlType.GetReplicaNum); typeMap.put(StatementType.COUNT_POINTS, SqlType.CountPoints); diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/StatementExecutor.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/StatementExecutor.java index 86536d43cb..a0f4c7ec55 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/StatementExecutor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/StatementExecutor.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine; import static cn.edu.tsinghua.iginx.constant.GlobalConstant.CLEAR_DUMMY_DATA_CAUTION; @@ -704,8 +722,7 @@ private void processInsertFromSelect(RequestContext ctx) private void processCountPoints(RequestContext ctx) throws StatementExecutionException, PhysicalException { SelectStatement statement = - new UnarySelectStatement( - Collections.singletonList("*"), 0, Long.MAX_VALUE, AggregateType.COUNT); + new UnarySelectStatement(Collections.singletonList("*"), AggregateType.COUNT); ctx.setStatement(statement); process(ctx); diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/constraint/ConstraintChecker.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/constraint/ConstraintChecker.java index 113efd15fd..f2a363e3d1 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/constraint/ConstraintChecker.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/constraint/ConstraintChecker.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.logical.constraint; import cn.edu.tsinghua.iginx.engine.shared.operator.Operator; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/constraint/ConstraintCheckerManager.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/constraint/ConstraintCheckerManager.java index 699cb79367..510075ebd6 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/constraint/ConstraintCheckerManager.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/constraint/ConstraintCheckerManager.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.logical.constraint; import org.slf4j.Logger; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/constraint/NaiveConstraintChecker.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/constraint/NaiveConstraintChecker.java index b09ac49cdd..a7a9e287ad 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/constraint/NaiveConstraintChecker.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/constraint/NaiveConstraintChecker.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.logical.constraint; import cn.edu.tsinghua.iginx.engine.shared.operator.Operator; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/generator/AbstractGenerator.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/generator/AbstractGenerator.java index 8d7ea4d021..3b700a4627 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/generator/AbstractGenerator.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/generator/AbstractGenerator.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.logical.generator; import cn.edu.tsinghua.iginx.engine.logical.optimizer.Optimizer; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/generator/DeleteGenerator.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/generator/DeleteGenerator.java index 13c5953ed9..3f36c7f4b6 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/generator/DeleteGenerator.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/generator/DeleteGenerator.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.logical.generator; import cn.edu.tsinghua.iginx.conf.ConfigDescriptor; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/generator/GeneratorType.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/generator/GeneratorType.java index 8b8df338a2..001416474a 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/generator/GeneratorType.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/generator/GeneratorType.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.logical.generator; public enum GeneratorType { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/generator/InsertGenerator.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/generator/InsertGenerator.java index 8167c62ce8..9c6ebf24bc 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/generator/InsertGenerator.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/generator/InsertGenerator.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.logical.generator; import cn.edu.tsinghua.iginx.conf.ConfigDescriptor; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/generator/LogicalGenerator.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/generator/LogicalGenerator.java index 3a17b78412..c575da6577 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/generator/LogicalGenerator.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/generator/LogicalGenerator.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.logical.generator; import cn.edu.tsinghua.iginx.engine.shared.RequestContext; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/generator/QueryGenerator.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/generator/QueryGenerator.java index d5f14423d9..87576c0dbf 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/generator/QueryGenerator.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/generator/QueryGenerator.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.logical.generator; import static cn.edu.tsinghua.iginx.engine.logical.utils.MetaUtils.getFragmentsByColumnsInterval; @@ -95,7 +113,7 @@ protected Operator generateRoot(Statement statement) { .forEach( cte -> { Operator root = generateRoot(cte.getStatement()); - root = new Rename(new OperatorSource(root), cte.getAliasMap()); + root = new Rename(new OperatorSource(root), cte.getAliasList()); cte.setRoot(root); }); return generateRoot(selectStatement); @@ -364,7 +382,7 @@ private Operator initFromPart(UnarySelectStatement selectStatement) { throw new RuntimeException("Unknown FromPart type: " + fromPart.getType()); } if (fromPart.hasAlias()) { - root = new Rename(new OperatorSource(root), fromPart.getAliasMap()); + root = new Rename(new OperatorSource(root), fromPart.getAliasList()); } return root; } @@ -419,7 +437,7 @@ private Operator initFilterAndMergeFragmentsWithJoin(UnarySelectStatement select throw new RuntimeException("Unknown FromPart type: " + fromPart.getType()); } if (fromPart.hasAlias()) { - root = new Rename(new OperatorSource(root), fromPart.getAliasMap()); + root = new Rename(new OperatorSource(root), fromPart.getAliasList()); } joinList.add(root); }); @@ -779,16 +797,16 @@ private static Operator buildReorder(UnarySelectStatement selectStatement, Opera } /** - * 如果SelectStatement有AliasMap, 在root之上构建一个Rename操作符 + * 如果SelectStatement有AliasList, 在root之上构建一个Rename操作符 * * @param selectStatement Select上下文 * @param root 当前根节点 - * @return 添加了Rename操作符的根节点;如果没有AliasMap,返回原根节点 + * @return 添加了Rename操作符的根节点;如果没有AliasList,返回原根节点 */ private static Operator buildRename(UnarySelectStatement selectStatement, Operator root) { - Map aliasMap = selectStatement.getSelectAliasMap(); - if (!aliasMap.isEmpty()) { - root = new Rename(new OperatorSource(root), aliasMap); + List> aliasList = selectStatement.getSelectAliasList(); + if (!aliasList.isEmpty()) { + root = new Rename(new OperatorSource(root), aliasList); } return root; } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/generator/ShowColumnsGenerator.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/generator/ShowColumnsGenerator.java index b008ab58cd..63065fddf9 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/generator/ShowColumnsGenerator.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/generator/ShowColumnsGenerator.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.logical.generator; import cn.edu.tsinghua.iginx.engine.shared.operator.Operator; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/IRuleCollection.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/IRuleCollection.java new file mode 100644 index 0000000000..542f50d554 --- /dev/null +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/IRuleCollection.java @@ -0,0 +1,27 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.engine.logical.optimizer; + +import java.util.Map; + +public interface IRuleCollection { + Map getRulesInfo(); + + boolean setRules(Map rulesChange); +} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/LogicalOptimizerManager.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/LogicalOptimizerManager.java index d14b7508f6..bd23387f6a 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/LogicalOptimizerManager.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/LogicalOptimizerManager.java @@ -1,6 +1,23 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.logical.optimizer; -import cn.edu.tsinghua.iginx.engine.logical.optimizer.rbo.RuleBasedOptimizer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -10,14 +27,11 @@ public class LogicalOptimizerManager { private static final LogicalOptimizerManager instance = new LogicalOptimizerManager(); - private static final String REMOVE_NOT = "remove_not"; - - private static final String FILTER_PUSH_DOWN = "filter_push_down"; - - private static final String FILTER_FRAGMENT = "filter_fragment"; - private static final String RULE_BASE = "rbo"; + private static final String RULE_BASE_class = + "cn.edu.tsinghua.iginx.logical.optimizer.rbo.RuleBasedOptimizer"; + private LogicalOptimizerManager() {} public static LogicalOptimizerManager getInstance() { @@ -29,18 +43,20 @@ public Optimizer getOptimizer(String name) { return null; } LOGGER.info("use {} as logical optimizer.", name); - - switch (name) { - case REMOVE_NOT: - return RemoveNotOptimizer.getInstance(); - case FILTER_PUSH_DOWN: - return FilterPushDownOptimizer.getInstance(); - case FILTER_FRAGMENT: - return FilterFragmentOptimizer.getInstance(); - case RULE_BASE: - return RuleBasedOptimizer.getInstance(); - default: - throw new IllegalArgumentException(String.format("unknown logical optimizer: %s", name)); + try { + switch (name) { + case RULE_BASE: + return Optimizer.class + .getClassLoader() + .loadClass(RULE_BASE_class) + .asSubclass(Optimizer.class) + .newInstance(); + default: + throw new IllegalArgumentException(String.format("unknown logical optimizer: %s", name)); + } + } catch (ClassNotFoundException | InstantiationException | IllegalAccessException e) { + LOGGER.error("Cannot load class: {}", name, e); } + return null; } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/Optimizer.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/Optimizer.java index 79c416f9e4..3b839fbe04 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/Optimizer.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/Optimizer.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.logical.optimizer; import cn.edu.tsinghua.iginx.engine.shared.operator.Operator; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/core/Operand.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/core/Operand.java deleted file mode 100644 index f07e42d6e5..0000000000 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/core/Operand.java +++ /dev/null @@ -1,29 +0,0 @@ -package cn.edu.tsinghua.iginx.engine.logical.optimizer.core; - -import cn.edu.tsinghua.iginx.engine.shared.operator.Operator; -import java.util.List; - -public class Operand { - - private final Class clazz; - - private final List children; - - public static final Operand ANY_OPERAND = new Operand(null, null); - - public Operand(Class clazz, List children) { - this.clazz = clazz; - this.children = children; - } - - public List getChildren() { - return children; - } - - public boolean matches(Operator operator) { - if (this == ANY_OPERAND) { - return true; - } - return clazz.isInstance(operator); - } -} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/core/Planner.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/core/Planner.java deleted file mode 100644 index a7badfa467..0000000000 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/core/Planner.java +++ /dev/null @@ -1,30 +0,0 @@ -package cn.edu.tsinghua.iginx.engine.logical.optimizer.core; - -import cn.edu.tsinghua.iginx.engine.logical.optimizer.core.iterator.MatchOrder; -import cn.edu.tsinghua.iginx.engine.logical.optimizer.rules.Rule; -import cn.edu.tsinghua.iginx.engine.shared.operator.Operator; -import java.util.List; - -public interface Planner { - - // unban a single rule - void unbanRule(Rule rule); - - // unban a set of rules,e.g.PPD - void unbanRuleCollection(List rules); - - // set up the unoptimized query tree and initialize the optimization context - void setRoot(Operator root); - - // set the maximum number of rule matches - void setMatchLimit(int matchLimit); - - // set the maximum time limit for rule matching, unit: ms - void setLimitTime(long limitTime); - - // set the Rule matching order for this query tree, depth-first, leveled etc - void setMatchOrder(MatchOrder order); - - // get the optimized query tree - Operator findBest(); -} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/core/iterator/DeepFirstIterator.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/core/iterator/DeepFirstIterator.java deleted file mode 100644 index 72b9648e3d..0000000000 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/core/iterator/DeepFirstIterator.java +++ /dev/null @@ -1,26 +0,0 @@ -package cn.edu.tsinghua.iginx.engine.logical.optimizer.core.iterator; - -import cn.edu.tsinghua.iginx.engine.shared.operator.Operator; -import cn.edu.tsinghua.iginx.engine.shared.operator.visitor.DeepFirstQueueVisitor; -import java.util.Queue; - -public class DeepFirstIterator implements TreeIterator { - - private final Queue queue; - - public DeepFirstIterator(Operator root) { - DeepFirstQueueVisitor visitor = new DeepFirstQueueVisitor(); - root.accept(visitor); - queue = visitor.getQueue(); - } - - @Override - public boolean hasNext() { - return !queue.isEmpty(); - } - - @Override - public Operator next() { - return queue.poll(); - } -} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/core/iterator/MatchOrder.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/core/iterator/MatchOrder.java deleted file mode 100644 index d9803ad785..0000000000 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/core/iterator/MatchOrder.java +++ /dev/null @@ -1,11 +0,0 @@ -package cn.edu.tsinghua.iginx.engine.logical.optimizer.core.iterator; - -public enum MatchOrder { - Leveled, - - ReverseLeveled, - - DeepFirst, - - ReverseDeepFirst -} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/core/iterator/ReverseDeepFirstIterator.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/core/iterator/ReverseDeepFirstIterator.java deleted file mode 100644 index 8e7ed32a6a..0000000000 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/core/iterator/ReverseDeepFirstIterator.java +++ /dev/null @@ -1,30 +0,0 @@ -package cn.edu.tsinghua.iginx.engine.logical.optimizer.core.iterator; - -import cn.edu.tsinghua.iginx.engine.shared.operator.Operator; -import cn.edu.tsinghua.iginx.engine.shared.operator.visitor.DeepFirstQueueVisitor; -import java.util.Deque; -import java.util.LinkedList; -import java.util.Queue; - -public class ReverseDeepFirstIterator implements TreeIterator { - private final Deque stack = new LinkedList<>(); - - public ReverseDeepFirstIterator(Operator root) { - DeepFirstQueueVisitor visitor = new DeepFirstQueueVisitor(); - root.accept(visitor); - Queue forward = visitor.getQueue(); - while (!forward.isEmpty()) { - stack.push(forward.poll()); - } - } - - @Override - public boolean hasNext() { - return !stack.isEmpty(); - } - - @Override - public Operator next() { - return stack.poll(); - } -} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/core/iterator/ReverseLeveledIterator.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/core/iterator/ReverseLeveledIterator.java deleted file mode 100644 index 43c01a3bd2..0000000000 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/core/iterator/ReverseLeveledIterator.java +++ /dev/null @@ -1,27 +0,0 @@ -package cn.edu.tsinghua.iginx.engine.logical.optimizer.core.iterator; - -import cn.edu.tsinghua.iginx.engine.shared.operator.Operator; -import java.util.Deque; -import java.util.LinkedList; - -public class ReverseLeveledIterator implements TreeIterator { - - private final Deque stack = new LinkedList<>(); - - public ReverseLeveledIterator(Operator root) { - LeveledIterator it = new LeveledIterator(root); - while (it.hasNext()) { - stack.push(it.next()); - } - } - - @Override - public boolean hasNext() { - return !stack.isEmpty(); - } - - @Override - public Operator next() { - return stack.poll(); - } -} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/core/iterator/TreeIterator.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/core/iterator/TreeIterator.java deleted file mode 100644 index 885e0c88a8..0000000000 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/core/iterator/TreeIterator.java +++ /dev/null @@ -1,11 +0,0 @@ -package cn.edu.tsinghua.iginx.engine.logical.optimizer.core.iterator; - -import cn.edu.tsinghua.iginx.engine.shared.operator.Operator; -import java.util.Iterator; - -public interface TreeIterator extends Iterator { - - boolean hasNext(); - - Operator next(); -} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rbo/RBORuleCall.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rbo/RBORuleCall.java deleted file mode 100644 index 444d7b74be..0000000000 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rbo/RBORuleCall.java +++ /dev/null @@ -1,16 +0,0 @@ -package cn.edu.tsinghua.iginx.engine.logical.optimizer.rbo; - -import cn.edu.tsinghua.iginx.engine.logical.optimizer.core.RuleCall; -import cn.edu.tsinghua.iginx.engine.shared.operator.Operator; -import java.util.List; -import java.util.Map; - -public class RBORuleCall extends RuleCall { - - public RBORuleCall( - Operator subRoot, - Map parentIndexMap, - Map> childrenIndex) { - super(subRoot, parentIndexMap, childrenIndex); - } -} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rbo/RuleBasedOptimizer.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rbo/RuleBasedOptimizer.java deleted file mode 100644 index b122626f82..0000000000 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rbo/RuleBasedOptimizer.java +++ /dev/null @@ -1,23 +0,0 @@ -package cn.edu.tsinghua.iginx.engine.logical.optimizer.rbo; - -import cn.edu.tsinghua.iginx.engine.logical.optimizer.Optimizer; -import cn.edu.tsinghua.iginx.engine.logical.optimizer.core.Planner; -import cn.edu.tsinghua.iginx.engine.shared.operator.Operator; - -public class RuleBasedOptimizer implements Optimizer { - - private static final class InstanceHolder { - static final RuleBasedOptimizer instance = new RuleBasedOptimizer(); - } - - public static RuleBasedOptimizer getInstance() { - return InstanceHolder.instance; - } - - @Override - public Operator optimize(Operator root) { - Planner planner = new RuleBasedPlanner(); - planner.setRoot(root); - return planner.findBest(); - } -} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FilterPushDownSelectRule.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FilterPushDownSelectRule.java deleted file mode 100644 index ed3eb9558c..0000000000 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FilterPushDownSelectRule.java +++ /dev/null @@ -1,41 +0,0 @@ -package cn.edu.tsinghua.iginx.engine.logical.optimizer.rules; - -import cn.edu.tsinghua.iginx.engine.logical.optimizer.core.RuleCall; -import cn.edu.tsinghua.iginx.engine.logical.utils.LogicalFilterUtils; -import cn.edu.tsinghua.iginx.engine.shared.operator.Select; - -public class FilterPushDownSelectRule extends Rule { - private static final class InstanceHolder { - static final FilterPushDownSelectRule INSTANCE = new FilterPushDownSelectRule(); - } - - public static FilterPushDownSelectRule getInstance() { - return InstanceHolder.INSTANCE; - } - - protected FilterPushDownSelectRule() { - /* - * we want to match the topology like: - * Select - * | - * Select - */ - super("FilterPushDownSelectRule", operand(Select.class, operand(Select.class, any()))); - } - - @Override - public boolean matches(RuleCall call) { - return super.matches(call); - } - - @Override - public void onMatch(RuleCall call) { - // 应当把两个Select合并,用and连接 - Select select = (Select) call.getMatchedRoot(); - Select childSelect = (Select) call.getChildrenIndex().get(select).get(0); - - childSelect.setFilter( - LogicalFilterUtils.mergeFilter(select.getFilter(), childSelect.getFilter())); - call.transformTo(childSelect); - } -} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/Rule.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/Rule.java deleted file mode 100644 index cac1297220..0000000000 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/Rule.java +++ /dev/null @@ -1,63 +0,0 @@ -package cn.edu.tsinghua.iginx.engine.logical.optimizer.rules; - -import cn.edu.tsinghua.iginx.engine.logical.optimizer.core.Operand; -import cn.edu.tsinghua.iginx.engine.logical.optimizer.core.RuleCall; -import cn.edu.tsinghua.iginx.engine.shared.operator.Operator; -import java.util.Arrays; - -public abstract class Rule { - - private final String ruleName; - - /** operand describes the local topology we want to match in this rule */ - private final Operand operand; - - private final RuleStrategy strategy; - - protected Rule(String ruleName, Operand operand) { - this.ruleName = ruleName; - this.operand = operand; - this.strategy = RuleStrategy.FIXED_POINT; - } - - protected Rule(String ruleName, Operand operand, RuleStrategy strategy) { - this.ruleName = ruleName; - this.operand = operand; - this.strategy = strategy; - } - - public String getRuleName() { - return ruleName; - } - - public Operand getOperand() { - return operand; - } - - public RuleStrategy getStrategy() { - return strategy; - } - - /** - * Returns whether this rule could possibly match the given operands. - * - *

This method is an opportunity to apply side-conditions to a rule. The Planner calls this - * method after matching all operands of the rule, and before calling method onMatch. - * - *

The default implementation of this method returns true. - */ - public boolean matches(RuleCall call) { - return true; - } - - /** This method is used to modify the local topology after rule matching. */ - public abstract void onMatch(RuleCall call); - - public static Operand any() { - return Operand.ANY_OPERAND; - } - - public static Operand operand(Class clazz, Operand... children) { - return new Operand(clazz, Arrays.asList(children)); - } -} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/RuleStrategy.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/RuleStrategy.java deleted file mode 100644 index 4fa03e64f2..0000000000 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/RuleStrategy.java +++ /dev/null @@ -1,6 +0,0 @@ -package cn.edu.tsinghua.iginx.engine.logical.optimizer.rules; - -public enum RuleStrategy { - FIXED_POINT, // 一直执行直到不再有匹配 - ONCE // 只执行一次 -} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/utils/LogicalFilterUtils.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/utils/LogicalFilterUtils.java index 3bac0c6368..7959bc73a9 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/utils/LogicalFilterUtils.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/utils/LogicalFilterUtils.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.logical.utils; import cn.edu.tsinghua.iginx.engine.physical.memory.execute.utils.ExprUtils; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/utils/MetaUtils.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/utils/MetaUtils.java index edb7add95c..5ab7f30758 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/utils/MetaUtils.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/utils/MetaUtils.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.logical.utils; import static cn.edu.tsinghua.iginx.metadata.utils.FragmentUtils.keyFromColumnsIntervalToKeyInterval; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/utils/OperatorUtils.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/utils/OperatorUtils.java index a8b963337b..76000d76ff 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/utils/OperatorUtils.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/utils/OperatorUtils.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.logical.utils; import static cn.edu.tsinghua.iginx.engine.physical.memory.execute.utils.FilterUtils.*; @@ -24,11 +42,11 @@ import cn.edu.tsinghua.iginx.engine.shared.source.OperatorSource; import cn.edu.tsinghua.iginx.engine.shared.source.Source; import cn.edu.tsinghua.iginx.engine.shared.source.SourceType; +import cn.edu.tsinghua.iginx.utils.Pair; import cn.edu.tsinghua.iginx.utils.StringUtils; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.stream.Collectors; public class OperatorUtils { @@ -47,7 +65,7 @@ public static Operator joinOperatorsByTime(List operators) { return joinOperators(operators, KEY); } - public static Operator joinOperators(List operators, String joinBy) { + public static Operator joinOperators(List operators, String joinBy) { if (operators == null || operators.isEmpty()) return null; if (operators.size() == 1) return operators.get(0); Operator join = operators.get(0); @@ -92,7 +110,7 @@ public static List findPathList(Operator operator) { } } - return pathList.stream().distinct().collect(Collectors.toList()); + return pathList.stream().distinct().sorted().collect(Collectors.toList()); } public static void findProjectOperators(List projectOperatorList, Operator operator) { @@ -316,7 +334,7 @@ private static Operator pushDownApply(Operator root, List correlatedVari root = new Rename( new OperatorSource(pushDownApply(apply, correlatedVariables)), - rename.getAliasMap(), + rename.getAliasList(), ignorePatterns); break; case CrossJoin: @@ -587,8 +605,8 @@ public static List getPatternFromOperatorChildren( Operator visitedOperator = visitedOperators.get(i); if (visitedOperator.getType() == OperatorType.Rename) { Rename rename = (Rename) visitedOperator; - Map aliasMap = rename.getAliasMap(); - patterns = renamePattern(aliasMap, patterns); + List> aliasList = rename.getAliasList(); + patterns = renamePattern(aliasList, patterns); } } return patterns; @@ -630,17 +648,18 @@ private static boolean isPatternMatched(String patternA, String patternB) { /** * 正向重命名模式列表中的pattern,将key中的pattern替换为value中的pattern * - * @param aliasMap 重命名规则, key为旧模式,value为新模式 + * @param aliasList 重命名规则, key为旧模式,value为新模式 * @param patterns 要重命名的模式列表 * @return */ - private static List renamePattern(Map aliasMap, List patterns) { + private static List renamePattern( + List> aliasList, List patterns) { List renamedPatterns = new ArrayList<>(); for (String pattern : patterns) { boolean matched = false; - for (Map.Entry entry : aliasMap.entrySet()) { - String oldPattern = entry.getKey().replace("*", "(.*)"); - String newPattern = entry.getValue().replace("*", "$1"); + for (Pair pair : aliasList) { + String oldPattern = pair.k.replace("*", "(.*)"); + String newPattern = pair.v.replace("*", "$1"); if (pattern.matches(oldPattern)) { if (newPattern.contains("$1") && !oldPattern.contains("*")) { newPattern = newPattern.replace("$1", "*"); @@ -650,12 +669,12 @@ private static List renamePattern(Map aliasMap, List. + */ + package cn.edu.tsinghua.iginx.engine.logical.utils; import cn.edu.tsinghua.iginx.metadata.entity.ColumnsInterval; +import cn.edu.tsinghua.iginx.utils.Pair; import cn.edu.tsinghua.iginx.utils.StringUtils; import java.util.Collection; import java.util.List; -import java.util.Map; import java.util.stream.Collectors; public class PathUtils { @@ -45,21 +63,21 @@ public static ColumnsInterval addSuffix(ColumnsInterval columnsInterval) { /** * 反向重命名模式列表中的模式 * - * @param aliasMap 重命名规则, key为旧模式,value为新模式,在这里我们要将新模式恢复为旧模式 + * @param aliasList 重命名规则, key为旧模式,value为新模式,在这里我们要将新模式恢复为旧模式 * @param patterns 要重命名的模式列表 * @return 重命名后的模式列表 */ public static List recoverRenamedPatterns( - Map aliasMap, List patterns) { + List> aliasList, List patterns) { return patterns.stream() - .map(pattern -> recoverRenamedPattern(aliasMap, pattern)) + .map(pattern -> recoverRenamedPattern(aliasList, pattern)) .collect(Collectors.toList()); } - public static String recoverRenamedPattern(Map aliasMap, String pattern) { - for (Map.Entry entry : aliasMap.entrySet()) { - String oldPattern = entry.getKey().replace("*", "$1"); // 通配符转换为正则的捕获组 - String newPattern = entry.getValue().replace("*", "(.*)"); // 使用反向引用保留原始匹配的部分 + public static String recoverRenamedPattern(List> aliasList, String pattern) { + for (Pair pair : aliasList) { + String oldPattern = pair.k.replace("*", "$1"); // 通配符转换为正则的捕获组 + String newPattern = pair.v.replace("*", "(.*)"); // 使用反向引用保留原始匹配的部分 if (pattern.matches(newPattern)) { // 如果旧模式中有通配符,但是新模式中没有,我们需要将新模式中的捕获组替换为通配符 if (oldPattern.contains("$1") && !newPattern.contains("*")) { @@ -67,9 +85,9 @@ public static String recoverRenamedPattern(Map aliasMap, String } return pattern.replaceAll(newPattern, oldPattern); } else if (newPattern.equals(pattern)) { - return entry.getKey(); + return pair.k; } else if (pattern.contains(".*") && newPattern.matches(StringUtils.reformatPath(pattern))) { - return entry.getKey(); + return pair.k; } } return pattern; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/PhysicalEngine.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/PhysicalEngine.java index 06cbb51471..ef194b1faf 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/PhysicalEngine.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/PhysicalEngine.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/PhysicalEngineImpl.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/PhysicalEngineImpl.java index df75c44cbf..e45dd0aa69 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/PhysicalEngineImpl.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/PhysicalEngineImpl.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/exception/InvalidOperatorParameterException.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/exception/InvalidOperatorParameterException.java index f7cb6db2ab..cb1b115f25 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/exception/InvalidOperatorParameterException.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/exception/InvalidOperatorParameterException.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.exception; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/exception/NonExecutablePhysicalTaskException.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/exception/NonExecutablePhysicalTaskException.java index 17376e14e1..fb99082c9e 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/exception/NonExecutablePhysicalTaskException.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/exception/NonExecutablePhysicalTaskException.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.exception; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/exception/NonExistedStorageException.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/exception/NonExistedStorageException.java index 76c949f43f..b922e9a8dd 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/exception/NonExistedStorageException.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/exception/NonExistedStorageException.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.exception; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/exception/NotSupportedOperatorException.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/exception/NotSupportedOperatorException.java index d3d9c6423e..dea19884ff 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/exception/NotSupportedOperatorException.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/exception/NotSupportedOperatorException.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.exception; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/exception/PhysicalException.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/exception/PhysicalException.java index ebd5e648b2..a0bbc0c89b 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/exception/PhysicalException.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/exception/PhysicalException.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.exception; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/exception/PhysicalRuntimeException.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/exception/PhysicalRuntimeException.java index 9d150f23a8..4a8b9e5609 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/exception/PhysicalRuntimeException.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/exception/PhysicalRuntimeException.java @@ -1,17 +1,19 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.exception; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/exception/PhysicalTaskExecuteFailureException.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/exception/PhysicalTaskExecuteFailureException.java index 4343e9cd2c..8ac03e33eb 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/exception/PhysicalTaskExecuteFailureException.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/exception/PhysicalTaskExecuteFailureException.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.exception; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/exception/RowFetchException.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/exception/RowFetchException.java index 9a9f3d6332..a75502f33c 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/exception/RowFetchException.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/exception/RowFetchException.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.exception; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/exception/StorageInitializationException.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/exception/StorageInitializationException.java index 960cd4ff52..fbc30f995e 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/exception/StorageInitializationException.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/exception/StorageInitializationException.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.exception; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/exception/TooManyPhysicalTasksException.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/exception/TooManyPhysicalTasksException.java index 9bb410a8d3..353f375ad2 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/exception/TooManyPhysicalTasksException.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/exception/TooManyPhysicalTasksException.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.exception; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/exception/UnexpectedOperatorException.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/exception/UnexpectedOperatorException.java index fa948c5bfa..f0c22ff9d5 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/exception/UnexpectedOperatorException.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/exception/UnexpectedOperatorException.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.exception; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/exception/UnimplementedOperatorException.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/exception/UnimplementedOperatorException.java index 0d0ee9e18d..ab7679d29f 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/exception/UnimplementedOperatorException.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/exception/UnimplementedOperatorException.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.exception; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/MemoryPhysicalTaskDispatcher.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/MemoryPhysicalTaskDispatcher.java index f4475b0027..bed478bbb9 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/MemoryPhysicalTaskDispatcher.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/MemoryPhysicalTaskDispatcher.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.memory; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/OperatorMemoryExecutor.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/OperatorMemoryExecutor.java index 84ed039e55..cddce86a1e 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/OperatorMemoryExecutor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/OperatorMemoryExecutor.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.memory.execute; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/OperatorMemoryExecutorFactory.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/OperatorMemoryExecutorFactory.java index c6dc3a01ba..d2ba5933f1 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/OperatorMemoryExecutorFactory.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/OperatorMemoryExecutorFactory.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.memory.execute; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/Table.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/Table.java index 7f4be1bf1e..133123511c 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/Table.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/Table.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.memory.execute; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/naive/NaiveOperatorMemoryExecutor.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/naive/NaiveOperatorMemoryExecutor.java index c13ed3f56f..6993594cce 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/naive/NaiveOperatorMemoryExecutor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/naive/NaiveOperatorMemoryExecutor.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.memory.execute.naive; @@ -29,7 +28,7 @@ import static cn.edu.tsinghua.iginx.engine.physical.memory.execute.utils.RowUtils.getSamePathWithSpecificPrefix; import static cn.edu.tsinghua.iginx.engine.physical.memory.execute.utils.RowUtils.isValueEqualRow; import static cn.edu.tsinghua.iginx.engine.physical.memory.execute.utils.RowUtils.removeDuplicateRows; -import static cn.edu.tsinghua.iginx.engine.shared.Constants.KEY; +import static cn.edu.tsinghua.iginx.engine.shared.Constants.*; import static cn.edu.tsinghua.iginx.engine.shared.function.FunctionUtils.isCanUseSetQuantifierFunction; import static cn.edu.tsinghua.iginx.engine.shared.function.system.utils.ValueUtils.getHash; import static cn.edu.tsinghua.iginx.sql.SQLConstant.DOT; @@ -90,15 +89,7 @@ import cn.edu.tsinghua.iginx.utils.Pair; import cn.edu.tsinghua.iginx.utils.StringUtils; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.TreeMap; +import java.util.*; import java.util.regex.Pattern; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -201,7 +192,7 @@ private Table transformToTable(RowStream stream) throws PhysicalException { return new Table(header, rows); } - private RowStream executeProject(Project project, Table table) throws PhysicalException { + private RowStream executeProject(Project project, Table table) { List patterns = project.getPatterns(); Header header = table.getHeader(); List targetFields = new ArrayList<>(); @@ -253,7 +244,7 @@ private RowStream executeSort(Sort sort, Table table) throws PhysicalException { return table; } - private RowStream executeLimit(Limit limit, Table table) throws PhysicalException { + private RowStream executeLimit(Limit limit, Table table) { int rowSize = table.getRowSize(); Header header = table.getHeader(); List rows = new ArrayList<>(); @@ -276,12 +267,20 @@ private RowStream executeDownsample(Downsample downsample, Table table) throws P List rows = table.getRows(); long bias = downsample.getKeyRange().getActualBeginKey(); long endKey = downsample.getKeyRange().getActualEndKey(); + if (downsample.notSetInterval()) { + if (table.getRowSize() <= 0) { + return Table.EMPTY_TABLE; + } + bias = table.getRow(0).getKey(); + endKey = table.getRow(table.getRowSize() - 1).getKey(); + } long precision = downsample.getPrecision(); long slideDistance = downsample.getSlideDistance(); // startKey + (n - 1) * slideDistance + precision - 1 >= endKey long n = (int) (Math.ceil((double) (endKey - bias - precision + 1) / slideDistance) + 1); List tableList = new ArrayList<>(); + boolean firstCol = true; for (FunctionCall functionCall : downsample.getFunctionCallList()) { SetMappingFunction function = (SetMappingFunction) functionCall.getFunction(); FunctionParams params = functionCall.getParams(); @@ -309,10 +308,12 @@ private RowStream executeDownsample(Downsample downsample, Table table) throws P } } } - List> transformedRawRows = new ArrayList<>(); + // < row> + List, Row>> transformedRawRows = new ArrayList<>(); try { for (Map.Entry> entry : groups.entrySet()) { - long time = entry.getKey(); + long windowStartKey = entry.getKey(); + long windowEndKey = windowStartKey + precision - 1; List group = entry.getValue(); if (params.isDistinct()) { @@ -329,7 +330,7 @@ private RowStream executeDownsample(Downsample downsample, Table table) throws P Row row = function.transform(new Table(header, group), params); if (row != null) { - transformedRawRows.add(new Pair<>(time, row)); + transformedRawRows.add(new Pair<>(new Pair<>(windowStartKey, windowEndKey), row)); } } } catch (Exception e) { @@ -340,19 +341,36 @@ private RowStream executeDownsample(Downsample downsample, Table table) throws P if (transformedRawRows.size() == 0) { return Table.EMPTY_TABLE; } - Header newHeader = new Header(Field.KEY, transformedRawRows.get(0).v.getHeader().getFields()); + + // 只让第一张表保留 window_start, window_end 列,这样按key join后无需删除重复列 + List fields = transformedRawRows.get(0).v.getHeader().getFields(); + if (firstCol) { + fields.add(0, new Field(WINDOW_START_COL, DataType.LONG)); + fields.add(1, new Field(WINDOW_END_COL, DataType.LONG)); + } + Header newHeader = new Header(Field.KEY, fields); List transformedRows = new ArrayList<>(); - for (Pair pair : transformedRawRows) { - transformedRows.add(new Row(newHeader, pair.k, pair.v.getValues())); + Object[] values = new Object[transformedRawRows.get(0).v.getValues().length + 2]; + for (Pair, Row> pair : transformedRawRows) { + if (firstCol) { + values[0] = pair.k.k; + values[1] = pair.k.v; + System.arraycopy(pair.v.getValues(), 0, values, 2, pair.v.getValues().length); + transformedRows.add(new Row(newHeader, pair.k.k, values)); + } else { + transformedRows.add(new Row(newHeader, pair.k.k, pair.v.getValues())); + } + values = new Object[transformedRawRows.get(0).v.getValues().length + 2]; } tableList.add(new Table(newHeader, transformedRows)); + firstCol = false; } + // key = window_start,而每个窗口长度一样,因此多表中key相同的列就是同一个窗口的结果,可以按key join return RowUtils.joinMultipleTablesByKey(tableList); } - private RowStream executeRowTransform(RowTransform rowTransform, Table table) - throws PhysicalException { + private RowStream executeRowTransform(RowTransform rowTransform, Table table) { List> list = new ArrayList<>(); rowTransform .getFunctionCallList() @@ -454,12 +472,10 @@ private RowStream executeMappingTransform(MappingTransform mappingTransform, Tab return RowUtils.calMappingTransform(table, functionCallList); } - private RowStream executeRename(Rename rename, Table table) throws PhysicalException { + private RowStream executeRename(Rename rename, Table table) { Header header = table.getHeader(); - Map aliasMap = rename.getAliasMap(); - - List ignorePatterns = rename.getIgnorePatterns(); - Header newHeader = header.renamedHeader(aliasMap, ignorePatterns); + List> aliasList = rename.getAliasList(); + Header newHeader = header.renamedHeader(aliasList, rename.getIgnorePatterns()); List rows = new ArrayList<>(); table @@ -476,8 +492,7 @@ private RowStream executeRename(Rename rename, Table table) throws PhysicalExcep return new Table(newHeader, rows); } - private RowStream executeAddSchemaPrefix(AddSchemaPrefix addSchemaPrefix, Table table) - throws PhysicalException { + private RowStream executeAddSchemaPrefix(AddSchemaPrefix addSchemaPrefix, Table table) { Header header = table.getHeader(); String schemaPrefix = addSchemaPrefix.getSchemaPrefix(); @@ -616,7 +631,7 @@ private RowStream executeJoin(Join join, Table tableA, Table tableB) throws Phys // 检查时间戳 if (!headerA.hasKey() || !headerB.hasKey()) { throw new InvalidOperatorParameterException( - "row streams for join operator by time should have timestamp."); + "row streams for join operator by key should have key."); } List newFields = new ArrayList<>(); newFields.addAll(headerA.getFields()); @@ -712,8 +727,7 @@ private RowStream executeJoin(Join join, Table tableA, Table tableB) throws Phys } } - private RowStream executeCrossJoin(CrossJoin crossJoin, Table tableA, Table tableB) - throws PhysicalException { + private RowStream executeCrossJoin(CrossJoin crossJoin, Table tableA, Table tableB) { Header newHeader = HeaderUtils.constructNewHead( tableA.getHeader(), tableB.getHeader(), crossJoin.getPrefixA(), crossJoin.getPrefixB()); diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/AddSchemaPrefixLazyStream.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/AddSchemaPrefixLazyStream.java index 04adf076bd..2630aa44cc 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/AddSchemaPrefixLazyStream.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/AddSchemaPrefixLazyStream.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.physical.memory.execute.stream; import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/BinaryLazyStream.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/BinaryLazyStream.java index 1dac6f588c..1e0a7d7d83 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/BinaryLazyStream.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/BinaryLazyStream.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.memory.execute.stream; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/CrossJoinLazyStream.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/CrossJoinLazyStream.java index 1ceb2b6ef4..a623342a30 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/CrossJoinLazyStream.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/CrossJoinLazyStream.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.physical.memory.execute.stream; import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/DistinctLazyStream.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/DistinctLazyStream.java index c4b552e0d9..f76ad15dec 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/DistinctLazyStream.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/DistinctLazyStream.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.physical.memory.execute.stream; import static cn.edu.tsinghua.iginx.engine.physical.memory.execute.utils.RowUtils.isEqualRow; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/DownsampleLazyStream.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/DownsampleLazyStream.java index 73eb34f42a..4291b20c29 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/DownsampleLazyStream.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/DownsampleLazyStream.java @@ -1,23 +1,25 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.memory.execute.stream; +import static cn.edu.tsinghua.iginx.engine.shared.Constants.WINDOW_END_COL; +import static cn.edu.tsinghua.iginx.engine.shared.Constants.WINDOW_START_COL; + import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalTaskExecuteFailureException; import cn.edu.tsinghua.iginx.engine.physical.memory.execute.Table; @@ -31,6 +33,7 @@ import cn.edu.tsinghua.iginx.engine.shared.function.FunctionParams; import cn.edu.tsinghua.iginx.engine.shared.function.SetMappingFunction; import cn.edu.tsinghua.iginx.engine.shared.operator.Downsample; +import cn.edu.tsinghua.iginx.thrift.DataType; import java.util.ArrayList; import java.util.List; @@ -82,7 +85,7 @@ private Row loadNext() throws PhysicalException { return nextTarget; } Row row = null; - long timestamp = 0; + long windowStartKey = 0; long bias = downsample.getKeyRange().getActualBeginKey(); long endKey = downsample.getKeyRange().getActualEndKey(); long precision = downsample.getPrecision(); @@ -90,9 +93,9 @@ private Row loadNext() throws PhysicalException { // startKey + (n - 1) * slideDistance + precision - 1 >= endKey int n = (int) (Math.ceil((double) (endKey - bias - precision + 1) / slideDistance) + 1); while (row == null && wrapper.hasNext()) { - timestamp = wrapper.nextTimestamp() - (wrapper.nextTimestamp() - bias) % precision; + windowStartKey = wrapper.nextTimestamp() - (wrapper.nextTimestamp() - bias) % precision; List rows = new ArrayList<>(); - while (wrapper.hasNext() && wrapper.nextTimestamp() < timestamp + precision) { + while (wrapper.hasNext() && wrapper.nextTimestamp() < windowStartKey + precision) { rows.add(wrapper.next()); } Table table = new Table(rows.get(0).getHeader(), rows); @@ -111,9 +114,18 @@ private Row loadNext() throws PhysicalException { } row = RowUtils.combineMultipleColumns(subRowList); } - return row == null - ? null - : new Row(new Header(Field.KEY, row.getHeader().getFields()), timestamp, row.getValues()); + if (row == null) { + return null; + } else { + List fields = row.getHeader().getFields(); + fields.add(0, new Field(WINDOW_START_COL, DataType.LONG)); + fields.add(1, new Field(WINDOW_END_COL, DataType.LONG)); + Object[] values = new Object[row.getValues().length + 2]; + values[0] = windowStartKey; + values[1] = windowStartKey + precision - 1; + System.arraycopy(row.getValues(), 0, values, 2, row.getValues().length); + return new Row(new Header(Field.KEY, fields), windowStartKey, values); + } } @Override diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/EmptyRowStream.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/EmptyRowStream.java index b0d7e3bf30..2c9bc61da9 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/EmptyRowStream.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/EmptyRowStream.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.memory.execute.stream; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/ExceptLazyStream.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/ExceptLazyStream.java index 0684c46484..00555aff9e 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/ExceptLazyStream.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/ExceptLazyStream.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.physical.memory.execute.stream; import static cn.edu.tsinghua.iginx.engine.physical.memory.execute.utils.HeaderUtils.checkHeadersComparable; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/GroupByLazyStream.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/GroupByLazyStream.java index 9cc825365a..66701761e5 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/GroupByLazyStream.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/GroupByLazyStream.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.physical.memory.execute.stream; import cn.edu.tsinghua.iginx.conf.Config; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/HashInnerJoinLazyStream.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/HashInnerJoinLazyStream.java index acb059fe06..a3ead0002e 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/HashInnerJoinLazyStream.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/HashInnerJoinLazyStream.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.physical.memory.execute.stream; import static cn.edu.tsinghua.iginx.engine.physical.memory.execute.utils.HeaderUtils.calculateHashJoinPath; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/HashMarkJoinLazyStream.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/HashMarkJoinLazyStream.java index a6357fe8a0..11512a91db 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/HashMarkJoinLazyStream.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/HashMarkJoinLazyStream.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.physical.memory.execute.stream; import static cn.edu.tsinghua.iginx.engine.physical.memory.execute.utils.FilterUtils.getJoinPathFromFilter; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/HashOuterJoinLazyStream.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/HashOuterJoinLazyStream.java index dd273d51b9..42a0e6be87 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/HashOuterJoinLazyStream.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/HashOuterJoinLazyStream.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.physical.memory.execute.stream; import static cn.edu.tsinghua.iginx.engine.physical.memory.execute.utils.HeaderUtils.calculateHashJoinPath; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/HashSingleJoinLazyStream.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/HashSingleJoinLazyStream.java index 73ab9d4297..fb231b3e66 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/HashSingleJoinLazyStream.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/HashSingleJoinLazyStream.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.physical.memory.execute.stream; import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/IntersectLazyStream.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/IntersectLazyStream.java index 8a589d990e..eb9f60ded1 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/IntersectLazyStream.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/IntersectLazyStream.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.physical.memory.execute.stream; import static cn.edu.tsinghua.iginx.engine.physical.memory.execute.utils.HeaderUtils.checkHeadersComparable; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/JoinLazyStream.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/JoinLazyStream.java index 03dbb9dcfe..ad30e3cce9 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/JoinLazyStream.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/JoinLazyStream.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.memory.execute.stream; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/LimitLazyStream.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/LimitLazyStream.java index 4d18e180d2..e03a7266ce 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/LimitLazyStream.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/LimitLazyStream.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.memory.execute.stream; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/MappingTransformLazyStream.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/MappingTransformLazyStream.java index fc78d98b5b..f3f741b7d9 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/MappingTransformLazyStream.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/MappingTransformLazyStream.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.memory.execute.stream; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/NestedLoopInnerJoinLazyStream.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/NestedLoopInnerJoinLazyStream.java index 609cdd2d4c..1564924fc2 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/NestedLoopInnerJoinLazyStream.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/NestedLoopInnerJoinLazyStream.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.physical.memory.execute.stream; import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/NestedLoopMarkJoinLazyStream.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/NestedLoopMarkJoinLazyStream.java index b8e27700f2..868299842d 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/NestedLoopMarkJoinLazyStream.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/NestedLoopMarkJoinLazyStream.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.physical.memory.execute.stream; import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/NestedLoopOuterJoinLazyStream.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/NestedLoopOuterJoinLazyStream.java index 0e93dc10d4..d8d5cfee7c 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/NestedLoopOuterJoinLazyStream.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/NestedLoopOuterJoinLazyStream.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.physical.memory.execute.stream; import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/NestedLoopSingleJoinLazyStream.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/NestedLoopSingleJoinLazyStream.java index 630482c6d8..c29b05175b 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/NestedLoopSingleJoinLazyStream.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/NestedLoopSingleJoinLazyStream.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.physical.memory.execute.stream; import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/PathUnionLazyStream.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/PathUnionLazyStream.java index 125eed1251..c86477d06b 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/PathUnionLazyStream.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/PathUnionLazyStream.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.memory.execute.stream; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/ProjectLazyStream.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/ProjectLazyStream.java index 717c547b32..a74ecc5a7a 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/ProjectLazyStream.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/ProjectLazyStream.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.memory.execute.stream; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/RenameLazyStream.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/RenameLazyStream.java index b61aeff0a4..eddb89df0a 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/RenameLazyStream.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/RenameLazyStream.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.physical.memory.execute.stream; import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; @@ -5,7 +23,6 @@ import cn.edu.tsinghua.iginx.engine.shared.data.read.Row; import cn.edu.tsinghua.iginx.engine.shared.data.read.RowStream; import cn.edu.tsinghua.iginx.engine.shared.operator.Rename; -import java.util.Map; public class RenameLazyStream extends UnaryLazyStream { @@ -22,9 +39,7 @@ public RenameLazyStream(Rename rename, RowStream stream) { public Header getHeader() throws PhysicalException { if (header == null) { Header header = stream.getHeader(); - Map aliasMap = rename.getAliasMap(); - - this.header = header.renamedHeader(aliasMap, rename.getIgnorePatterns()); + this.header = header.renamedHeader(rename.getAliasList(), rename.getIgnorePatterns()); } return header; } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/ReorderLazyStream.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/ReorderLazyStream.java index e5389716c6..6d8d271ac0 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/ReorderLazyStream.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/ReorderLazyStream.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.physical.memory.execute.stream; import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/RowTransformLazyStream.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/RowTransformLazyStream.java index 6e936ef514..78866b5140 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/RowTransformLazyStream.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/RowTransformLazyStream.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.memory.execute.stream; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/SelectLazyStream.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/SelectLazyStream.java index ced8417777..90eac6eaf7 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/SelectLazyStream.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/SelectLazyStream.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.memory.execute.stream; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/SetTransformLazyStream.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/SetTransformLazyStream.java index 1a8ff74354..d302703516 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/SetTransformLazyStream.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/SetTransformLazyStream.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.memory.execute.stream; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/SortLazyStream.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/SortLazyStream.java index 9d4132f7a8..264f623e0c 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/SortLazyStream.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/SortLazyStream.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.memory.execute.stream; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/SortedMergeInnerJoinLazyStream.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/SortedMergeInnerJoinLazyStream.java index 9b20e0a5e6..07aaced97c 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/SortedMergeInnerJoinLazyStream.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/SortedMergeInnerJoinLazyStream.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.physical.memory.execute.stream; import cn.edu.tsinghua.iginx.engine.physical.exception.InvalidOperatorParameterException; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/SortedMergeOuterJoinLazyStream.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/SortedMergeOuterJoinLazyStream.java index d30e3c1776..78b1062617 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/SortedMergeOuterJoinLazyStream.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/SortedMergeOuterJoinLazyStream.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.physical.memory.execute.stream; import cn.edu.tsinghua.iginx.engine.physical.exception.InvalidOperatorParameterException; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/StreamOperatorMemoryExecutor.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/StreamOperatorMemoryExecutor.java index b126cbcfc7..eb2c151449 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/StreamOperatorMemoryExecutor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/StreamOperatorMemoryExecutor.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.memory.execute.stream; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/UnaryLazyStream.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/UnaryLazyStream.java index b9f5e3ea15..2e53bc117f 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/UnaryLazyStream.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/UnaryLazyStream.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.memory.execute.stream; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/UnionAllLazyStream.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/UnionAllLazyStream.java index b3eb01c643..161cce5fe5 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/UnionAllLazyStream.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/UnionAllLazyStream.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.physical.memory.execute.stream; import static cn.edu.tsinghua.iginx.engine.physical.memory.execute.utils.HeaderUtils.checkHeadersComparable; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/UnionDistinctLazyStream.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/UnionDistinctLazyStream.java index 6ca096bc33..0158cf7bac 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/UnionDistinctLazyStream.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/UnionDistinctLazyStream.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.physical.memory.execute.stream; import static cn.edu.tsinghua.iginx.engine.physical.memory.execute.utils.HeaderUtils.checkHeadersComparable; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/ValueToSelectedPathLazyStream.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/ValueToSelectedPathLazyStream.java index e3b45a4ac1..4d7d97218b 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/ValueToSelectedPathLazyStream.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/ValueToSelectedPathLazyStream.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.physical.memory.execute.stream; import static cn.edu.tsinghua.iginx.sql.SQLConstant.DOT; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/utils/ExprUtils.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/utils/ExprUtils.java index 7db1d54772..161ce0d8de 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/utils/ExprUtils.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/utils/ExprUtils.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.physical.memory.execute.utils; import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/utils/FilterUtils.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/utils/FilterUtils.java index d78f01b032..f857ea1cc8 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/utils/FilterUtils.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/utils/FilterUtils.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.memory.execute.utils; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/utils/GroupByKey.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/utils/GroupByKey.java index 2b6c4e4e68..cc01bbcae2 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/utils/GroupByKey.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/utils/GroupByKey.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.physical.memory.execute.utils; import java.util.ArrayList; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/utils/HeaderUtils.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/utils/HeaderUtils.java index f0da9b0002..b569ac7758 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/utils/HeaderUtils.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/utils/HeaderUtils.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.physical.memory.execute.utils; import static cn.edu.tsinghua.iginx.engine.shared.function.system.utils.ValueUtils.isNumericType; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/utils/RowUtils.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/utils/RowUtils.java index 0a7b2c54ae..07a97f5d2e 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/utils/RowUtils.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/utils/RowUtils.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.memory.execute.utils; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/queue/MemoryPhysicalTaskQueue.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/queue/MemoryPhysicalTaskQueue.java index 2f5b8e7705..27d824ada3 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/queue/MemoryPhysicalTaskQueue.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/queue/MemoryPhysicalTaskQueue.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.memory.queue; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/queue/MemoryPhysicalTaskQueueImpl.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/queue/MemoryPhysicalTaskQueueImpl.java index 17952fad88..65005348d3 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/queue/MemoryPhysicalTaskQueueImpl.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/queue/MemoryPhysicalTaskQueueImpl.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.memory.queue; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/optimizer/PhysicalOptimizer.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/optimizer/PhysicalOptimizer.java index 7d7cdeb0bf..8ceffa6d32 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/optimizer/PhysicalOptimizer.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/optimizer/PhysicalOptimizer.java @@ -1,29 +1,26 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.optimizer; -import cn.edu.tsinghua.iginx.engine.physical.optimizer.rule.Rule; import cn.edu.tsinghua.iginx.engine.physical.task.PhysicalTask; import cn.edu.tsinghua.iginx.engine.shared.RequestContext; import cn.edu.tsinghua.iginx.engine.shared.constraint.ConstraintManager; import cn.edu.tsinghua.iginx.engine.shared.operator.Operator; -import java.util.Collection; public interface PhysicalOptimizer { @@ -32,6 +29,4 @@ public interface PhysicalOptimizer { ConstraintManager getConstraintManager(); ReplicaDispatcher getReplicaDispatcher(); - - void setRules(Collection rules); } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/optimizer/PhysicalOptimizerManager.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/optimizer/PhysicalOptimizerManager.java index 4e14ba2c04..e335959ec0 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/optimizer/PhysicalOptimizerManager.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/optimizer/PhysicalOptimizerManager.java @@ -1,27 +1,23 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.optimizer; -import cn.edu.tsinghua.iginx.engine.physical.optimizer.naive.NaivePhysicalOptimizer; -import cn.edu.tsinghua.iginx.engine.physical.optimizer.rule.Rule; -import java.util.Collection; -import java.util.Collections; +import cn.edu.tsinghua.iginx.engine.logical.optimizer.Optimizer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,6 +27,9 @@ public class PhysicalOptimizerManager { private static final String NAIVE = "naive"; + private static final String NAIVE_CLASS = + "cn.edu.tsinghua.iginx.physical.optimizer.naive.NaivePhysicalOptimizer"; + private static final PhysicalOptimizerManager INSTANCE = new PhysicalOptimizerManager(); private PhysicalOptimizerManager() {} @@ -44,21 +43,30 @@ public PhysicalOptimizer getOptimizer(String name) { return null; } PhysicalOptimizer optimizer = null; - switch (name) { - case NAIVE: - LOGGER.info("use {} as physical optimizer.", name); - optimizer = NaivePhysicalOptimizer.getInstance(); - break; - default: - LOGGER.error("unknown physical optimizer {}, use {} as default.", name, NAIVE); - optimizer = NaivePhysicalOptimizer.getInstance(); + try { + switch (name) { + case NAIVE: + LOGGER.info("use {} as physical optimizer.", name); + optimizer = + Optimizer.class + .getClassLoader() + .loadClass(NAIVE_CLASS) + .asSubclass(PhysicalOptimizer.class) + .newInstance(); + break; + default: + LOGGER.error("unknown physical optimizer {}, use {} as default.", name, NAIVE); + optimizer = + Optimizer.class + .getClassLoader() + .loadClass(NAIVE_CLASS) + .asSubclass(PhysicalOptimizer.class) + .newInstance(); + } + } catch (ClassNotFoundException | InstantiationException | IllegalAccessException e) { + LOGGER.error("Cannot load class: {}", name, e); } - optimizer.setRules(getRules()); - return optimizer; - } - private Collection getRules() { - // TODO: get rule from conf - return Collections.emptyList(); + return optimizer; } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/optimizer/ReplicaDispatcher.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/optimizer/ReplicaDispatcher.java index 3d3cddbacd..a691323dd6 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/optimizer/ReplicaDispatcher.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/optimizer/ReplicaDispatcher.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.optimizer; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/optimizer/naive/NaiveReplicaDispatcher.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/optimizer/naive/NaiveReplicaDispatcher.java deleted file mode 100644 index b8b7a2f301..0000000000 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/optimizer/naive/NaiveReplicaDispatcher.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package cn.edu.tsinghua.iginx.engine.physical.optimizer.naive; - -import cn.edu.tsinghua.iginx.engine.physical.optimizer.ReplicaDispatcher; -import cn.edu.tsinghua.iginx.engine.physical.task.StoragePhysicalTask; - -public class NaiveReplicaDispatcher implements ReplicaDispatcher { - - private static final NaiveReplicaDispatcher INSTANCE = new NaiveReplicaDispatcher(); - - private NaiveReplicaDispatcher() {} - - @Override - public String chooseReplica(StoragePhysicalTask task) { - if (task == null) { - return null; - } - return task.getTargetFragment().getMasterStorageUnitId(); - } - - public static NaiveReplicaDispatcher getInstance() { - return INSTANCE; - } -} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/optimizer/rule/Rule.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/optimizer/rule/Rule.java deleted file mode 100644 index 12b3ae64ef..0000000000 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/optimizer/rule/Rule.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package cn.edu.tsinghua.iginx.engine.physical.optimizer.rule; - -public interface Rule { - - // TODO: how to define rule - -} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/IStorage.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/IStorage.java index b2d83e47c7..a29ae04223 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/IStorage.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/IStorage.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.storage; @@ -26,10 +25,13 @@ import cn.edu.tsinghua.iginx.engine.shared.operator.Insert; import cn.edu.tsinghua.iginx.engine.shared.operator.Project; import cn.edu.tsinghua.iginx.engine.shared.operator.Select; +import cn.edu.tsinghua.iginx.engine.shared.operator.SetTransform; +import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; import cn.edu.tsinghua.iginx.metadata.entity.ColumnsInterval; import cn.edu.tsinghua.iginx.metadata.entity.KeyInterval; import cn.edu.tsinghua.iginx.utils.Pair; import java.util.List; +import java.util.Set; public interface IStorage { /** 对非叠加分片查询数据 */ @@ -48,6 +50,15 @@ public interface IStorage { TaskExecuteResult executeProjectDummyWithSelect( Project project, Select select, DataArea dataArea); + default boolean isSupportProjectWithSetTransform(SetTransform setTransform, DataArea dataArea) { + return false; + } + + default TaskExecuteResult executeProjectWithSetTransform( + Project project, SetTransform setTransform, DataArea dataArea) { + throw new UnsupportedOperationException(); + } + /** 对非叠加分片删除数据 */ TaskExecuteResult executeDelete(Delete delete, DataArea dataArea); @@ -55,7 +66,7 @@ TaskExecuteResult executeProjectDummyWithSelect( TaskExecuteResult executeInsert(Insert insert, DataArea dataArea); /** 获取所有列信息 */ - List getColumns() throws PhysicalException; + List getColumns(Set patterns, TagFilter tagFilter) throws PhysicalException; /** 获取指定前缀的数据边界 */ Pair getBoundaryOfStorage(String prefix) throws PhysicalException; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/StorageEngineClassLoader.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/StorageEngineClassLoader.java index 4eec0860a4..8ee468e4fd 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/StorageEngineClassLoader.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/StorageEngineClassLoader.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.storage; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/StorageManager.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/StorageManager.java index 4ecf7e58c1..3f88625a89 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/StorageManager.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/StorageManager.java @@ -1,24 +1,24 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.storage; import cn.edu.tsinghua.iginx.conf.ConfigDescriptor; +import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; import cn.edu.tsinghua.iginx.metadata.entity.ColumnsInterval; import cn.edu.tsinghua.iginx.metadata.entity.KeyInterval; import cn.edu.tsinghua.iginx.metadata.entity.StorageEngineMeta; @@ -213,4 +213,32 @@ public static IStorage initStorageInstance(StorageEngineMeta meta) { return null; } } + + public boolean releaseStorage(StorageEngineMeta meta) throws PhysicalException { + long id = meta.getId(); + Pair pair = storageMap.get(id); + if (pair == null) { + LOGGER.warn("Storage id {} not found", id); + return false; + } + + ThreadPoolExecutor dispatcher = pair.getV(); + dispatcher.shutdown(); // 停止接收新任务 + try { + if (!dispatcher.awaitTermination(60, TimeUnit.SECONDS)) { // 等待任务完成 + dispatcher.shutdownNow(); // 如果时间内未完成任务,强制关闭 + if (!dispatcher.awaitTermination(60, TimeUnit.SECONDS)) { + LOGGER.error("Executor did not terminate"); + } + } + } catch (InterruptedException ie) { + dispatcher.shutdownNow(); + LOGGER.error("unexpected exception occurred in releasing storage engine: ", ie); + return false; + } + + pair.getK().release(); + storageMap.remove(id); + return true; + } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/domain/Column.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/domain/Column.java index 1adcf33f8b..accf53af7d 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/domain/Column.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/domain/Column.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.storage.domain; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/domain/ColumnKey.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/domain/ColumnKey.java index 09379e589c..68a4338a59 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/domain/ColumnKey.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/domain/ColumnKey.java @@ -1,11 +1,35 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.physical.storage.domain; +import cn.edu.tsinghua.iginx.engine.shared.Constants; import java.util.*; public class ColumnKey { + public static final ColumnKey KEY = new ColumnKey(Constants.KEY, Collections.emptyMap()); private final String path; private final SortedMap tags; + public ColumnKey(String path) { + this(path, Collections.emptyMap()); + } + public ColumnKey(String path, Map tagList) { this.path = Objects.requireNonNull(path); this.tags = Collections.unmodifiableSortedMap(new TreeMap<>(tagList)); diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/domain/DataArea.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/domain/DataArea.java index b60eda7580..9ba8de6f2e 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/domain/DataArea.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/domain/DataArea.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.physical.storage.domain; import cn.edu.tsinghua.iginx.metadata.entity.KeyInterval; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/execute/StoragePhysicalTaskExecutor.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/execute/StoragePhysicalTaskExecutor.java index 074e5f1834..36d42e3406 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/execute/StoragePhysicalTaskExecutor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/execute/StoragePhysicalTaskExecutor.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.storage.execute; @@ -25,24 +24,26 @@ import cn.edu.tsinghua.iginx.engine.physical.exception.TooManyPhysicalTasksException; import cn.edu.tsinghua.iginx.engine.physical.exception.UnexpectedOperatorException; import cn.edu.tsinghua.iginx.engine.physical.memory.MemoryPhysicalTaskDispatcher; +import cn.edu.tsinghua.iginx.engine.physical.memory.execute.OperatorMemoryExecutor; +import cn.edu.tsinghua.iginx.engine.physical.memory.execute.OperatorMemoryExecutorFactory; import cn.edu.tsinghua.iginx.engine.physical.optimizer.ReplicaDispatcher; import cn.edu.tsinghua.iginx.engine.physical.storage.IStorage; import cn.edu.tsinghua.iginx.engine.physical.storage.StorageManager; import cn.edu.tsinghua.iginx.engine.physical.storage.domain.Column; import cn.edu.tsinghua.iginx.engine.physical.storage.domain.DataArea; import cn.edu.tsinghua.iginx.engine.physical.storage.queue.StoragePhysicalTaskQueue; -import cn.edu.tsinghua.iginx.engine.physical.storage.utils.TagKVUtils; import cn.edu.tsinghua.iginx.engine.physical.task.GlobalPhysicalTask; import cn.edu.tsinghua.iginx.engine.physical.task.MemoryPhysicalTask; import cn.edu.tsinghua.iginx.engine.physical.task.StoragePhysicalTask; import cn.edu.tsinghua.iginx.engine.physical.task.TaskExecuteResult; +import cn.edu.tsinghua.iginx.engine.shared.data.read.RowStream; import cn.edu.tsinghua.iginx.engine.shared.operator.Delete; import cn.edu.tsinghua.iginx.engine.shared.operator.Insert; import cn.edu.tsinghua.iginx.engine.shared.operator.Operator; import cn.edu.tsinghua.iginx.engine.shared.operator.Project; import cn.edu.tsinghua.iginx.engine.shared.operator.Select; +import cn.edu.tsinghua.iginx.engine.shared.operator.SetTransform; import cn.edu.tsinghua.iginx.engine.shared.operator.ShowColumns; -import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; import cn.edu.tsinghua.iginx.engine.shared.operator.type.OperatorType; import cn.edu.tsinghua.iginx.metadata.DefaultMetaManager; import cn.edu.tsinghua.iginx.metadata.IMetaManager; @@ -60,7 +61,6 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ThreadPoolExecutor; -import java.util.regex.Pattern; import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -159,11 +159,21 @@ private StoragePhysicalTaskExecutor() { pair.k.isSupportProjectWithSelect() && operators.size() == 2 && operators.get(1).getType() == OperatorType.Select; + boolean needSetTransformPushDown = + operators.size() == 2 + && operators.get(1).getType() + == OperatorType.SetTransform; + boolean canSetTransformPushDown = + needSetTransformPushDown + && pair.k.isSupportProjectWithSetTransform( + (SetTransform) operators.get(1), dataArea); if (isDummyStorageUnit) { if (needSelectPushDown) { result = pair.k.executeProjectDummyWithSelect( (Project) op, (Select) operators.get(1), dataArea); + } else if (needSetTransformPushDown) { + throw new IllegalStateException(); } else { result = pair.k.executeProjectDummy((Project) op, dataArea); } @@ -172,6 +182,36 @@ private StoragePhysicalTaskExecutor() { result = pair.k.executeProjectWithSelect( (Project) op, (Select) operators.get(1), dataArea); + } else if (needSetTransformPushDown) { + if (canSetTransformPushDown) { + result = + pair.k.executeProjectWithSetTransform( + (Project) op, + (SetTransform) operators.get(1), + dataArea); + } else { + TaskExecuteResult tempResult = + pair.k.executeProject((Project) op, dataArea); + if (tempResult.getException() != null) { + result = tempResult; + } else { + // set transform push down is not supported, execute set + // transform in memory + OperatorMemoryExecutor executor = + OperatorMemoryExecutorFactory.getInstance() + .getMemoryExecutor(); + try { + RowStream rowStream = + executor.executeUnaryOperator( + (SetTransform) operators.get(1), + tempResult.getRowStream(), + task.getContext()); + result = new TaskExecuteResult(rowStream); + } catch (PhysicalException e) { + result = new TaskExecuteResult(e); + } + } + } } else { result = pair.k.executeProject((Project) op, dataArea); } @@ -261,6 +301,18 @@ private StoragePhysicalTaskExecutor() { if (after.getCreatedBy() != metaManager.getIginxId()) { storageManager.addStorage(after); } + } else if (before != null && after == null) { // 删除引擎时,需要release(目前仅支持dummy & read only) + try { + if (!storageManager.releaseStorage(before)) { + LOGGER.error( + "Fail to release deleted storage engine. Please look into server log."); + } + LOGGER.info("Release storage with id={} succeeded.", before.getId()); + } catch (PhysicalException e) { + LOGGER.error( + "unexpected exception during in releasing storage engine, please contact developer to check: ", + e); + } } }; metaManager.registerStorageEngineChangeHook(storageEngineChangeHook); @@ -316,7 +368,7 @@ public TaskExecuteResult executeGlobalTask(GlobalPhysicalTask task) { public TaskExecuteResult executeShowColumns(ShowColumns showColumns) { List storageList = metaManager.getStorageEngineList(); - Set columnSet = new HashSet<>(); + TreeSet targetColumns = new TreeSet<>(Comparator.comparing(Column::getPhysicalPath)); for (StorageEngineMeta storage : storageList) { long id = storage.getId(); Pair pair = storageManager.getStorage(id); @@ -324,56 +376,47 @@ public TaskExecuteResult executeShowColumns(ShowColumns showColumns) { continue; } try { - List columnList = pair.k.getColumns(); - // fix the schemaPrefix + Set patterns = showColumns.getPathRegexSet(); String schemaPrefix = storage.getSchemaPrefix(); + // schemaPrefix是在IGinX中定义的,数据源的路径中没有该前缀,因此需要剪掉patterns中前缀是schemaPrefix的部分 + patterns = StringUtils.cutSchemaPrefix(schemaPrefix, patterns); + if (patterns.isEmpty()) { + continue; + } + // 求patterns与dataPrefix的交集 + patterns = StringUtils.intersectDataPrefix(storage.getDataPrefix(), patterns); + if (patterns.isEmpty()) { + continue; + } + if (patterns.contains("*")) { + patterns = Collections.emptySet(); + } + List columnList = pair.k.getColumns(patterns, showColumns.getTagFilter()); + + // 列名前加上schemaPrefix if (schemaPrefix != null) { - for (Column column : columnList) { - if (column.isDummy()) { - column.setPath(schemaPrefix + "." + column.getPath()); - } - } + columnList.forEach( + column -> { + column.setPath(schemaPrefix + "." + column.getPath()); + targetColumns.add(column); + }); + } else { + targetColumns.addAll(columnList); } - columnSet.addAll(columnList); } catch (PhysicalException e) { return new TaskExecuteResult(e); } } - Set pathRegexSet = showColumns.getPathRegexSet(); - TagFilter tagFilter = showColumns.getTagFilter(); - - TreeSet tsSetAfterFilter = new TreeSet<>(Comparator.comparing(Column::getPhysicalPath)); - for (Column column : columnSet) { - boolean isTarget = true; - if (!pathRegexSet.isEmpty()) { - isTarget = false; - for (String pathRegex : pathRegexSet) { - if (Pattern.matches(StringUtils.reformatPath(pathRegex), column.getPath())) { - isTarget = true; - break; - } - } - } - if (tagFilter != null) { - if (!TagKVUtils.match(column.getTags(), tagFilter)) { - isTarget = false; - } - } - if (isTarget) { - tsSetAfterFilter.add(column); - } - } - int limit = showColumns.getLimit(); int offset = showColumns.getOffset(); if (limit == Integer.MAX_VALUE && offset == 0) { - return new TaskExecuteResult(Column.toRowStream(tsSetAfterFilter)); + return new TaskExecuteResult(Column.toRowStream(targetColumns)); } else { // only need part of data. List tsList = new ArrayList<>(); - int cur = 0, size = tsSetAfterFilter.size(); - for (Iterator iter = tsSetAfterFilter.iterator(); iter.hasNext(); cur++) { + int cur = 0, size = targetColumns.size(); + for (Iterator iter = targetColumns.iterator(); iter.hasNext(); cur++) { if (cur >= size || cur - offset >= limit) { break; } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/queue/StoragePhysicalTaskQueue.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/queue/StoragePhysicalTaskQueue.java index 7fc371d2c2..cb5fa5fb3c 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/queue/StoragePhysicalTaskQueue.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/queue/StoragePhysicalTaskQueue.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.storage.queue; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/utils/ColumnKeyTranslator.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/utils/ColumnKeyTranslator.java index cf3d2b2025..2e6f7fa972 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/utils/ColumnKeyTranslator.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/utils/ColumnKeyTranslator.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.physical.storage.utils; import cn.edu.tsinghua.iginx.engine.physical.storage.domain.ColumnKey; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/utils/TagKVUtils.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/utils/TagKVUtils.java index 2202931d68..2acb04a057 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/utils/TagKVUtils.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/utils/TagKVUtils.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.physical.storage.utils; import cn.edu.tsinghua.iginx.engine.shared.operator.tag.*; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/AbstractPhysicalTask.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/AbstractPhysicalTask.java index 30a31be860..32727aa5ff 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/AbstractPhysicalTask.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/AbstractPhysicalTask.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.task; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/BinaryMemoryPhysicalTask.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/BinaryMemoryPhysicalTask.java index f92df4aa45..1152970171 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/BinaryMemoryPhysicalTask.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/BinaryMemoryPhysicalTask.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.task; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/CompletedFoldedPhysicalTask.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/CompletedFoldedPhysicalTask.java index 179982d791..8f24ad52de 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/CompletedFoldedPhysicalTask.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/CompletedFoldedPhysicalTask.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.physical.task; import cn.edu.tsinghua.iginx.engine.shared.RequestContext; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/FoldedMemoryPhysicalTask.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/FoldedMemoryPhysicalTask.java index 5a240f3e53..434633184c 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/FoldedMemoryPhysicalTask.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/FoldedMemoryPhysicalTask.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.physical.task; import static cn.edu.tsinghua.iginx.engine.logical.utils.MetaUtils.getFragmentsByColumnsInterval; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/GlobalPhysicalTask.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/GlobalPhysicalTask.java index a6054b42b6..409628d41a 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/GlobalPhysicalTask.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/GlobalPhysicalTask.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.task; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/Measurable.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/Measurable.java index df53ac9fcd..e173fead78 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/Measurable.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/Measurable.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.physical.task; public interface Measurable { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/MemoryPhysicalTask.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/MemoryPhysicalTask.java index e218ba69ea..2bcb274cd6 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/MemoryPhysicalTask.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/MemoryPhysicalTask.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.task; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/MultipleMemoryPhysicalTask.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/MultipleMemoryPhysicalTask.java index 265de1b718..92c7f67f7b 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/MultipleMemoryPhysicalTask.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/MultipleMemoryPhysicalTask.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.physical.task; import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/PhysicalTask.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/PhysicalTask.java index f5a4a9e62d..2e5a3177cc 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/PhysicalTask.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/PhysicalTask.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.task; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/StoragePhysicalTask.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/StoragePhysicalTask.java index 91972369d7..5fd3c1159a 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/StoragePhysicalTask.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/StoragePhysicalTask.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.task; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/TaskExecuteResult.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/TaskExecuteResult.java index 02f73466d0..fc5f87742a 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/TaskExecuteResult.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/TaskExecuteResult.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.task; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/TaskType.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/TaskType.java index fbfee578fa..944d27350d 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/TaskType.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/TaskType.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.task; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/UnaryMemoryPhysicalTask.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/UnaryMemoryPhysicalTask.java index d871066948..8240947cf1 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/UnaryMemoryPhysicalTask.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/UnaryMemoryPhysicalTask.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.physical.task; import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/utils/TaskUtils.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/utils/TaskUtils.java index 276e26cd32..ea72b82bd4 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/utils/TaskUtils.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/utils/TaskUtils.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.physical.task.utils; import cn.edu.tsinghua.iginx.engine.physical.task.BinaryMemoryPhysicalTask; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/visitor/TaskInfoVisitor.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/visitor/TaskInfoVisitor.java index 7208d8d8dc..020150cfd2 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/visitor/TaskInfoVisitor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/visitor/TaskInfoVisitor.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.physical.task.visitor; import cn.edu.tsinghua.iginx.engine.physical.task.BinaryMemoryPhysicalTask; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/visitor/TaskVisitor.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/visitor/TaskVisitor.java index 4a1eb3d259..b82c3d2205 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/visitor/TaskVisitor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/task/visitor/TaskVisitor.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.physical.task.visitor; import cn.edu.tsinghua.iginx.engine.physical.task.BinaryMemoryPhysicalTask; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/Constants.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/Constants.java index 1645e8f241..20fe07f2c8 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/Constants.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/Constants.java @@ -1,23 +1,26 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + public class Constants { public static final String KEY = "key"; @@ -29,4 +32,11 @@ public class Constants { public static final String UDF_CLASS = "t"; public static final String UDF_FUNC = "transform"; + + public static final String WINDOW_START_COL = "window_start"; + public static final String WINDOW_END_COL = "window_end"; + + // 保留列名,会在reorder时保留,并按原顺序出现在表的最前面 + public static final Set RESERVED_COLS = + new HashSet<>(Arrays.asList(WINDOW_START_COL, WINDOW_END_COL)); } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/KeyRange.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/KeyRange.java index 35000ca367..83656eccac 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/KeyRange.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/KeyRange.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/RequestContext.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/RequestContext.java index 293e2b431c..c9bb244ae9 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/RequestContext.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/RequestContext.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared; import cn.edu.tsinghua.iginx.engine.physical.task.PhysicalTask; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/Result.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/Result.java index 67d0a3cf41..37c4239667 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/Result.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/Result.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared; import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/constraint/ConstraintManager.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/constraint/ConstraintManager.java index 6f88d8573f..f61d34cbc2 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/constraint/ConstraintManager.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/constraint/ConstraintManager.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.constraint; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/ExecuteDetail.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/ExecuteDetail.java index 563bf12a99..98223b3a64 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/ExecuteDetail.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/ExecuteDetail.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.data; import java.util.List; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/Value.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/Value.java index c9b4c10e02..3cffd7cf4b 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/Value.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/Value.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.data; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/read/ClearEmptyRowStreamWrapper.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/read/ClearEmptyRowStreamWrapper.java index a867c26cc8..d6106c0003 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/read/ClearEmptyRowStreamWrapper.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/read/ClearEmptyRowStreamWrapper.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.data.read; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/read/Field.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/read/Field.java index afb1c18e7d..4ebfc7a12c 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/read/Field.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/read/Field.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.data.read; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/read/FilterRowStreamWrapper.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/read/FilterRowStreamWrapper.java index ded35cfecb..4e51e0f271 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/read/FilterRowStreamWrapper.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/read/FilterRowStreamWrapper.java @@ -1,17 +1,19 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.data.read; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/read/Header.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/read/Header.java index 5a6614d463..44a07a114f 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/read/Header.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/read/Header.java @@ -1,23 +1,24 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.data.read; +import static cn.edu.tsinghua.iginx.engine.shared.Constants.RESERVED_COLS; + import cn.edu.tsinghua.iginx.utils.Pair; import cn.edu.tsinghua.iginx.utils.StringUtils; import java.util.*; @@ -126,52 +127,72 @@ public boolean equals(Object o) { && Objects.equals(indexMap, header.indexMap); } - public Header renamedHeader(Map aliasMap, List ignorePatterns) { + public Header renamedHeader(List> aliasList, List ignorePatterns) { List newFields = new ArrayList<>(); - fields.forEach( - field -> { - // 如果列名在ignorePatterns中,对该列不执行rename - for (String ignorePattern : ignorePatterns) { - if (StringUtils.match(field.getName(), ignorePattern)) { - newFields.add(field); - return; - } + int size = getFieldSize(); + for (int i = 0; i < size; i++) { + Field field = fields.get(i); + // 如果列名在ignorePatterns中,对该列不执行rename + boolean ignore = false; + for (String ignorePattern : ignorePatterns) { + if (StringUtils.match(field.getName(), ignorePattern)) { + newFields.add(field); + ignore = true; + break; + } + } + if (ignore) { + continue; + } + String alias = ""; + for (Pair pair : aliasList) { + String oldPattern = pair.k; + String newPattern = pair.v; + if (oldPattern.equals("*") && newPattern.endsWith(".*")) { + String newPrefix = newPattern.substring(0, newPattern.length() - 1); + alias = newPrefix + field.getName(); + } else if (oldPattern.endsWith(".*") && newPattern.endsWith(".*")) { + String oldPrefix = oldPattern.substring(0, oldPattern.length() - 1); + String newPrefix = newPattern.substring(0, newPattern.length() - 1); + if (field.getName().startsWith(oldPrefix)) { + alias = field.getName().replaceFirst(oldPrefix, newPrefix); } - String alias = ""; - for (String oldPattern : aliasMap.keySet()) { - String newPattern = aliasMap.get(oldPattern); - if (oldPattern.equals("*") && newPattern.endsWith(".*")) { - String newPrefix = newPattern.substring(0, newPattern.length() - 1); - alias = newPrefix + field.getName(); - } else if (oldPattern.endsWith(".*") && newPattern.endsWith(".*")) { - String oldPrefix = oldPattern.substring(0, oldPattern.length() - 1); - String newPrefix = newPattern.substring(0, newPattern.length() - 1); - if (field.getName().startsWith(oldPrefix)) { - alias = field.getName().replaceFirst(oldPrefix, newPrefix); - } - break; - } else if (oldPattern.equals(field.getFullName())) { - alias = newPattern; - break; + break; + } else if (oldPattern.equals(field.getName())) { + alias = newPattern; + Set> tagSet = new HashSet<>(); + Field nextField = i < size - 1 ? fields.get(i + 1) : null; + tagSet.add(field.getTags()); + // 处理同一列但不同tag的情况 + while (nextField != null + && oldPattern.equals(nextField.getName()) + && !tagSet.contains(nextField.getTags())) { + newFields.add(new Field(alias, field.getType(), field.getTags())); + field = nextField; + i++; + nextField = i < size - 1 ? fields.get(i + 1) : null; + tagSet.add(field.getTags()); + } + aliasList.remove(pair); + break; + } else { + if (StringUtils.match(field.getName(), oldPattern)) { + if (newPattern.endsWith("." + oldPattern)) { + String prefix = newPattern.substring(0, newPattern.length() - oldPattern.length()); + alias = prefix + field.getName(); } else { - if (StringUtils.match(field.getName(), oldPattern)) { - if (newPattern.endsWith("." + oldPattern)) { - String prefix = - newPattern.substring(0, newPattern.length() - oldPattern.length()); - alias = prefix + field.getName(); - } else { - alias = newPattern; - } - break; - } + alias = newPattern; } + break; } - if (alias.isEmpty()) { - newFields.add(field); - } else { - newFields.add(new Field(alias, field.getType(), field.getTags())); - } - }); + } + } + if (alias.isEmpty()) { + newFields.add(field); + } else { + newFields.add(new Field(alias, field.getType(), field.getTags())); + } + } return new Header(getKey(), newFields); } @@ -212,6 +233,15 @@ public ReorderedHeaderWrapped reorderedHeaderWrapped( List targetFields = new ArrayList<>(); Map reorderMap = new HashMap<>(); + // 保留关键字列 + for (int i = 0; i < fields.size(); i++) { + Field field = getField(i); + if (RESERVED_COLS.contains(field.getName())) { + reorderMap.put(targetFields.size(), i); + targetFields.add(field); + } + } + for (int index = 0; index < patterns.size(); index++) { String pattern = patterns.get(index); List> matchedFields = new ArrayList<>(); diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/read/MergeFieldRowStreamWrapper.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/read/MergeFieldRowStreamWrapper.java index 3f1a0ad77e..4a207c4858 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/read/MergeFieldRowStreamWrapper.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/read/MergeFieldRowStreamWrapper.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.data.read; import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/read/MergeTimeRowStreamWrapper.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/read/MergeTimeRowStreamWrapper.java index 3694be599e..ddd592d982 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/read/MergeTimeRowStreamWrapper.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/read/MergeTimeRowStreamWrapper.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.data.read; import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/read/Row.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/read/Row.java index cbee7591de..77a33bcc2c 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/read/Row.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/read/Row.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.data.read; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/read/RowStream.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/read/RowStream.java index dbb3dbac86..782fcbde9e 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/read/RowStream.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/read/RowStream.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.data.read; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/read/RowStreamWrapper.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/read/RowStreamWrapper.java index 71a5ae617d..c1b088924b 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/read/RowStreamWrapper.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/read/RowStreamWrapper.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.data.read; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/write/BitmapView.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/write/BitmapView.java index a902cf9991..9c43e422eb 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/write/BitmapView.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/write/BitmapView.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.data.write; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/write/ColumnDataView.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/write/ColumnDataView.java index ed7928f08b..14cc789164 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/write/ColumnDataView.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/write/ColumnDataView.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.data.write; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/write/DataView.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/write/DataView.java index 179cbb361c..e3b5f2644f 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/write/DataView.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/write/DataView.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.data.write; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/write/RawData.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/write/RawData.java index 587e1187d1..f3926b80cf 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/write/RawData.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/write/RawData.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.data.write; import cn.edu.tsinghua.iginx.thrift.DataType; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/write/RawDataType.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/write/RawDataType.java index aec570d6de..7b74e1063b 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/write/RawDataType.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/write/RawDataType.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.data.write; public enum RawDataType { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/write/RowDataView.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/write/RowDataView.java index 379f8963dd..b0acb1d81f 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/write/RowDataView.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/write/RowDataView.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.data.write; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/exception/StatementExecutionException.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/exception/StatementExecutionException.java index 1c02083dd5..48c8392adc 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/exception/StatementExecutionException.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/exception/StatementExecutionException.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.exception; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/expr/BaseExpression.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/expr/BaseExpression.java index 22a040b4b0..852af9629d 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/expr/BaseExpression.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/expr/BaseExpression.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.expr; public class BaseExpression implements Expression { @@ -34,7 +52,7 @@ public ExpressionType getType() { @Override public boolean hasAlias() { - return alias != null && !alias.equals(""); + return alias != null && !alias.isEmpty(); } @Override diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/expr/BinaryExpression.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/expr/BinaryExpression.java index ec230c75c7..6a6d8009b0 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/expr/BinaryExpression.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/expr/BinaryExpression.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.expr; public class BinaryExpression implements Expression { @@ -59,7 +77,7 @@ public ExpressionType getType() { @Override public boolean hasAlias() { - return alias != null && !alias.equals(""); + return alias != null && !alias.isEmpty(); } @Override diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/expr/BracketExpression.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/expr/BracketExpression.java index de69413d9f..9695abea76 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/expr/BracketExpression.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/expr/BracketExpression.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.expr; public class BracketExpression implements Expression { @@ -34,7 +52,7 @@ public ExpressionType getType() { @Override public boolean hasAlias() { - return alias != null && !alias.equals(""); + return alias != null && !alias.isEmpty(); } @Override diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/expr/ConstantExpression.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/expr/ConstantExpression.java index 9a276f08d5..86015e072a 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/expr/ConstantExpression.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/expr/ConstantExpression.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.expr; public class ConstantExpression implements Expression { @@ -34,7 +52,7 @@ public ExpressionType getType() { @Override public boolean hasAlias() { - return alias != null && !alias.equals(""); + return alias != null && !alias.isEmpty(); } @Override diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/expr/Expression.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/expr/Expression.java index 930981c3b1..0260761ce4 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/expr/Expression.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/expr/Expression.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.expr; public interface Expression { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/expr/ExpressionVisitor.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/expr/ExpressionVisitor.java index 53658a175c..0537836302 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/expr/ExpressionVisitor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/expr/ExpressionVisitor.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.expr; public interface ExpressionVisitor { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/expr/FromValueExpression.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/expr/FromValueExpression.java index bef0169155..9134b21561 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/expr/FromValueExpression.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/expr/FromValueExpression.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.expr; import cn.edu.tsinghua.iginx.sql.statement.select.SelectStatement; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/expr/FuncExpression.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/expr/FuncExpression.java index bcd357ffd8..14fc68c9ea 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/expr/FuncExpression.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/expr/FuncExpression.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.expr; import cn.edu.tsinghua.iginx.engine.shared.function.FunctionUtils; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/expr/MultipleExpression.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/expr/MultipleExpression.java index 2179f79757..8f73f354f4 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/expr/MultipleExpression.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/expr/MultipleExpression.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.expr; import java.util.List; @@ -74,7 +92,7 @@ public ExpressionType getType() { @Override public boolean hasAlias() { - return alias != null && !alias.equals(""); + return alias != null && !alias.isEmpty(); } @Override diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/expr/Operator.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/expr/Operator.java index d3e8876408..3ea2def34c 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/expr/Operator.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/expr/Operator.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.expr; public enum Operator { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/expr/UnaryExpression.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/expr/UnaryExpression.java index 2164d791dd..de051e6478 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/expr/UnaryExpression.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/expr/UnaryExpression.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.expr; public class UnaryExpression implements Expression { @@ -40,7 +58,7 @@ public ExpressionType getType() { @Override public boolean hasAlias() { - return alias != null && !alias.equals(""); + return alias != null && !alias.isEmpty(); } @Override diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/file/CSVFile.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/file/CSVFile.java index 10d168dda1..0881537a9c 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/file/CSVFile.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/file/CSVFile.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.file; import cn.edu.tsinghua.iginx.utils.CSVUtils; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/file/FileType.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/file/FileType.java index 742ddfdc7c..d014c40e15 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/file/FileType.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/file/FileType.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.file; public enum FileType { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/file/read/ImportCsv.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/file/read/ImportCsv.java index bd09c8067f..1f540dfb69 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/file/read/ImportCsv.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/file/read/ImportCsv.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.file.read; import cn.edu.tsinghua.iginx.engine.shared.file.CSVFile; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/file/read/ImportFile.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/file/read/ImportFile.java index 1066dc5026..b4544890ec 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/file/read/ImportFile.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/file/read/ImportFile.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.file.read; import cn.edu.tsinghua.iginx.engine.shared.file.FileType; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/file/write/ExportByteStream.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/file/write/ExportByteStream.java index f83bb63610..86942a68dc 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/file/write/ExportByteStream.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/file/write/ExportByteStream.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.file.write; import cn.edu.tsinghua.iginx.engine.shared.file.FileType; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/file/write/ExportCsv.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/file/write/ExportCsv.java index 9ef752b127..253d7a16c6 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/file/write/ExportCsv.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/file/write/ExportCsv.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.file.write; import cn.edu.tsinghua.iginx.engine.shared.file.CSVFile; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/file/write/ExportFile.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/file/write/ExportFile.java index 8b53644bf1..791ce16b73 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/file/write/ExportFile.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/file/write/ExportFile.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.file.write; import cn.edu.tsinghua.iginx.engine.shared.file.FileType; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/Function.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/Function.java index 163ea3e348..3fa9d887dd 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/Function.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/Function.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.function; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/FunctionCall.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/FunctionCall.java index 201e57790b..0713e56422 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/FunctionCall.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/FunctionCall.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.function; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/FunctionParams.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/FunctionParams.java index 947645a351..0fa0bf2f01 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/FunctionParams.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/FunctionParams.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.function; import cn.edu.tsinghua.iginx.engine.shared.expr.Expression; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/FunctionType.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/FunctionType.java index 2ca683a519..c03ef8f525 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/FunctionType.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/FunctionType.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.function; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/FunctionUtils.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/FunctionUtils.java index cf0edb7e65..8d7e7ae571 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/FunctionUtils.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/FunctionUtils.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.function; import static cn.edu.tsinghua.iginx.utils.DataTypeUtils.isWholeNumber; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/MappingFunction.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/MappingFunction.java index 43b8f1f2aa..1ccece57d4 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/MappingFunction.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/MappingFunction.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.function; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/MappingType.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/MappingType.java index b8bb17c047..a449452030 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/MappingType.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/MappingType.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.function; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/RowMappingFunction.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/RowMappingFunction.java index 95972d8f0c..de2b2eadf1 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/RowMappingFunction.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/RowMappingFunction.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.function; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/SetMappingFunction.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/SetMappingFunction.java index 45f227b932..3aded20a5e 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/SetMappingFunction.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/SetMappingFunction.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.function; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/manager/FunctionManager.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/manager/FunctionManager.java index ba74d76572..c3d89886fa 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/manager/FunctionManager.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/manager/FunctionManager.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.function.manager; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/ArithmeticExpr.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/ArithmeticExpr.java index f3502f413f..6580c3daa1 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/ArithmeticExpr.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/ArithmeticExpr.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.function.system; import cn.edu.tsinghua.iginx.engine.physical.memory.execute.utils.ExprUtils; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/Avg.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/Avg.java index 043c08a23a..acba2fe4b4 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/Avg.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/Avg.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.function.system; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/Count.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/Count.java index baf7dbe337..5cb698ad9d 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/Count.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/Count.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.function.system; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/First.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/First.java index 6ca685a957..1a3d32611c 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/First.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/First.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.function.system; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/FirstValue.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/FirstValue.java index efc120d819..690b43d8d2 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/FirstValue.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/FirstValue.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.function.system; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/Last.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/Last.java index 09a2cd5657..672cc68ead 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/Last.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/Last.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.function.system; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/LastValue.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/LastValue.java index 77e626a0cc..d57c79560b 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/LastValue.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/LastValue.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.function.system; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/Max.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/Max.java index 8e9b0e7621..dafe2ceaad 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/Max.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/Max.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.function.system; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/Min.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/Min.java index 9952778dd6..bd234e950d 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/Min.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/Min.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.function.system; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/Ratio.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/Ratio.java index 9539f9964b..42e6ccbf5b 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/Ratio.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/Ratio.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.function.system; import cn.edu.tsinghua.iginx.engine.shared.data.Value; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/Sum.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/Sum.java index 606c66fb32..71986e0710 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/Sum.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/Sum.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.function.system; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/utils/GroupByUtils.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/utils/GroupByUtils.java index de8c6db186..1770ef61b5 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/utils/GroupByUtils.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/utils/GroupByUtils.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.function.system.utils; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/utils/ValueUtils.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/utils/ValueUtils.java index 1e7c4a6e94..773f5c6070 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/utils/ValueUtils.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/system/utils/ValueUtils.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.function.system.utils; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/udf/UDAF.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/udf/UDAF.java index 70f679ab05..12a5b24e0d 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/udf/UDAF.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/udf/UDAF.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.function.udf; import cn.edu.tsinghua.iginx.engine.shared.function.SetMappingFunction; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/udf/UDSF.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/udf/UDSF.java index 9aa87b8c90..cc278938ce 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/udf/UDSF.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/udf/UDSF.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.function.udf; import cn.edu.tsinghua.iginx.engine.shared.function.MappingFunction; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/udf/UDTF.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/udf/UDTF.java index 78cbdeba38..2ddc562991 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/udf/UDTF.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/udf/UDTF.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.function.udf; import cn.edu.tsinghua.iginx.engine.shared.function.RowMappingFunction; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/udf/python/PyUDAF.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/udf/python/PyUDAF.java index 9c24c5a20d..c44b470515 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/udf/python/PyUDAF.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/udf/python/PyUDAF.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.function.udf.python; import static cn.edu.tsinghua.iginx.engine.shared.Constants.UDF_CLASS; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/udf/python/PyUDSF.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/udf/python/PyUDSF.java index 36da7166ab..210f8d1211 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/udf/python/PyUDSF.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/udf/python/PyUDSF.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.function.udf.python; import static cn.edu.tsinghua.iginx.engine.shared.Constants.UDF_CLASS; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/udf/python/PyUDTF.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/udf/python/PyUDTF.java index 3b4d89b114..4ed1701161 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/udf/python/PyUDTF.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/udf/python/PyUDTF.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.function.udf.python; import static cn.edu.tsinghua.iginx.engine.shared.Constants.UDF_CLASS; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/udf/utils/CheckUtils.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/udf/utils/CheckUtils.java index a5789729f0..701f27e855 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/udf/utils/CheckUtils.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/udf/utils/CheckUtils.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.function.udf.utils; import cn.edu.tsinghua.iginx.engine.shared.function.FunctionParams; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/udf/utils/DataUtils.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/udf/utils/DataUtils.java index b2bf283aa4..ad244952d0 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/udf/utils/DataUtils.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/udf/utils/DataUtils.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.function.udf.utils; import cn.edu.tsinghua.iginx.engine.physical.memory.execute.Table; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/udf/utils/RowUtils.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/udf/utils/RowUtils.java index 06165f5d3e..59daec650c 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/udf/utils/RowUtils.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/function/udf/utils/RowUtils.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.function.udf.utils; import cn.edu.tsinghua.iginx.engine.physical.memory.execute.Table; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/AbstractBinaryOperator.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/AbstractBinaryOperator.java index 6466ec0464..b77a725fdb 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/AbstractBinaryOperator.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/AbstractBinaryOperator.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.operator; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/AbstractJoin.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/AbstractJoin.java index 762f105e03..ce481dfc9e 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/AbstractJoin.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/AbstractJoin.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.operator; import cn.edu.tsinghua.iginx.engine.shared.operator.type.JoinAlgType; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/AbstractMultipleOperator.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/AbstractMultipleOperator.java index 954a07f84e..d63774cb8e 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/AbstractMultipleOperator.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/AbstractMultipleOperator.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.operator; import cn.edu.tsinghua.iginx.engine.shared.operator.type.OperatorType; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/AbstractOperator.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/AbstractOperator.java index 4ee81b0022..9c06b2b21a 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/AbstractOperator.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/AbstractOperator.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.operator; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/AbstractUnaryOperator.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/AbstractUnaryOperator.java index c30e06c9a9..18d9713876 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/AbstractUnaryOperator.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/AbstractUnaryOperator.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.operator; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/AddSchemaPrefix.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/AddSchemaPrefix.java index 04ccd02bf3..c33ca326c9 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/AddSchemaPrefix.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/AddSchemaPrefix.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.operator; import cn.edu.tsinghua.iginx.engine.shared.operator.type.OperatorType; @@ -30,4 +48,16 @@ public String getInfo() { public String getSchemaPrefix() { return schemaPrefix; } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (object == null || getClass() != object.getClass()) { + return false; + } + AddSchemaPrefix that = (AddSchemaPrefix) object; + return schemaPrefix.equals(that.schemaPrefix); + } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/BinaryOperator.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/BinaryOperator.java index 4ad30c2a1d..ee2adfa05b 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/BinaryOperator.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/BinaryOperator.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.operator; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/CombineNonQuery.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/CombineNonQuery.java index 6334e28223..b69d4f69ca 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/CombineNonQuery.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/CombineNonQuery.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.operator; import cn.edu.tsinghua.iginx.engine.shared.operator.type.OperatorType; @@ -24,4 +42,12 @@ public MultipleOperator copyWithSource(List sources) { public String getInfo() { return ""; } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + return object != null && getClass() == object.getClass(); + } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/CrossJoin.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/CrossJoin.java index f6aa73092b..4a272a6371 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/CrossJoin.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/CrossJoin.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.operator; import cn.edu.tsinghua.iginx.engine.shared.operator.type.JoinAlgType; @@ -48,4 +66,18 @@ public BinaryOperator copyWithSource(Source sourceA, Source sourceB) { public String getInfo() { return "PrefixA: " + getPrefixA() + ", PrefixB: " + getPrefixB(); } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (object == null || getClass() != object.getClass()) { + return false; + } + CrossJoin that = (CrossJoin) object; + return getPrefixA().equals(that.getPrefixA()) + && getPrefixB().equals(that.getPrefixB()) + && getExtraJoinPrefix().equals(that.getExtraJoinPrefix()); + } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Delete.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Delete.java index 72ec71ecc6..448131187d 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Delete.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Delete.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.operator; import cn.edu.tsinghua.iginx.engine.shared.KeyRange; @@ -66,4 +84,21 @@ public String getInfo() { } return builder.toString(); } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (object == null || getClass() != object.getClass()) { + return false; + } + if (!super.equals(object)) { + return false; + } + Delete that = (Delete) object; + return keyRanges.equals(that.keyRanges) + && patterns.equals(that.patterns) + && tagFilter.equals(that.tagFilter); + } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Distinct.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Distinct.java index 3a4faf76ba..311062969f 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Distinct.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Distinct.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.operator; import cn.edu.tsinghua.iginx.engine.shared.operator.type.OperatorType; @@ -32,4 +50,16 @@ public UnaryOperator copyWithSource(Source source) { public String getInfo() { return "Patterns: " + String.join(",", patterns); } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (object == null || getClass() != object.getClass()) { + return false; + } + Distinct that = (Distinct) object; + return patterns.equals(that.patterns); + } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Downsample.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Downsample.java index e1b6b6f1ac..03414af68d 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Downsample.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Downsample.java @@ -1,23 +1,25 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.operator; +import static cn.edu.tsinghua.iginx.constant.GlobalConstant.KEY_MAX_VAL; +import static cn.edu.tsinghua.iginx.constant.GlobalConstant.KEY_MIN_VAL; + import cn.edu.tsinghua.iginx.engine.shared.KeyRange; import cn.edu.tsinghua.iginx.engine.shared.function.FunctionCall; import cn.edu.tsinghua.iginx.engine.shared.function.MappingType; @@ -125,4 +127,23 @@ public String getInfo() { return sb.toString(); } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (object == null || getClass() != object.getClass()) { + return false; + } + Downsample that = (Downsample) object; + return precision == that.precision + && slideDistance == that.slideDistance + && functionCallList.equals(that.functionCallList) + && keyRange.equals(that.keyRange); + } + + public boolean notSetInterval() { + return getKeyRange().getBeginKey() == KEY_MIN_VAL && getKeyRange().getEndKey() == KEY_MAX_VAL; + } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Except.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Except.java index ee841d088a..0eb29b76f2 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Except.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Except.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.operator; import cn.edu.tsinghua.iginx.engine.shared.operator.type.OperatorType; @@ -77,4 +95,18 @@ public String getInfo() { builder.append(" isDistinct: ").append(isDistinct); return builder.toString(); } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (object == null || getClass() != object.getClass()) { + return false; + } + Except that = (Except) object; + return leftOrder.equals(that.leftOrder) + && rightOrder.equals(that.rightOrder) + && isDistinct == that.isDistinct; + } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/FoldedOperator.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/FoldedOperator.java index 47d345c9a3..b39e832ab7 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/FoldedOperator.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/FoldedOperator.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.operator; import cn.edu.tsinghua.iginx.engine.shared.operator.type.OperatorType; @@ -32,4 +50,12 @@ public MultipleOperator copyWithSource(List sources) { public String getInfo() { return ""; } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + return object != null && getClass() == object.getClass(); + } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/GroupBy.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/GroupBy.java index c0f2317114..1af269b5f9 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/GroupBy.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/GroupBy.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.operator; import cn.edu.tsinghua.iginx.engine.shared.function.FunctionCall; @@ -67,4 +85,16 @@ public String getInfo() { } return builder.toString(); } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (object == null || getClass() != object.getClass()) { + return false; + } + GroupBy that = (GroupBy) object; + return groupByCols.equals(that.groupByCols) && functionCallList.equals(that.functionCallList); + } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/InnerJoin.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/InnerJoin.java index 5cac1359a1..50400907c3 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/InnerJoin.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/InnerJoin.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.operator; import static cn.edu.tsinghua.iginx.engine.shared.operator.type.JoinAlgType.chooseJoinAlg; @@ -222,4 +240,20 @@ public String getInfo() { } return builder.toString(); } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (object == null || getClass() != object.getClass()) { + return false; + } + InnerJoin that = (InnerJoin) object; + return isNaturalJoin == that.isNaturalJoin + && joinColumns.equals(that.joinColumns) + && filter.equals(that.filter) + && tagFilter.equals(that.tagFilter) + && getExtraJoinPrefix().equals(that.getExtraJoinPrefix()); + } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Insert.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Insert.java index 44bc02bd22..dcdf292868 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Insert.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Insert.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.operator; import cn.edu.tsinghua.iginx.engine.shared.data.write.DataView; @@ -36,4 +54,17 @@ public UnaryOperator copyWithSource(Source source) { public String getInfo() { return ""; } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (object == null || getClass() != object.getClass()) { + return false; + } + + Insert that = (Insert) object; + return data.equals(that.data); + } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Intersect.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Intersect.java index 5eaac74c00..1914114709 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Intersect.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Intersect.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.operator; import cn.edu.tsinghua.iginx.engine.shared.operator.type.OperatorType; @@ -77,4 +95,18 @@ public String getInfo() { builder.append(" isDistinct: ").append(isDistinct); return builder.toString(); } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (object == null || getClass() != object.getClass()) { + return false; + } + Intersect that = (Intersect) object; + return leftOrder.equals(that.leftOrder) + && rightOrder.equals(that.rightOrder) + && isDistinct == that.isDistinct; + } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Join.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Join.java index 4eb706d0d2..11752b836a 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Join.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Join.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.operator; @@ -56,4 +55,16 @@ public BinaryOperator copyWithSource(Source sourceA, Source sourceB) { public String getInfo() { return "JoinBy: " + joinBy; } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (object == null || getClass() != object.getClass()) { + return false; + } + Join join = (Join) object; + return joinBy.equals(join.joinBy); + } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Limit.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Limit.java index a3ed3180bc..473cb42e78 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Limit.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Limit.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.operator; @@ -58,4 +57,16 @@ public UnaryOperator copyWithSource(Source source) { public String getInfo() { return "Limit: " + limit + ", Offset: " + offset; } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (object == null || getClass() != object.getClass()) { + return false; + } + Limit limit1 = (Limit) object; + return limit == limit1.limit && offset == limit1.offset; + } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/MappingTransform.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/MappingTransform.java index 171bbd3d6b..d40e0bc8c2 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/MappingTransform.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/MappingTransform.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.operator; @@ -72,4 +71,16 @@ public String getInfo() { return sb.toString(); } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (object == null || getClass() != object.getClass()) { + return false; + } + MappingTransform that = (MappingTransform) object; + return functionCallList.equals(that.functionCallList); + } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/MarkJoin.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/MarkJoin.java index e11769ac0d..e2322644a9 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/MarkJoin.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/MarkJoin.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.operator; import static cn.edu.tsinghua.iginx.engine.shared.operator.type.JoinAlgType.chooseJoinAlg; @@ -126,4 +144,19 @@ public String getInfo() { } return builder.toString(); } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (object == null || getClass() != object.getClass()) { + return false; + } + MarkJoin that = (MarkJoin) object; + return filter.equals(that.filter) + && markColumn.equals(that.markColumn) + && isAntiJoin == that.isAntiJoin + && getExtraJoinPrefix().equals(that.getExtraJoinPrefix()); + } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Migration.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Migration.java index b558ec4776..ace8c354e4 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Migration.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Migration.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.operator; import cn.edu.tsinghua.iginx.engine.shared.operator.type.OperatorType; @@ -51,4 +69,21 @@ public UnaryOperator copyWithSource(Source source) { public String getInfo() { return ""; } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (object == null || getClass() != object.getClass()) { + return false; + } + if (!super.equals(object)) { + return false; + } + Migration that = (Migration) object; + return fragmentMeta.equals(that.fragmentMeta) + && paths.equals(that.paths) + && targetStorageUnitMeta.equals(that.targetStorageUnitMeta); + } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/MultipleOperator.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/MultipleOperator.java index cd1132c68c..6d658406d9 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/MultipleOperator.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/MultipleOperator.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.operator; import cn.edu.tsinghua.iginx.engine.shared.source.Source; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Operator.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Operator.java index 93162f9c62..147bd4fb95 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Operator.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Operator.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.operator; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/OuterJoin.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/OuterJoin.java index 32c6fe9a61..2a2ea3f3b7 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/OuterJoin.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/OuterJoin.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.operator; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; @@ -169,4 +187,20 @@ public String getInfo() { } return builder.toString(); } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (object == null || getClass() != object.getClass()) { + return false; + } + OuterJoin that = (OuterJoin) object; + return outerJoinType == that.outerJoinType + && filter.equals(that.filter) + && joinColumns.equals(that.joinColumns) + && isNaturalJoin == that.isNaturalJoin + && getExtraJoinPrefix().equals(that.getExtraJoinPrefix()); + } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/PathUnion.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/PathUnion.java index 357fe177d3..56434a6bf4 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/PathUnion.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/PathUnion.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.operator; @@ -41,4 +40,12 @@ public BinaryOperator copyWithSource(Source sourceA, Source sourceB) { public String getInfo() { return ""; } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + return object != null && getClass() == object.getClass(); + } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Project.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Project.java index e58b6d6090..8282eb5a75 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Project.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Project.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.operator; import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; @@ -8,6 +26,7 @@ import cn.edu.tsinghua.iginx.metadata.entity.FragmentMeta; import java.util.ArrayList; import java.util.List; +import java.util.Objects; public class Project extends AbstractUnaryOperator { @@ -108,4 +127,19 @@ public String getInfo() { } return builder.toString(); } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (object == null || getClass() != object.getClass()) { + return false; + } + Project that = (Project) object; + return patterns.equals(that.patterns) + && (Objects.equals(tagFilter, that.tagFilter)) + && remainKey == that.remainKey + && needSelectedPath == that.needSelectedPath; + } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/ProjectWaitingForPath.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/ProjectWaitingForPath.java index 0bc20fd39f..346d0120e8 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/ProjectWaitingForPath.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/ProjectWaitingForPath.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.operator; import cn.edu.tsinghua.iginx.engine.shared.operator.type.OperatorType; @@ -37,4 +55,17 @@ public UnaryOperator copyWithSource(Source source) { public String getInfo() { return ""; } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (object == null || getClass() != object.getClass()) { + return false; + } + + ProjectWaitingForPath that = (ProjectWaitingForPath) object; + return incompleteStatement.equals(that.incompleteStatement); + } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Rename.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Rename.java index 45346f456d..5a26576916 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Rename.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Rename.java @@ -1,33 +1,50 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.operator; import cn.edu.tsinghua.iginx.engine.shared.operator.type.OperatorType; import cn.edu.tsinghua.iginx.engine.shared.source.Source; +import cn.edu.tsinghua.iginx.utils.Pair; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; public class Rename extends AbstractUnaryOperator { - private final Map aliasMap; + private final List> aliasList; private final List ignorePatterns; // 不进行重命名的列 - public Rename(Source source, Map aliasMap) { - this(source, aliasMap, new ArrayList<>()); + public Rename(Source source, List> aliasList) { + this(source, aliasList, new ArrayList<>()); } - public Rename(Source source, Map aliasMap, List ignorePatterns) { + public Rename(Source source, List> aliasList, List ignorePatterns) { super(OperatorType.Rename, source); - if (aliasMap == null) { - throw new IllegalArgumentException("aliasMap shouldn't be null"); + if (aliasList == null) { + throw new IllegalArgumentException("aliasList shouldn't be null"); } - this.aliasMap = aliasMap; + this.aliasList = aliasList; this.ignorePatterns = ignorePatterns; } - public Map getAliasMap() { - return aliasMap; + public List> getAliasList() { + return aliasList; } public List getIgnorePatterns() { @@ -36,25 +53,36 @@ public List getIgnorePatterns() { @Override public Operator copy() { - return new Rename(getSource().copy(), new HashMap<>(aliasMap)); + return new Rename(getSource().copy(), new ArrayList<>(aliasList)); } @Override public UnaryOperator copyWithSource(Source source) { - return new Rename(source, new HashMap<>(aliasMap)); + return new Rename(source, new ArrayList<>(aliasList)); } @Override public String getInfo() { StringBuilder builder = new StringBuilder(); - builder.append("AliasMap: "); - aliasMap.forEach( - (k, v) -> builder.append("(").append(k).append(", ").append(v).append(")").append(",")); + builder.append("AliasList: "); + aliasList.forEach( + p -> builder.append("(").append(p.k).append(", ").append(p.v).append(")").append(",")); builder.deleteCharAt(builder.length() - 1); if (!ignorePatterns.isEmpty()) { - builder.append(", IgnorePatterns: "); - builder.append(ignorePatterns); + builder.append(", IgnorePatterns: ").append(ignorePatterns); } return builder.toString(); } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (object == null || getClass() != object.getClass()) { + return false; + } + Rename rename = (Rename) object; + return aliasList.equals(rename.aliasList) && ignorePatterns.equals(rename.ignorePatterns); + } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Reorder.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Reorder.java index 0f609f87ab..c451366e9d 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Reorder.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Reorder.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.operator; import cn.edu.tsinghua.iginx.engine.shared.function.FunctionUtils; @@ -97,4 +115,18 @@ private static boolean isUdfPath(String path) { } return false; } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (object == null || getClass() != object.getClass()) { + return false; + } + Reorder reorder = (Reorder) object; + return patterns.equals(reorder.patterns) + && isPyUDF.equals(reorder.isPyUDF) + && needSelectedPath == reorder.needSelectedPath; + } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/RowTransform.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/RowTransform.java index f7e4739a28..4705c83437 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/RowTransform.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/RowTransform.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.operator; @@ -70,4 +69,16 @@ public String getInfo() { return sb.toString(); } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (object == null || getClass() != object.getClass()) { + return false; + } + RowTransform that = (RowTransform) object; + return functionCallList.equals(that.functionCallList); + } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Select.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Select.java index 2c2f371f5f..531db819e4 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Select.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Select.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.operator; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; @@ -56,4 +74,19 @@ public String getInfo() { } return builder.toString(); } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (object == null || getClass() != object.getClass()) { + return false; + } + if (!super.equals(object)) { + return false; + } + Select select = (Select) object; + return filter.equals(select.filter) && tagFilter.equals(select.tagFilter); + } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/SetTransform.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/SetTransform.java index 689f491e9e..336d018a57 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/SetTransform.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/SetTransform.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.operator; @@ -102,4 +101,16 @@ public String getInfo() { return sb.toString(); } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (object == null || getClass() != object.getClass()) { + return false; + } + SetTransform that = (SetTransform) object; + return functionCallList.equals(that.functionCallList); + } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/ShowColumns.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/ShowColumns.java index e01a7b6c75..69fc49f842 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/ShowColumns.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/ShowColumns.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.operator; import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; @@ -83,4 +101,19 @@ public String getInfo() { } return builder.toString(); } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (object == null || getClass() != object.getClass()) { + return false; + } + ShowColumns that = (ShowColumns) object; + return limit == that.limit + && offset == that.offset + && pathRegexSet.equals(that.pathRegexSet) + && tagFilter.equals(that.tagFilter); + } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/SingleJoin.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/SingleJoin.java index bc16a4ec41..773c7fd09f 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/SingleJoin.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/SingleJoin.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.operator; import static cn.edu.tsinghua.iginx.engine.shared.operator.type.JoinAlgType.chooseJoinAlg; @@ -90,4 +108,19 @@ public String getInfo() { } return builder.toString(); } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (object == null || getClass() != object.getClass()) { + return false; + } + if (!super.equals(object)) { + return false; + } + SingleJoin that = (SingleJoin) object; + return filter.equals(that.filter) && tagFilter.equals(that.tagFilter); + } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Sort.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Sort.java index efaa4f86e7..dbee447e64 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Sort.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Sort.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.operator; @@ -68,4 +67,16 @@ public enum SortType { public String getInfo() { return "SortBy: " + String.join(",", sortByCols) + ", SortType: " + sortType; } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (object == null || getClass() != object.getClass()) { + return false; + } + Sort sort = (Sort) object; + return sortByCols.equals(sort.sortByCols) && sortType == sort.sortType; + } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/UnaryOperator.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/UnaryOperator.java index 39bf9da77f..8ffbe5a80a 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/UnaryOperator.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/UnaryOperator.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.operator; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Union.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Union.java index ea2ae11b7c..c5d4d55ea4 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Union.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/Union.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.operator; import cn.edu.tsinghua.iginx.engine.shared.operator.type.OperatorType; @@ -77,4 +95,21 @@ public String getInfo() { builder.append(" isDistinct: ").append(isDistinct); return builder.toString(); } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (object == null || getClass() != object.getClass()) { + return false; + } + if (!super.equals(object)) { + return false; + } + Union union = (Union) object; + return leftOrder.equals(union.leftOrder) + && rightOrder.equals(union.rightOrder) + && isDistinct == union.isDistinct; + } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/ValueToSelectedPath.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/ValueToSelectedPath.java index a7bd115753..6775bf3b7d 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/ValueToSelectedPath.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/ValueToSelectedPath.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.operator; import cn.edu.tsinghua.iginx.engine.shared.operator.type.OperatorType; @@ -35,4 +53,19 @@ public String getInfo() { } return builder.toString(); } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (object == null || getClass() != object.getClass()) { + return false; + } + if (!super.equals(object)) { + return false; + } + ValueToSelectedPath that = (ValueToSelectedPath) object; + return prefix.equals(that.prefix); + } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/filter/AndFilter.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/filter/AndFilter.java index 54644f91a1..51e79a08f7 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/filter/AndFilter.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/filter/AndFilter.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.operator.filter; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/filter/BoolFilter.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/filter/BoolFilter.java index 4081b340e4..3d103aa79d 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/filter/BoolFilter.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/filter/BoolFilter.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.operator.filter; import java.util.Objects; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/filter/ExprFilter.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/filter/ExprFilter.java index cc8b78f44d..bd7746065e 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/filter/ExprFilter.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/filter/ExprFilter.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.operator.filter; import cn.edu.tsinghua.iginx.engine.shared.expr.Expression; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/filter/Filter.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/filter/Filter.java index f63f52cd37..69fb910cf6 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/filter/Filter.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/filter/Filter.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.operator.filter; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/filter/FilterType.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/filter/FilterType.java index 3c66321206..fa6c8601c7 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/filter/FilterType.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/filter/FilterType.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.operator.filter; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/filter/FilterVisitor.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/filter/FilterVisitor.java index 108913a628..75b7fe8ebf 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/filter/FilterVisitor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/filter/FilterVisitor.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.operator.filter; public interface FilterVisitor { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/filter/KeyFilter.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/filter/KeyFilter.java index be3a89bdf1..ff8298f0a2 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/filter/KeyFilter.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/filter/KeyFilter.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.operator.filter; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/filter/NotFilter.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/filter/NotFilter.java index 6f6a0bf09f..31902831ce 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/filter/NotFilter.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/filter/NotFilter.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.operator.filter; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/filter/Op.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/filter/Op.java index 73d8cd878f..770d931ea0 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/filter/Op.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/filter/Op.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.operator.filter; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/filter/OrFilter.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/filter/OrFilter.java index fbf8809b05..f8e63ffa50 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/filter/OrFilter.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/filter/OrFilter.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.operator.filter; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/filter/PathFilter.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/filter/PathFilter.java index 7bb19d168f..ef689230f2 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/filter/PathFilter.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/filter/PathFilter.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.operator.filter; import java.util.Objects; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/filter/ValueFilter.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/filter/ValueFilter.java index 09a4998d35..8c38e3f7d3 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/filter/ValueFilter.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/filter/ValueFilter.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.operator.filter; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/tag/AndTagFilter.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/tag/AndTagFilter.java index 56ec19c4c6..9eb747e559 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/tag/AndTagFilter.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/tag/AndTagFilter.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.operator.tag; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/tag/BasePreciseTagFilter.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/tag/BasePreciseTagFilter.java index e031344da1..04b909b2b3 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/tag/BasePreciseTagFilter.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/tag/BasePreciseTagFilter.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.operator.tag; import java.util.Map; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/tag/BaseTagFilter.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/tag/BaseTagFilter.java index 865c162d85..1713a5d062 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/tag/BaseTagFilter.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/tag/BaseTagFilter.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.operator.tag; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/tag/OrTagFilter.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/tag/OrTagFilter.java index 65e3b8cced..2a9598aec9 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/tag/OrTagFilter.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/tag/OrTagFilter.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.operator.tag; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/tag/PreciseTagFilter.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/tag/PreciseTagFilter.java index d890b49117..d9960c856a 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/tag/PreciseTagFilter.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/tag/PreciseTagFilter.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.operator.tag; import java.util.ArrayList; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/tag/TagFilter.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/tag/TagFilter.java index 171a32032d..7cb3792876 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/tag/TagFilter.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/tag/TagFilter.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.operator.tag; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/tag/TagFilterType.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/tag/TagFilterType.java index 7600b9c0f8..5db040c8f7 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/tag/TagFilterType.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/tag/TagFilterType.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.operator.tag; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/tag/WithoutTagFilter.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/tag/WithoutTagFilter.java index 83f001e361..9b805bc65d 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/tag/WithoutTagFilter.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/tag/WithoutTagFilter.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.operator.tag; public class WithoutTagFilter implements TagFilter { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/type/FuncType.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/type/FuncType.java index 809b6d2f30..995d69d32f 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/type/FuncType.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/type/FuncType.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.operator.type; public enum FuncType { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/type/JoinAlgType.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/type/JoinAlgType.java index f1e8fd0e87..dea2bc5bf5 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/type/JoinAlgType.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/type/JoinAlgType.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.operator.type; import cn.edu.tsinghua.iginx.engine.physical.memory.execute.utils.FilterUtils; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/type/OperatorType.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/type/OperatorType.java index cbf9ee3dbc..c697f3e2e3 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/type/OperatorType.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/type/OperatorType.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.operator.type; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/type/OuterJoinType.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/type/OuterJoinType.java index e2c35eb62c..f650c269c9 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/type/OuterJoinType.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/type/OuterJoinType.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.operator.type; public enum OuterJoinType { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/visitor/DeepFirstQueueVisitor.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/visitor/DeepFirstQueueVisitor.java index 3b2b11c2a5..191e510e08 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/visitor/DeepFirstQueueVisitor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/visitor/DeepFirstQueueVisitor.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.operator.visitor; import cn.edu.tsinghua.iginx.engine.shared.operator.BinaryOperator; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/visitor/IndexVisitor.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/visitor/IndexVisitor.java index 8da7c2e967..93099a1c06 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/visitor/IndexVisitor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/visitor/IndexVisitor.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.operator.visitor; import cn.edu.tsinghua.iginx.engine.shared.operator.BinaryOperator; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/visitor/OperatorInfoVisitor.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/visitor/OperatorInfoVisitor.java index d021729df3..a76658f8e5 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/visitor/OperatorInfoVisitor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/visitor/OperatorInfoVisitor.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.operator.visitor; import cn.edu.tsinghua.iginx.engine.shared.operator.BinaryOperator; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/visitor/OperatorVisitor.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/visitor/OperatorVisitor.java index db19ffa6de..e636650099 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/visitor/OperatorVisitor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/visitor/OperatorVisitor.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.operator.visitor; import cn.edu.tsinghua.iginx.engine.shared.operator.BinaryOperator; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/visitor/TreeInfoVisitor.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/visitor/TreeInfoVisitor.java index d0f21f0e39..1f5b0cdee4 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/visitor/TreeInfoVisitor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/operator/visitor/TreeInfoVisitor.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.operator.visitor; import cn.edu.tsinghua.iginx.engine.shared.operator.BinaryOperator; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/processor/PostExecuteProcessor.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/processor/PostExecuteProcessor.java index e7ef303e30..c7ab490688 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/processor/PostExecuteProcessor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/processor/PostExecuteProcessor.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.processor; public interface PostExecuteProcessor extends Processor {} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/processor/PostLogicalProcessor.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/processor/PostLogicalProcessor.java index 0fd4ed5c2f..2172f0eed6 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/processor/PostLogicalProcessor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/processor/PostLogicalProcessor.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.processor; public interface PostLogicalProcessor extends Processor {} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/processor/PostParseProcessor.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/processor/PostParseProcessor.java index a7197b99c5..425d53ae81 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/processor/PostParseProcessor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/processor/PostParseProcessor.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.processor; public interface PostParseProcessor extends Processor {} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/processor/PostPhysicalProcessor.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/processor/PostPhysicalProcessor.java index ea0c04858e..9d03887836 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/processor/PostPhysicalProcessor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/processor/PostPhysicalProcessor.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.processor; public interface PostPhysicalProcessor extends Processor {} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/processor/PreExecuteProcessor.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/processor/PreExecuteProcessor.java index ec7101963c..6153755987 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/processor/PreExecuteProcessor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/processor/PreExecuteProcessor.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.processor; public interface PreExecuteProcessor extends Processor {} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/processor/PreLogicalProcessor.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/processor/PreLogicalProcessor.java index 6def269021..f61afdb718 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/processor/PreLogicalProcessor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/processor/PreLogicalProcessor.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.processor; public interface PreLogicalProcessor extends Processor {} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/processor/PreParseProcessor.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/processor/PreParseProcessor.java index 02a6bf4db1..60040b7fc1 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/processor/PreParseProcessor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/processor/PreParseProcessor.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.processor; public interface PreParseProcessor extends Processor {} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/processor/PrePhysicalProcessor.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/processor/PrePhysicalProcessor.java index 59b515b282..9f3f808db4 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/processor/PrePhysicalProcessor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/processor/PrePhysicalProcessor.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.processor; public interface PrePhysicalProcessor extends Processor {} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/processor/Processor.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/processor/Processor.java index 3a25b03966..d40491a30c 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/processor/Processor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/processor/Processor.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.processor; import cn.edu.tsinghua.iginx.engine.shared.RequestContext; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/source/AbstractSource.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/source/AbstractSource.java index 84604867a9..d4d024ef5c 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/source/AbstractSource.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/source/AbstractSource.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.source; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/source/EmptySource.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/source/EmptySource.java index d91888d779..0bf4126f4f 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/source/EmptySource.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/source/EmptySource.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.source; public class EmptySource implements Source { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/source/FragmentSource.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/source/FragmentSource.java index 3c4254c466..b488e20b7e 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/source/FragmentSource.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/source/FragmentSource.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.source; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/source/GlobalSource.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/source/GlobalSource.java index ad178a8623..8032b613d3 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/source/GlobalSource.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/source/GlobalSource.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.shared.source; public class GlobalSource extends AbstractSource { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/source/OperatorSource.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/source/OperatorSource.java index 9972b2ba59..6dc74a16d0 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/source/OperatorSource.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/source/OperatorSource.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.source; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/source/Source.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/source/Source.java index d848017d3e..163ec34339 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/source/Source.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/source/Source.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.source; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/source/SourceType.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/source/SourceType.java index c207473c74..fb2a9eb2e4 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/source/SourceType.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/source/SourceType.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.source; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/exception/IginxException.java b/core/src/main/java/cn/edu/tsinghua/iginx/exception/IginxException.java index 87a051f3d2..cc37c09724 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/exception/IginxException.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/exception/IginxException.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.exception; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/DefaultMetaManager.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/DefaultMetaManager.java index 356e28be5e..59e95b1234 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/DefaultMetaManager.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/DefaultMetaManager.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.metadata; @@ -439,8 +438,11 @@ private void addStorageEngine(long storageEngineId, StorageEngineMeta storageEng public boolean removeDummyStorageEngine(long storageEngineId) { try { storage.removeDummyStorageEngine(storageEngineId); + // release 对接层 + for (StorageEngineChangeHook hook : storageEngineChangeHooks) { + hook.onChange(getStorageEngine(storageEngineId), null); + } return cache.removeDummyStorageEngine(storageEngineId); - // TODO 由于当前 StorageEngineChangeHook 和 StorageUnitHook 只会处理新增事件,因此不必调用相关 onChange 函数 } catch (MetaStorageException e) { LOGGER.error("remove dummy storage engine {} error: ", storageEngineId, e); } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/IMetaManager.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/IMetaManager.java index f47a37dbff..8a9c1a3f52 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/IMetaManager.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/IMetaManager.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.metadata; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/MetaManagerMock.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/MetaManagerMock.java index 4fac461607..3438c7a002 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/MetaManagerMock.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/MetaManagerMock.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.metadata; import cn.edu.tsinghua.iginx.metadata.entity.*; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/MetaManagerWrapper.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/MetaManagerWrapper.java index aeab5a637e..69fd0bc6b6 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/MetaManagerWrapper.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/MetaManagerWrapper.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.metadata; import cn.edu.tsinghua.iginx.conf.ConfigDescriptor; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/cache/DefaultMetaCache.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/cache/DefaultMetaCache.java index f0cddafb90..4dd3513e9c 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/cache/DefaultMetaCache.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/cache/DefaultMetaCache.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.metadata.cache; @@ -216,30 +215,36 @@ private static List searchFragmentList( @Override public void initFragment(Map> fragmentListMap) { storageUnitLock.readLock().lock(); - fragmentListMap - .values() - .forEach( - e -> - e.forEach( - f -> - f.setMasterStorageUnit( - storageUnitMetaMap.get(f.getMasterStorageUnitId())))); - storageUnitLock.readLock().unlock(); + try { + fragmentListMap + .values() + .forEach( + e -> + e.forEach( + f -> + f.setMasterStorageUnit( + storageUnitMetaMap.get(f.getMasterStorageUnitId())))); + } finally { + storageUnitLock.readLock().unlock(); + } fragmentLock.writeLock().lock(); - sortedFragmentMetaLists.addAll( - fragmentListMap.entrySet().stream() - .sorted(Map.Entry.comparingByKey()) - .map(e -> new Pair<>(e.getKey(), e.getValue())) - .collect(Collectors.toList())); - fragmentListMap.forEach(fragmentMetaListMap::put); - if (enableFragmentCacheControl) { - // 统计分片总数 - fragmentCacheSize = sortedFragmentMetaLists.stream().mapToInt(e -> e.v.size()).sum(); - while (fragmentCacheSize > fragmentCacheMaxSize) { - kickOffHistoryFragment(); + try { + sortedFragmentMetaLists.addAll( + fragmentListMap.entrySet().stream() + .sorted(Map.Entry.comparingByKey()) + .map(e -> new Pair<>(e.getKey(), e.getValue())) + .collect(Collectors.toList())); + fragmentListMap.forEach(fragmentMetaListMap::put); + if (enableFragmentCacheControl) { + // 统计分片总数 + fragmentCacheSize = sortedFragmentMetaLists.stream().mapToInt(e -> e.v.size()).sum(); + while (fragmentCacheSize > fragmentCacheMaxSize) { + kickOffHistoryFragment(); + } } + } finally { + fragmentLock.writeLock().unlock(); } - fragmentLock.writeLock().unlock(); } private void kickOffHistoryFragment() { @@ -262,26 +267,28 @@ private void kickOffHistoryFragment() { @Override public void addFragment(FragmentMeta fragmentMeta) { fragmentLock.writeLock().lock(); - // 更新 fragmentMetaListMap - List fragmentMetaList = - fragmentMetaListMap.computeIfAbsent( - fragmentMeta.getColumnsInterval(), v -> new ArrayList<>()); - if (fragmentMetaList.size() == 0) { - // 更新 sortedFragmentMetaLists - updateSortedFragmentsList(fragmentMeta.getColumnsInterval(), fragmentMetaList); - } - fragmentMetaList.add(fragmentMeta); - if (enableFragmentCacheControl) { - if (fragmentMeta.getKeyInterval().getStartKey() < minKey) { - minKey = fragmentMeta.getKeyInterval().getStartKey(); + try { + // 更新 fragmentMetaListMap + List fragmentMetaList = + fragmentMetaListMap.computeIfAbsent( + fragmentMeta.getColumnsInterval(), v -> new ArrayList<>()); + if (fragmentMetaList.size() == 0) { + // 更新 sortedFragmentMetaLists + updateSortedFragmentsList(fragmentMeta.getColumnsInterval(), fragmentMetaList); } - fragmentCacheSize++; - while (fragmentCacheSize > fragmentCacheMaxSize) { - kickOffHistoryFragment(); + fragmentMetaList.add(fragmentMeta); + if (enableFragmentCacheControl) { + if (fragmentMeta.getKeyInterval().getStartKey() < minKey) { + minKey = fragmentMeta.getKeyInterval().getStartKey(); + } + fragmentCacheSize++; + while (fragmentCacheSize > fragmentCacheMaxSize) { + kickOffHistoryFragment(); + } } + } finally { + fragmentLock.writeLock().unlock(); } - - fragmentLock.writeLock().unlock(); } private void updateSortedFragmentsList( @@ -313,11 +320,14 @@ private void updateSortedFragmentsList( @Override public void updateFragment(FragmentMeta fragmentMeta) { fragmentLock.writeLock().lock(); - // 更新 fragmentMetaListMap - List fragmentMetaList = - fragmentMetaListMap.get(fragmentMeta.getColumnsInterval()); - fragmentMetaList.set(fragmentMetaList.size() - 1, fragmentMeta); - fragmentLock.writeLock().unlock(); + try { + // 更新 fragmentMetaListMap + List fragmentMetaList = + fragmentMetaListMap.get(fragmentMeta.getColumnsInterval()); + fragmentMetaList.set(fragmentMetaList.size() - 1, fragmentMeta); + } finally { + fragmentLock.writeLock().unlock(); + } } @Override @@ -372,23 +382,29 @@ public Map> getFragmentMapByColumnsInterval( ColumnsInterval columnsInterval) { Map> resultMap = new HashMap<>(); fragmentLock.readLock().lock(); - searchFragmentSeriesList(sortedFragmentMetaLists, columnsInterval) - .forEach(e -> resultMap.put(e.k, e.v)); - fragmentLock.readLock().unlock(); + try { + searchFragmentSeriesList(sortedFragmentMetaLists, columnsInterval) + .forEach(e -> resultMap.put(e.k, e.v)); + } finally { + fragmentLock.readLock().unlock(); + } return resultMap; } @Override public List getDummyFragmentsByColumnsInterval(ColumnsInterval columnsInterval) { - fragmentLock.readLock().lock(); List results = new ArrayList<>(); - for (FragmentMeta fragmentMeta : dummyFragments) { - if (fragmentMeta.isValid() - && fragmentMeta.getColumnsInterval().isIntersect(columnsInterval)) { - results.add(fragmentMeta); + fragmentLock.readLock().lock(); + try { + for (FragmentMeta fragmentMeta : dummyFragments) { + if (fragmentMeta.isValid() + && fragmentMeta.getColumnsInterval().isIntersect(columnsInterval)) { + results.add(fragmentMeta); + } } + } finally { + fragmentLock.readLock().unlock(); } - fragmentLock.readLock().unlock(); return results; } @@ -396,11 +412,14 @@ public List getDummyFragmentsByColumnsInterval(ColumnsInterval col public Map getLatestFragmentMap() { Map latestFragmentMap = new HashMap<>(); fragmentLock.readLock().lock(); - sortedFragmentMetaLists.stream() - .map(e -> e.v.get(e.v.size() - 1)) - .filter(e -> e.getKeyInterval().getEndKey() == Long.MAX_VALUE) - .forEach(e -> latestFragmentMap.put(e.getColumnsInterval(), e)); - fragmentLock.readLock().unlock(); + try { + sortedFragmentMetaLists.stream() + .map(e -> e.v.get(e.v.size() - 1)) + .filter(e -> e.getKeyInterval().getEndKey() == Long.MAX_VALUE) + .forEach(e -> latestFragmentMap.put(e.getColumnsInterval(), e)); + } finally { + fragmentLock.readLock().unlock(); + } return latestFragmentMap; } @@ -409,11 +428,14 @@ public Map getLatestFragmentMapByColumnsInterval( ColumnsInterval columnsInterval) { Map latestFragmentMap = new HashMap<>(); fragmentLock.readLock().lock(); - searchFragmentSeriesList(sortedFragmentMetaLists, columnsInterval).stream() - .map(e -> e.v.get(e.v.size() - 1)) - .filter(e -> e.getKeyInterval().getEndKey() == Long.MAX_VALUE) - .forEach(e -> latestFragmentMap.put(e.getColumnsInterval(), e)); - fragmentLock.readLock().unlock(); + try { + searchFragmentSeriesList(sortedFragmentMetaLists, columnsInterval).stream() + .map(e -> e.v.get(e.v.size() - 1)) + .filter(e -> e.getKeyInterval().getEndKey() == Long.MAX_VALUE) + .forEach(e -> latestFragmentMap.put(e.getColumnsInterval(), e)); + } finally { + fragmentLock.readLock().unlock(); + } return latestFragmentMap; } @@ -422,31 +444,37 @@ public Map> getFragmentMapByColumnsIntervalA ColumnsInterval columnsInterval, KeyInterval keyInterval) { Map> resultMap = new HashMap<>(); fragmentLock.readLock().lock(); - searchFragmentSeriesList(sortedFragmentMetaLists, columnsInterval) - .forEach( - e -> { - List fragmentMetaList = searchFragmentList(e.v, keyInterval); - if (!fragmentMetaList.isEmpty()) { - resultMap.put(e.k, fragmentMetaList); - } - }); - fragmentLock.readLock().unlock(); + try { + searchFragmentSeriesList(sortedFragmentMetaLists, columnsInterval) + .forEach( + e -> { + List fragmentMetaList = searchFragmentList(e.v, keyInterval); + if (!fragmentMetaList.isEmpty()) { + resultMap.put(e.k, fragmentMetaList); + } + }); + } finally { + fragmentLock.readLock().unlock(); + } return resultMap; } @Override public List getDummyFragmentsByColumnsIntervalAndKeyInterval( ColumnsInterval columnsInterval, KeyInterval keyInterval) { - fragmentLock.readLock().lock(); List results = new ArrayList<>(); - for (FragmentMeta fragmentMeta : dummyFragments) { - if (fragmentMeta.isValid() - && fragmentMeta.getColumnsInterval().isIntersect(columnsInterval) - && fragmentMeta.getKeyInterval().isIntersect(keyInterval)) { - results.add(fragmentMeta); + fragmentLock.readLock().lock(); + try { + for (FragmentMeta fragmentMeta : dummyFragments) { + if (fragmentMeta.isValid() + && fragmentMeta.getColumnsInterval().isIntersect(columnsInterval) + && fragmentMeta.getKeyInterval().isIntersect(keyInterval)) { + results.add(fragmentMeta); + } } + } finally { + fragmentLock.readLock().unlock(); } - fragmentLock.readLock().unlock(); return results; } @@ -454,22 +482,25 @@ public List getDummyFragmentsByColumnsIntervalAndKeyInterval( public List getFragmentListByColumnName(String columnName) { List resultList; fragmentLock.readLock().lock(); - resultList = - searchFragmentSeriesList(sortedFragmentMetaLists, columnName).stream() - .map(e -> e.v) - .flatMap(List::stream) - .sorted( - (o1, o2) -> { - if (o1.getColumnsInterval().getStartColumn() == null - && o2.getColumnsInterval().getStartColumn() == null) return 0; - else if (o1.getColumnsInterval().getStartColumn() == null) return -1; - else if (o2.getColumnsInterval().getStartColumn() == null) return 1; - return o1.getColumnsInterval() - .getStartColumn() - .compareTo(o2.getColumnsInterval().getStartColumn()); - }) - .collect(Collectors.toList()); - fragmentLock.readLock().unlock(); + try { + resultList = + searchFragmentSeriesList(sortedFragmentMetaLists, columnName).stream() + .map(e -> e.v) + .flatMap(List::stream) + .sorted( + (o1, o2) -> { + if (o1.getColumnsInterval().getStartColumn() == null + && o2.getColumnsInterval().getStartColumn() == null) return 0; + else if (o1.getColumnsInterval().getStartColumn() == null) return -1; + else if (o2.getColumnsInterval().getStartColumn() == null) return 1; + return o1.getColumnsInterval() + .getStartColumn() + .compareTo(o2.getColumnsInterval().getStartColumn()); + }) + .collect(Collectors.toList()); + } finally { + fragmentLock.readLock().unlock(); + } return resultList; } @@ -477,14 +508,17 @@ public List getFragmentListByColumnName(String columnName) { public FragmentMeta getLatestFragmentByColumnName(String columnName) { FragmentMeta result; fragmentLock.readLock().lock(); - result = - searchFragmentSeriesList(sortedFragmentMetaLists, columnName).stream() - .map(e -> e.v) - .flatMap(List::stream) - .filter(e -> e.getKeyInterval().getEndKey() == Long.MAX_VALUE) - .findFirst() - .orElse(null); - fragmentLock.readLock().unlock(); + try { + result = + searchFragmentSeriesList(sortedFragmentMetaLists, columnName).stream() + .map(e -> e.v) + .flatMap(List::stream) + .filter(e -> e.getKeyInterval().getEndKey() == Long.MAX_VALUE) + .findFirst() + .orElse(null); + } finally { + fragmentLock.readLock().unlock(); + } return result; } @@ -508,14 +542,17 @@ public List getFragmentListByColumnNameAndKeyInterval( String columnName, KeyInterval keyInterval) { List resultList; fragmentLock.readLock().lock(); - List fragmentMetas = - searchFragmentSeriesList(sortedFragmentMetaLists, columnName).stream() - .map(e -> e.v) - .flatMap(List::stream) - .sorted(Comparator.comparingLong(o -> o.getKeyInterval().getStartKey())) - .collect(Collectors.toList()); - resultList = searchFragmentList(fragmentMetas, keyInterval); - fragmentLock.readLock().unlock(); + try { + List fragmentMetas = + searchFragmentSeriesList(sortedFragmentMetaLists, columnName).stream() + .map(e -> e.v) + .flatMap(List::stream) + .sorted(Comparator.comparingLong(o -> o.getKeyInterval().getStartKey())) + .collect(Collectors.toList()); + resultList = searchFragmentList(fragmentMetas, keyInterval); + } finally { + fragmentLock.readLock().unlock(); + } return resultList; } @@ -523,14 +560,17 @@ public List getFragmentListByColumnNameAndKeyInterval( public List getFragmentListByStorageUnitId(String storageUnitId) { List resultList; fragmentLock.readLock().lock(); - List fragmentMetas = - sortedFragmentMetaLists.stream() - .map(e -> e.v) - .flatMap(List::stream) - .sorted(Comparator.comparingLong(o -> o.getKeyInterval().getStartKey())) - .collect(Collectors.toList()); - resultList = searchFragmentList(fragmentMetas, storageUnitId); - fragmentLock.readLock().unlock(); + try { + List fragmentMetas = + sortedFragmentMetaLists.stream() + .map(e -> e.v) + .flatMap(List::stream) + .sorted(Comparator.comparingLong(o -> o.getKeyInterval().getStartKey())) + .collect(Collectors.toList()); + resultList = searchFragmentList(fragmentMetas, storageUnitId); + } finally { + fragmentLock.readLock().unlock(); + } return resultList; } @@ -547,22 +587,28 @@ public boolean hasStorageUnit() { @Override public void initStorageUnit(Map storageUnits) { storageUnitLock.writeLock().lock(); - for (StorageUnitMeta storageUnit : storageUnits.values()) { - storageUnitMetaMap.put(storageUnit.getId(), storageUnit); - getStorageEngine(storageUnit.getStorageEngineId()).addStorageUnit(storageUnit); + try { + for (StorageUnitMeta storageUnit : storageUnits.values()) { + storageUnitMetaMap.put(storageUnit.getId(), storageUnit); + getStorageEngine(storageUnit.getStorageEngineId()).addStorageUnit(storageUnit); + } + } finally { + storageUnitLock.writeLock().unlock(); } - storageUnitLock.writeLock().unlock(); } @Override public StorageUnitMeta getStorageUnit(String id) { StorageUnitMeta storageUnit; storageUnitLock.readLock().lock(); - storageUnit = storageUnitMetaMap.get(id); - if (storageUnit == null) { - storageUnit = dummyStorageUnitMetaMap.get(id); + try { + storageUnit = storageUnitMetaMap.get(id); + if (storageUnit == null) { + storageUnit = dummyStorageUnitMetaMap.get(id); + } + } finally { + storageUnitLock.readLock().unlock(); } - storageUnitLock.readLock().unlock(); return storageUnit; } @@ -570,18 +616,21 @@ public StorageUnitMeta getStorageUnit(String id) { public Map getStorageUnits(Set ids) { Map resultMap = new HashMap<>(); storageUnitLock.readLock().lock(); - for (String id : ids) { - StorageUnitMeta storageUnit = storageUnitMetaMap.get(id); - if (storageUnit != null) { - resultMap.put(id, storageUnit); - } else { - storageUnit = dummyStorageUnitMetaMap.get(id); + try { + for (String id : ids) { + StorageUnitMeta storageUnit = storageUnitMetaMap.get(id); if (storageUnit != null) { resultMap.put(id, storageUnit); + } else { + storageUnit = dummyStorageUnitMetaMap.get(id); + if (storageUnit != null) { + resultMap.put(id, storageUnit); + } } } + } finally { + storageUnitLock.readLock().unlock(); } - storageUnitLock.readLock().unlock(); return resultMap; } @@ -589,24 +638,33 @@ public Map getStorageUnits(Set ids) { public List getStorageUnits() { List storageUnitMetaList; storageUnitLock.readLock().lock(); - storageUnitMetaList = new ArrayList<>(storageUnitMetaMap.values()); - storageUnitMetaList.addAll(dummyStorageUnitMetaMap.values()); - storageUnitLock.readLock().unlock(); + try { + storageUnitMetaList = new ArrayList<>(storageUnitMetaMap.values()); + storageUnitMetaList.addAll(dummyStorageUnitMetaMap.values()); + } finally { + storageUnitLock.readLock().unlock(); + } return storageUnitMetaList; } @Override public void addStorageUnit(StorageUnitMeta storageUnitMeta) { storageUnitLock.writeLock().lock(); - storageUnitMetaMap.put(storageUnitMeta.getId(), storageUnitMeta); - storageUnitLock.writeLock().unlock(); + try { + storageUnitMetaMap.put(storageUnitMeta.getId(), storageUnitMeta); + } finally { + storageUnitLock.writeLock().unlock(); + } } @Override public void updateStorageUnit(StorageUnitMeta storageUnitMeta) { storageUnitLock.writeLock().lock(); - storageUnitMetaMap.put(storageUnitMeta.getId(), storageUnitMeta); - storageUnitLock.writeLock().unlock(); + try { + storageUnitMetaMap.put(storageUnitMeta.getId(), storageUnitMeta); + } finally { + storageUnitLock.writeLock().unlock(); + } } @Override @@ -628,38 +686,42 @@ public void removeIginx(long id) { public void addStorageEngine(StorageEngineMeta storageEngineMeta) { storageUnitLock.writeLock().lock(); fragmentLock.writeLock().lock(); - if (!storageEngineMetaMap.containsKey(storageEngineMeta.getId())) { - storageEngineMetaMap.put(storageEngineMeta.getId(), storageEngineMeta); - if (storageEngineMeta.isHasData()) { - StorageUnitMeta dummyStorageUnit = storageEngineMeta.getDummyStorageUnit(); - FragmentMeta dummyFragment = storageEngineMeta.getDummyFragment(); - dummyFragment.setMasterStorageUnit(dummyStorageUnit); - dummyStorageUnitMetaMap.put(dummyStorageUnit.getId(), dummyStorageUnit); - dummyFragments.add(dummyFragment); + try { + if (!storageEngineMetaMap.containsKey(storageEngineMeta.getId())) { + storageEngineMetaMap.put(storageEngineMeta.getId(), storageEngineMeta); + if (storageEngineMeta.isHasData()) { + StorageUnitMeta dummyStorageUnit = storageEngineMeta.getDummyStorageUnit(); + FragmentMeta dummyFragment = storageEngineMeta.getDummyFragment(); + dummyFragment.setMasterStorageUnit(dummyStorageUnit); + dummyStorageUnitMetaMap.put(dummyStorageUnit.getId(), dummyStorageUnit); + dummyFragments.add(dummyFragment); + } } + } finally { + fragmentLock.writeLock().unlock(); + storageUnitLock.writeLock().unlock(); } - fragmentLock.writeLock().unlock(); - storageUnitLock.writeLock().unlock(); } @Override public boolean removeDummyStorageEngine(long storageEngineId) { storageUnitLock.writeLock().lock(); fragmentLock.writeLock().lock(); - - if (!storageEngineMetaMap.containsKey(storageEngineId)) { - LOGGER.error("unexpected dummy storage engine {} to be removed", storageEngineId); - return false; + try { + if (!storageEngineMetaMap.containsKey(storageEngineId)) { + LOGGER.error("unexpected dummy storage engine {} to be removed", storageEngineId); + return false; + } + String dummyStorageUnitId = generateDummyStorageUnitId(storageEngineId); + StorageEngineMeta oldStorageEngineMeta = storageEngineMetaMap.get(storageEngineId); + assert oldStorageEngineMeta.isHasData(); + dummyFragments.removeIf(e -> e.getMasterStorageUnitId().equals(dummyStorageUnitId)); + dummyStorageUnitMetaMap.remove(dummyStorageUnitId); + storageEngineMetaMap.remove(storageEngineId); + } finally { + fragmentLock.writeLock().unlock(); + storageUnitLock.writeLock().unlock(); } - String dummyStorageUnitId = generateDummyStorageUnitId(storageEngineId); - StorageEngineMeta oldStorageEngineMeta = storageEngineMetaMap.get(storageEngineId); - assert oldStorageEngineMeta.isHasData(); - dummyFragments.removeIf(e -> e.getMasterStorageUnitId().equals(dummyStorageUnitId)); - dummyStorageUnitMetaMap.remove(dummyStorageUnitId); - storageEngineMetaMap.remove(storageEngineId); - - fragmentLock.writeLock().unlock(); - storageUnitLock.writeLock().unlock(); return true; } @@ -676,11 +738,14 @@ public StorageEngineMeta getStorageEngine(long id) { @Override public List getFragments() { List fragments = new ArrayList<>(); - this.fragmentLock.readLock().lock(); - for (Pair> pair : sortedFragmentMetaLists) { - fragments.addAll(pair.v); + fragmentLock.readLock().lock(); + try { + for (Pair> pair : sortedFragmentMetaLists) { + fragments.addAll(pair.v); + } + } finally { + fragmentLock.readLock().unlock(); } - this.fragmentLock.readLock().unlock(); return fragments; } @@ -761,68 +826,72 @@ public void timeSeriesIsUpdated(int node, int version) { @Override public void saveColumnsData(InsertStatement statement) { insertRecordLock.writeLock().lock(); - long now = System.currentTimeMillis(); - - RawData data = statement.getRawData(); - List paths = data.getPaths(); - if (data.isColumnData()) { - DataView view = new ColumnDataView(data, 0, data.getPaths().size(), 0, data.getKeys().size()); - for (int i = 0; i < view.getPathNum(); i++) { - long minn = Long.MAX_VALUE; - long maxx = Long.MIN_VALUE; - long totalByte = 0L; - int count = 0; - BitmapView bitmapView = view.getBitmapView(i); - for (int j = 0; j < view.getKeySize(); j++) { - if (bitmapView.get(j)) { - minn = Math.min(minn, view.getKey(j)); - maxx = Math.max(maxx, view.getKey(j)); - if (view.getDataType(i) == DataType.BINARY) { - totalByte += ((byte[]) view.getValue(i, j)).length; - } else { - totalByte += transDatatypeToByte(view.getDataType(i)); + try { + long now = System.currentTimeMillis(); + + RawData data = statement.getRawData(); + List paths = data.getPaths(); + if (data.isColumnData()) { + DataView view = + new ColumnDataView(data, 0, data.getPaths().size(), 0, data.getKeys().size()); + for (int i = 0; i < view.getPathNum(); i++) { + long minn = Long.MAX_VALUE; + long maxx = Long.MIN_VALUE; + long totalByte = 0L; + int count = 0; + BitmapView bitmapView = view.getBitmapView(i); + for (int j = 0; j < view.getKeySize(); j++) { + if (bitmapView.get(j)) { + minn = Math.min(minn, view.getKey(j)); + maxx = Math.max(maxx, view.getKey(j)); + if (view.getDataType(i) == DataType.BINARY) { + totalByte += ((byte[]) view.getValue(i, j)).length; + } else { + totalByte += transDatatypeToByte(view.getDataType(i)); + } + count++; } - count++; + } + if (count > 0) { + updateColumnCalDOConcurrentHashMap(paths.get(i), now, minn, maxx, totalByte, count); } } - if (count > 0) { - updateColumnCalDOConcurrentHashMap(paths.get(i), now, minn, maxx, totalByte, count); - } - } - } else { - DataView view = new RowDataView(data, 0, data.getPaths().size(), 0, data.getKeys().size()); - long[] totalByte = new long[view.getPathNum()]; - int[] count = new int[view.getPathNum()]; - long[] minn = new long[view.getPathNum()]; - long[] maxx = new long[view.getPathNum()]; - Arrays.fill(minn, Long.MAX_VALUE); - Arrays.fill(maxx, Long.MIN_VALUE); - - for (int i = 0; i < view.getKeySize(); i++) { - BitmapView bitmapView = view.getBitmapView(i); - int index = 0; - for (int j = 0; j < view.getPathNum(); j++) { - if (bitmapView.get(j)) { - minn[j] = Math.min(minn[j], view.getKey(i)); - maxx[j] = Math.max(maxx[j], view.getKey(i)); - if (view.getDataType(j) == DataType.BINARY) { - totalByte[j] += ((byte[]) view.getValue(i, index)).length; - } else { - totalByte[j] += transDatatypeToByte(view.getDataType(j)); + } else { + DataView view = new RowDataView(data, 0, data.getPaths().size(), 0, data.getKeys().size()); + long[] totalByte = new long[view.getPathNum()]; + int[] count = new int[view.getPathNum()]; + long[] minn = new long[view.getPathNum()]; + long[] maxx = new long[view.getPathNum()]; + Arrays.fill(minn, Long.MAX_VALUE); + Arrays.fill(maxx, Long.MIN_VALUE); + + for (int i = 0; i < view.getKeySize(); i++) { + BitmapView bitmapView = view.getBitmapView(i); + int index = 0; + for (int j = 0; j < view.getPathNum(); j++) { + if (bitmapView.get(j)) { + minn[j] = Math.min(minn[j], view.getKey(i)); + maxx[j] = Math.max(maxx[j], view.getKey(i)); + if (view.getDataType(j) == DataType.BINARY) { + totalByte[j] += ((byte[]) view.getValue(i, index)).length; + } else { + totalByte[j] += transDatatypeToByte(view.getDataType(j)); + } + count[j]++; + index++; } - count[j]++; - index++; } } - } - for (int i = 0; i < count.length; i++) { - if (count[i] > 0) { - updateColumnCalDOConcurrentHashMap( - paths.get(i), now, minn[i], maxx[i], totalByte[i], count[i]); + for (int i = 0; i < count.length; i++) { + if (count[i] > 0) { + updateColumnCalDOConcurrentHashMap( + paths.get(i), now, minn[i], maxx[i], totalByte[i], count[i]); + } } } + } finally { + insertRecordLock.writeLock().unlock(); } - insertRecordLock.writeLock().unlock(); } private void updateColumnCalDOConcurrentHashMap( @@ -853,21 +922,28 @@ private long transDatatypeToByte(DataType dataType) { @Override public List getMaxValueFromColumns() { + List ret; insertRecordLock.readLock().lock(); - List ret = - columnCalDOConcurrentHashMap.values().stream() - .filter(e -> random.nextDouble() < config.getCachedTimeseriesProb()) - .collect(Collectors.toList()); - insertRecordLock.readLock().unlock(); + try { + ret = + columnCalDOConcurrentHashMap.values().stream() + .filter(e -> random.nextDouble() < config.getCachedTimeseriesProb()) + .collect(Collectors.toList()); + } finally { + insertRecordLock.readLock().unlock(); + } return ret; } @Override public double getSumFromColumns() { + double ret; insertRecordLock.readLock().lock(); - double ret = - columnCalDOConcurrentHashMap.values().stream().mapToDouble(ColumnCalDO::getValue).sum(); - insertRecordLock.readLock().unlock(); + try { + ret = columnCalDOConcurrentHashMap.values().stream().mapToDouble(ColumnCalDO::getValue).sum(); + } finally { + insertRecordLock.readLock().unlock(); + } return ret; } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/cache/IMetaCache.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/cache/IMetaCache.java index 350f575c4d..083b022140 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/cache/IMetaCache.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/cache/IMetaCache.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.metadata.cache; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/entity/ColumnsInterval.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/entity/ColumnsInterval.java index 6686eff59c..36d9b1c10e 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/entity/ColumnsInterval.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/entity/ColumnsInterval.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.metadata.entity; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/entity/FragmentMeta.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/entity/FragmentMeta.java index 67815532d9..bdb8205510 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/entity/FragmentMeta.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/entity/FragmentMeta.java @@ -1,38 +1,36 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.metadata.entity; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/entity/IginxMeta.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/entity/IginxMeta.java index baaa00b901..6e5011d62f 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/entity/IginxMeta.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/entity/IginxMeta.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.metadata.entity; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/entity/KeyInterval.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/entity/KeyInterval.java index d9e9d78e22..ed54d672cd 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/entity/KeyInterval.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/entity/KeyInterval.java @@ -1,23 +1,25 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.metadata.entity; +import static cn.edu.tsinghua.iginx.constant.GlobalConstant.KEY_MAX_VAL; +import static cn.edu.tsinghua.iginx.constant.GlobalConstant.KEY_MIN_VAL; + import java.util.Objects; public final class KeyInterval { @@ -89,6 +91,6 @@ public KeyInterval getIntersectWithLCRO(KeyInterval keyInterval) { } public static KeyInterval getDefaultKeyInterval() { - return new KeyInterval(Long.MIN_VALUE, Long.MAX_VALUE); + return new KeyInterval(KEY_MIN_VAL, KEY_MAX_VAL); } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/entity/StorageEngineMeta.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/entity/StorageEngineMeta.java index 689b317d79..991329ba1c 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/entity/StorageEngineMeta.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/entity/StorageEngineMeta.java @@ -1,27 +1,30 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.metadata.entity; +import static cn.edu.tsinghua.iginx.utils.HostUtils.isLocalHost; + import cn.edu.tsinghua.iginx.thrift.StorageEngineType; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Objects; +import javax.validation.constraints.NotNull; public final class StorageEngineMeta { @@ -213,6 +216,10 @@ public void setExtraParams(Map extraParams) { this.extraParams = extraParams; } + public void updateExtraParams(Map newParams) { + this.extraParams.putAll(newParams); + } + public StorageEngineType getStorageEngine() { return storageEngine; } @@ -278,6 +285,29 @@ public void setNeedReAllocate(boolean needReAllocate) { this.needReAllocate = needReAllocate; } + public static String extractEmbeddedPrefix(@NotNull String dummyDirPath) { + if (dummyDirPath.isEmpty()) { + return null; + } + String separator = System.getProperty("file.separator"); + // dummyDirPath是规范路径,一定不会以separator结尾 + String prefix = dummyDirPath.substring(dummyDirPath.lastIndexOf(separator) + 1); + // "/" also can be used on windows, just in case + return prefix.substring(prefix.lastIndexOf("/") + 1); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + StorageEngineMeta that = (StorageEngineMeta) o; + return (ip.equals(that.ip) || (isLocalHost(ip) && isLocalHost(that.ip))) + && port == that.port + && storageEngine == that.storageEngine + && Objects.equals(schemaPrefix, that.getSchemaPrefix()) + && Objects.equals(dataPrefix, that.getDataPrefix()); + } + @Override public String toString() { return "StorageEngineMeta {" diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/entity/StorageUnitMeta.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/entity/StorageUnitMeta.java index bcfa84763e..8f5bb698f0 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/entity/StorageUnitMeta.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/entity/StorageUnitMeta.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.metadata.entity; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/entity/TransformTaskMeta.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/entity/TransformTaskMeta.java index eff7f4abc2..24055fea9f 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/entity/TransformTaskMeta.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/entity/TransformTaskMeta.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.metadata.entity; import cn.edu.tsinghua.iginx.thrift.UDFType; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/entity/UserMeta.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/entity/UserMeta.java index 74ed4081d5..cab0a919f1 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/entity/UserMeta.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/entity/UserMeta.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.metadata.entity; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/exception/MetaStorageException.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/exception/MetaStorageException.java index 0741a10b6b..c6dc8e64ec 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/exception/MetaStorageException.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/exception/MetaStorageException.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.metadata.exception; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/EnableMonitorChangeHook.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/EnableMonitorChangeHook.java index fe68217cba..95f2e37072 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/EnableMonitorChangeHook.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/EnableMonitorChangeHook.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.metadata.hook; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/FragmentChangeHook.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/FragmentChangeHook.java index 9811cb561e..70def91f30 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/FragmentChangeHook.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/FragmentChangeHook.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.metadata.hook; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/IginxChangeHook.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/IginxChangeHook.java index 7c1e3c2c6e..fcddf7ef26 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/IginxChangeHook.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/IginxChangeHook.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.metadata.hook; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/MaxActiveEndKeyStatisticsChangeHook.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/MaxActiveEndKeyStatisticsChangeHook.java index 16c146842f..8c67c8d340 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/MaxActiveEndKeyStatisticsChangeHook.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/MaxActiveEndKeyStatisticsChangeHook.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.metadata.hook; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/ReshardCounterChangeHook.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/ReshardCounterChangeHook.java index dbb54c60c6..99cb403b19 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/ReshardCounterChangeHook.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/ReshardCounterChangeHook.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.metadata.hook; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/ReshardStatusChangeHook.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/ReshardStatusChangeHook.java index 47314c00ae..c3c2e2a63c 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/ReshardStatusChangeHook.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/ReshardStatusChangeHook.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.metadata.hook; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/SchemaMappingChangeHook.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/SchemaMappingChangeHook.java index 46cbfa1597..109cc878ee 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/SchemaMappingChangeHook.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/SchemaMappingChangeHook.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.metadata.hook; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/StorageChangeHook.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/StorageChangeHook.java index c7c8adee7d..f17ac27db6 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/StorageChangeHook.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/StorageChangeHook.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.metadata.hook; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/StorageEngineChangeHook.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/StorageEngineChangeHook.java index 75e914886c..0466112d07 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/StorageEngineChangeHook.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/StorageEngineChangeHook.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.metadata.hook; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/StorageUnitChangeHook.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/StorageUnitChangeHook.java index faa07a88a7..2469958a0e 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/StorageUnitChangeHook.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/StorageUnitChangeHook.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.metadata.hook; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/StorageUnitHook.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/StorageUnitHook.java index 74ee074672..9ae348fc13 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/StorageUnitHook.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/StorageUnitHook.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.metadata.hook; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/TimeSeriesChangeHook.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/TimeSeriesChangeHook.java index f74a120aac..63e7b9acdc 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/TimeSeriesChangeHook.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/TimeSeriesChangeHook.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.metadata.hook; public interface TimeSeriesChangeHook { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/TransformChangeHook.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/TransformChangeHook.java index 2030f10d67..61e8cc8f95 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/TransformChangeHook.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/TransformChangeHook.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.metadata.hook; import cn.edu.tsinghua.iginx.metadata.entity.TransformTaskMeta; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/UserChangeHook.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/UserChangeHook.java index 2745f817f0..9cf1a0639d 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/UserChangeHook.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/UserChangeHook.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.metadata.hook; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/VersionChangeHook.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/VersionChangeHook.java index a8b5f262ec..8a83308ad8 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/VersionChangeHook.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/hook/VersionChangeHook.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.metadata.hook; public interface VersionChangeHook { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/storage/IMetaStorage.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/storage/IMetaStorage.java index b8d4bff2f1..728ebfcd37 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/storage/IMetaStorage.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/storage/IMetaStorage.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.metadata.storage; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/storage/constant/Constant.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/storage/constant/Constant.java index b7b6dcfa0a..d186bb97e6 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/storage/constant/Constant.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/storage/constant/Constant.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.metadata.storage.constant; public class Constant { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/storage/etcd/ETCDMetaStorage.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/storage/etcd/ETCDMetaStorage.java index cfb420dcad..c60eb0457a 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/storage/etcd/ETCDMetaStorage.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/storage/etcd/ETCDMetaStorage.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.metadata.storage.etcd; @@ -122,7 +121,7 @@ public class ETCDMetaStorage implements IMetaStorage { private final int STORAGE_ENGINE_NODE_LENGTH = 10; private String generateID(String prefix, long idLength, long val) { - return String.format(prefix + "%0" + idLength + "d", (int) val); + return String.format(prefix + "%0" + idLength + "d", val); } public ETCDMetaStorage() { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/storage/zk/ZooKeeperMetaStorage.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/storage/zk/ZooKeeperMetaStorage.java index a84944e4ff..aae1bc1ae2 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/storage/zk/ZooKeeperMetaStorage.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/storage/zk/ZooKeeperMetaStorage.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.metadata.storage.zk; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/sync/proposal/ProposalListener.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/sync/proposal/ProposalListener.java index a28d18bec4..de9d81e940 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/sync/proposal/ProposalListener.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/sync/proposal/ProposalListener.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.metadata.sync.proposal; public interface ProposalListener { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/sync/proposal/SyncProposal.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/sync/proposal/SyncProposal.java index 4a0969bae7..7dfdedd80d 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/sync/proposal/SyncProposal.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/sync/proposal/SyncProposal.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.metadata.sync.proposal; public class SyncProposal { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/sync/proposal/SyncVote.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/sync/proposal/SyncVote.java index 0cc3fc9951..b421e341cd 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/sync/proposal/SyncVote.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/sync/proposal/SyncVote.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.metadata.sync.proposal; public class SyncVote { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/sync/proposal/VoteListener.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/sync/proposal/VoteListener.java index e1fc200ad8..cbff39a55b 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/sync/proposal/VoteListener.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/sync/proposal/VoteListener.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.metadata.sync.proposal; public interface VoteListener { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/sync/protocol/ExecutionException.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/sync/protocol/ExecutionException.java index cccd0f37ff..a240c327f8 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/sync/protocol/ExecutionException.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/sync/protocol/ExecutionException.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.metadata.sync.protocol; public class ExecutionException extends Exception { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/sync/protocol/NetworkException.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/sync/protocol/NetworkException.java index 219f573b5b..8cacb3c714 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/sync/protocol/NetworkException.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/sync/protocol/NetworkException.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.metadata.sync.protocol; public class NetworkException extends Exception { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/sync/protocol/SyncProtocol.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/sync/protocol/SyncProtocol.java index 8752c8faac..3790256d27 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/sync/protocol/SyncProtocol.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/sync/protocol/SyncProtocol.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.metadata.sync.protocol; import cn.edu.tsinghua.iginx.metadata.sync.proposal.ProposalListener; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/sync/protocol/VoteExpiredException.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/sync/protocol/VoteExpiredException.java index aa69423a78..a78e0f4713 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/sync/protocol/VoteExpiredException.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/sync/protocol/VoteExpiredException.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.metadata.sync.protocol; public class VoteExpiredException extends Exception { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/sync/protocol/etcd/ETCDSyncProtocolImpl.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/sync/protocol/etcd/ETCDSyncProtocolImpl.java index a2111e2218..fc56388cd7 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/sync/protocol/etcd/ETCDSyncProtocolImpl.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/sync/protocol/etcd/ETCDSyncProtocolImpl.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.metadata.sync.protocol.etcd; import cn.edu.tsinghua.iginx.metadata.sync.proposal.ProposalListener; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/sync/protocol/zk/ZooKeeperSyncProtocolImpl.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/sync/protocol/zk/ZooKeeperSyncProtocolImpl.java index 5e39eaa3cf..284e58ab9d 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/sync/protocol/zk/ZooKeeperSyncProtocolImpl.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/sync/protocol/zk/ZooKeeperSyncProtocolImpl.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.metadata.sync.protocol.zk; import cn.edu.tsinghua.iginx.metadata.sync.proposal.ProposalListener; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/utils/ColumnsIntervalUtils.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/utils/ColumnsIntervalUtils.java index ddc85698ab..db887f1471 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/utils/ColumnsIntervalUtils.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/utils/ColumnsIntervalUtils.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.metadata.utils; import cn.edu.tsinghua.iginx.metadata.entity.ColumnsInterval; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/utils/FragmentUtils.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/utils/FragmentUtils.java index 009f8a439a..b508fa78c0 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/utils/FragmentUtils.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/utils/FragmentUtils.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.metadata.utils; import cn.edu.tsinghua.iginx.metadata.entity.ColumnsInterval; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/utils/IdUtils.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/utils/IdUtils.java index d6181c3245..b28feef207 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/utils/IdUtils.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/utils/IdUtils.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.metadata.utils; import static cn.edu.tsinghua.iginx.conf.Constants.LENGTH_OF_SEQUENCE_NUMBER; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/utils/ReshardStatus.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/utils/ReshardStatus.java index 9a4d25eab2..cd78980b3c 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/utils/ReshardStatus.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/utils/ReshardStatus.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.metadata.utils; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/utils/StorageEngineUtils.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/utils/StorageEngineUtils.java index 7130b56892..090a350d68 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/utils/StorageEngineUtils.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/utils/StorageEngineUtils.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.metadata.utils; import static cn.edu.tsinghua.iginx.conf.Constants.*; @@ -42,7 +60,7 @@ public static boolean checkEmbeddedStorageExtraParams( if (iginxPort == null || iginxPort.isEmpty()) { return false; } - boolean hasData = Boolean.parseBoolean(extraParams.getOrDefault(HAS_DATA, "false")); + boolean hasData = Boolean.parseBoolean(extraParams.getOrDefault(Constants.HAS_DATA, "false")); boolean readOnly = Boolean.parseBoolean(extraParams.getOrDefault(Constants.IS_READ_ONLY, "false")); if (hasData) { @@ -63,10 +81,7 @@ public static boolean checkEmbeddedStorageExtraParams( return false; } } - String separator = System.getProperty("file.separator"); - // dummyDirPath是规范路径,一定不会以separator结尾 - String dirPrefix = dummyDirPath.substring(dummyDirPath.lastIndexOf(separator) + 1); - extraParams.put(EMBEDDED_PREFIX, dirPrefix); + extraParams.put(EMBEDDED_PREFIX, StorageEngineMeta.extractEmbeddedPrefix(dummyDirPath)); } else { // hasData=false readOnly=true 无意义的引擎 if (readOnly) { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/migration/GreedyMigrationPolicy.java b/core/src/main/java/cn/edu/tsinghua/iginx/migration/GreedyMigrationPolicy.java index 3e02552b12..1135e4e028 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/migration/GreedyMigrationPolicy.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/migration/GreedyMigrationPolicy.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.migration; import cn.edu.tsinghua.iginx.metadata.entity.FragmentMeta; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/migration/MigrationManager.java b/core/src/main/java/cn/edu/tsinghua/iginx/migration/MigrationManager.java index d7300a0188..2e401a8f2f 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/migration/MigrationManager.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/migration/MigrationManager.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.migration; import cn.edu.tsinghua.iginx.conf.ConfigDescriptor; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/migration/MigrationPhysicalExecutor.java b/core/src/main/java/cn/edu/tsinghua/iginx/migration/MigrationPhysicalExecutor.java index f9b7f141f4..0ef8412bcc 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/migration/MigrationPhysicalExecutor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/migration/MigrationPhysicalExecutor.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.migration; import cn.edu.tsinghua.iginx.conf.ConfigDescriptor; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/migration/MigrationPolicy.java b/core/src/main/java/cn/edu/tsinghua/iginx/migration/MigrationPolicy.java index ed17816ccb..97f7dd20ff 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/migration/MigrationPolicy.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/migration/MigrationPolicy.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.migration; import cn.edu.tsinghua.iginx.conf.Config; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/migration/MigrationTask.java b/core/src/main/java/cn/edu/tsinghua/iginx/migration/MigrationTask.java index 198ab8b975..514e69f808 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/migration/MigrationTask.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/migration/MigrationTask.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.migration; import cn.edu.tsinghua.iginx.metadata.entity.FragmentMeta; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/migration/MigrationType.java b/core/src/main/java/cn/edu/tsinghua/iginx/migration/MigrationType.java index ddaa2870b4..33be62cd1a 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/migration/MigrationType.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/migration/MigrationType.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.migration; public enum MigrationType { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/migration/MigrationUtils.java b/core/src/main/java/cn/edu/tsinghua/iginx/migration/MigrationUtils.java index ba1a2d7144..387b04d29b 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/migration/MigrationUtils.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/migration/MigrationUtils.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.migration; import java.util.Collection; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/migration/SimulationBasedMigrationPolicy.java b/core/src/main/java/cn/edu/tsinghua/iginx/migration/SimulationBasedMigrationPolicy.java index 72c7c439f9..e5547f743e 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/migration/SimulationBasedMigrationPolicy.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/migration/SimulationBasedMigrationPolicy.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.migration; import cn.edu.tsinghua.iginx.metadata.entity.FragmentMeta; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/migration/recover/MigrationExecuteTask.java b/core/src/main/java/cn/edu/tsinghua/iginx/migration/recover/MigrationExecuteTask.java index fa68c44b18..6319c7a161 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/migration/recover/MigrationExecuteTask.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/migration/recover/MigrationExecuteTask.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.migration.recover; import cn.edu.tsinghua.iginx.metadata.entity.FragmentMeta; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/migration/recover/MigrationExecuteType.java b/core/src/main/java/cn/edu/tsinghua/iginx/migration/recover/MigrationExecuteType.java index 760b27232e..be1b2ea346 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/migration/recover/MigrationExecuteType.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/migration/recover/MigrationExecuteType.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.migration.recover; public enum MigrationExecuteType { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/migration/recover/MigrationLogger.java b/core/src/main/java/cn/edu/tsinghua/iginx/migration/recover/MigrationLogger.java index be1a9c4136..b016a6c299 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/migration/recover/MigrationLogger.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/migration/recover/MigrationLogger.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.migration.recover; import cn.edu.tsinghua.iginx.migration.MigrationTask; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/migration/recover/MigrationLoggerAnalyzer.java b/core/src/main/java/cn/edu/tsinghua/iginx/migration/recover/MigrationLoggerAnalyzer.java index 96c928f11a..a1cea1e480 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/migration/recover/MigrationLoggerAnalyzer.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/migration/recover/MigrationLoggerAnalyzer.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.migration.recover; import static cn.edu.tsinghua.iginx.migration.recover.MigrationLogger.MIGRATION_EXECUTE_TASK_END; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/monitor/HotSpotMonitor.java b/core/src/main/java/cn/edu/tsinghua/iginx/monitor/HotSpotMonitor.java index 586d6ca945..08e02bcbd5 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/monitor/HotSpotMonitor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/monitor/HotSpotMonitor.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.monitor; import cn.edu.tsinghua.iginx.conf.ConfigDescriptor; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/monitor/IMonitor.java b/core/src/main/java/cn/edu/tsinghua/iginx/monitor/IMonitor.java index 2b0ca31c26..8b5753fd5f 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/monitor/IMonitor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/monitor/IMonitor.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.monitor; public interface IMonitor { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/monitor/MonitorManager.java b/core/src/main/java/cn/edu/tsinghua/iginx/monitor/MonitorManager.java index 72307fea18..8a18f413a1 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/monitor/MonitorManager.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/monitor/MonitorManager.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.monitor; import cn.edu.tsinghua.iginx.compaction.CompactionManager; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/monitor/RequestsMonitor.java b/core/src/main/java/cn/edu/tsinghua/iginx/monitor/RequestsMonitor.java index 5e3eb899cf..0d7b77feff 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/monitor/RequestsMonitor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/monitor/RequestsMonitor.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.monitor; import cn.edu.tsinghua.iginx.conf.ConfigDescriptor; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/mqtt/BrokerAuthenticator.java b/core/src/main/java/cn/edu/tsinghua/iginx/mqtt/BrokerAuthenticator.java index 3a6c74af04..c1d4e2f1c4 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/mqtt/BrokerAuthenticator.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/mqtt/BrokerAuthenticator.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.mqtt; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/mqtt/IPayloadFormatter.java b/core/src/main/java/cn/edu/tsinghua/iginx/mqtt/IPayloadFormatter.java index 71bd9e2b46..390b10a014 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/mqtt/IPayloadFormatter.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/mqtt/IPayloadFormatter.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.mqtt; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/mqtt/JsonPayloadFormatter.java b/core/src/main/java/cn/edu/tsinghua/iginx/mqtt/JsonPayloadFormatter.java index 6b5e991742..8d75e19f2b 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/mqtt/JsonPayloadFormatter.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/mqtt/JsonPayloadFormatter.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.mqtt; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/mqtt/MQTTService.java b/core/src/main/java/cn/edu/tsinghua/iginx/mqtt/MQTTService.java index 9af969cf02..0bac762728 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/mqtt/MQTTService.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/mqtt/MQTTService.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.mqtt; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/mqtt/Message.java b/core/src/main/java/cn/edu/tsinghua/iginx/mqtt/Message.java index 3a93fa7fba..4fd3042e34 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/mqtt/Message.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/mqtt/Message.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.mqtt; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/mqtt/PayloadFormatManager.java b/core/src/main/java/cn/edu/tsinghua/iginx/mqtt/PayloadFormatManager.java index d6c29baa59..f054fde751 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/mqtt/PayloadFormatManager.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/mqtt/PayloadFormatManager.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.mqtt; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/mqtt/PublishHandler.java b/core/src/main/java/cn/edu/tsinghua/iginx/mqtt/PublishHandler.java index 6e7073c33f..4366ef3460 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/mqtt/PublishHandler.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/mqtt/PublishHandler.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.mqtt; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/notice/EmailNotifier.java b/core/src/main/java/cn/edu/tsinghua/iginx/notice/EmailNotifier.java index 72bfa22149..8bc6ade0c2 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/notice/EmailNotifier.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/notice/EmailNotifier.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.notice; import cn.edu.tsinghua.iginx.conf.Config; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/policy/AbstractPolicy.java b/core/src/main/java/cn/edu/tsinghua/iginx/policy/AbstractPolicy.java index a9643f1f04..1b426a0922 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/policy/AbstractPolicy.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/policy/AbstractPolicy.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.policy; import cn.edu.tsinghua.iginx.metadata.IMetaManager; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/policy/IPolicy.java b/core/src/main/java/cn/edu/tsinghua/iginx/policy/IPolicy.java index 7df5602b8c..d693ef58ef 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/policy/IPolicy.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/policy/IPolicy.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.policy; import cn.edu.tsinghua.iginx.metadata.IMetaManager; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/policy/PolicyManager.java b/core/src/main/java/cn/edu/tsinghua/iginx/policy/PolicyManager.java index e502257687..d86372fceb 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/policy/PolicyManager.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/policy/PolicyManager.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.policy; import cn.edu.tsinghua.iginx.metadata.DefaultMetaManager; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/policy/Utils.java b/core/src/main/java/cn/edu/tsinghua/iginx/policy/Utils.java index 185b2b8944..53f3033b54 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/policy/Utils.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/policy/Utils.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.policy; import cn.edu.tsinghua.iginx.conf.Constants; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/policy/historical/HistoricalPolicy.java b/core/src/main/java/cn/edu/tsinghua/iginx/policy/historical/HistoricalPolicy.java index bdebb5e47b..10e698483d 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/policy/historical/HistoricalPolicy.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/policy/historical/HistoricalPolicy.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.policy.historical; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/policy/naive/NaivePolicy.java b/core/src/main/java/cn/edu/tsinghua/iginx/policy/naive/NaivePolicy.java index 471b9e0e67..3372fa62bc 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/policy/naive/NaivePolicy.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/policy/naive/NaivePolicy.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.policy.naive; import cn.edu.tsinghua.iginx.conf.ConfigDescriptor; @@ -158,7 +176,7 @@ public Pair, List> generateInitialFragmentsA private Pair>, List> generateInitialFragmentsAndStorageUnitsByClients( List paths, KeyInterval keyInterval) { - Map> fragmentMap = new HashMap<>(); + Map> fragmentMap = new TreeMap<>(); List storageUnitList = new ArrayList<>(); List storageEngineList = iMetaManager.getWritableStorageEngineList(); @@ -166,13 +184,13 @@ public Pair, List> generateInitialFragmentsA String[] clients = ConfigDescriptor.getInstance().getConfig().getClients().split(","); int instancesNumPerClient = - ConfigDescriptor.getInstance().getConfig().getInstancesNumPerClient() - 1; - int replicaNum = + ConfigDescriptor.getInstance().getConfig().getInstancesNumPerClient(); + int totalReplicaNum = Math.min(1 + ConfigDescriptor.getInstance().getConfig().getReplicaNum(), storageEngineNum); String[] prefixes = new String[clients.length * instancesNumPerClient]; for (int i = 0; i < clients.length; i++) { for (int j = 0; j < instancesNumPerClient; j++) { - prefixes[i * instancesNumPerClient + j] = clients[i] + (j + 2); + prefixes[i * instancesNumPerClient + j] = clients[i] + (j + 1); } } Arrays.sort(prefixes); @@ -180,23 +198,55 @@ public Pair, List> generateInitialFragmentsA List fragmentMetaList; String masterId; StorageUnitMeta storageUnit; - for (int i = 0; i < clients.length * instancesNumPerClient - 1; i++) { + + // (null, prefixes[1]) + // 包括prefixes[0] + fragmentMetaList = new ArrayList<>(); + masterId = RandomStringUtils.randomAlphanumeric(16); + storageUnit = new StorageUnitMeta(masterId, storageEngineList.get(0).getId(), masterId, true); + for (int i = 1; i < totalReplicaNum; i++) { + storageUnit.addReplica( + new StorageUnitMeta( + RandomStringUtils.randomAlphanumeric(16), + storageEngineList.get(i).getId(), + masterId, + false)); + } + storageUnitList.add(storageUnit); + fragmentMetaList.add(new FragmentMeta(null, prefixes[1], 0, Long.MAX_VALUE, masterId)); + fragmentMap.put(new ColumnsInterval(null, prefixes[1]), fragmentMetaList); + + for (int i = 1; i < clients.length * instancesNumPerClient - 1; i++) { + int masterIndex = i / instancesNumPerClient; fragmentMetaList = new ArrayList<>(); masterId = RandomStringUtils.randomAlphanumeric(16); + // TODO 全链路同机 storageUnit = - new StorageUnitMeta( - masterId, storageEngineList.get(i % storageEngineNum).getId(), masterId, true); + new StorageUnitMeta(masterId, storageEngineList.get(masterIndex).getId(), masterId, true); + // TODO Round Robin + // storageUnit = + // new StorageUnitMeta( + // masterId, storageEngineList.get(i % storageEngineNum).getId(), masterId, + // true); // storageUnit = new StorageUnitMeta(masterId, getStorageEngineList().get(i * // 2 % getStorageEngineList().size()).getId(), masterId, true); - for (int j = i + 1; j < i + replicaNum; j++) { + for (int j = 1; j < totalReplicaNum; j++) { + int replicaIndex = (masterIndex + j) % storageEngineNum; + // TODO 全链路同机 storageUnit.addReplica( new StorageUnitMeta( RandomStringUtils.randomAlphanumeric(16), - storageEngineList.get(j % storageEngineNum).getId(), + storageEngineList.get(replicaIndex).getId(), masterId, false)); - // storageUnit.addReplica(new - // StorageUnitMeta(RandomStringUtils.randomAlphanumeric(16), + // TODO Round Robin + // storageUnit.addReplica( + // new StorageUnitMeta( + // RandomStringUtils.randomAlphanumeric(16), + // storageEngineList.get(j % storageEngineNum).getId(), + // masterId, + // false)); + // storageUnit.addReplica(new StorageUnitMeta(RandomStringUtils.randomAlphanumeric(16), // getStorageEngineList().get((i * 2 + 1) % getStorageEngineList().size()).getId(), // masterId, false)); } @@ -206,31 +256,17 @@ public Pair, List> generateInitialFragmentsA fragmentMap.put(new ColumnsInterval(prefixes[i], prefixes[i + 1]), fragmentMetaList); } - fragmentMetaList = new ArrayList<>(); - masterId = RandomStringUtils.randomAlphanumeric(16); - storageUnit = new StorageUnitMeta(masterId, storageEngineList.get(0).getId(), masterId, true); - for (int i = 1; i < replicaNum; i++) { - storageUnit.addReplica( - new StorageUnitMeta( - RandomStringUtils.randomAlphanumeric(16), - storageEngineList.get(i).getId(), - masterId, - false)); - } - storageUnitList.add(storageUnit); - fragmentMetaList.add(new FragmentMeta(null, prefixes[0], 0, Long.MAX_VALUE, masterId)); - fragmentMap.put(new ColumnsInterval(null, prefixes[0]), fragmentMetaList); - + // [prefixes[clients.length * instancesNumPerClient - 1], null) fragmentMetaList = new ArrayList<>(); masterId = RandomStringUtils.randomAlphanumeric(16); storageUnit = new StorageUnitMeta( masterId, storageEngineList.get(storageEngineNum - 1).getId(), masterId, true); - for (int i = 1; i < replicaNum; i++) { + for (int i = 1; i < totalReplicaNum; i++) { storageUnit.addReplica( new StorageUnitMeta( RandomStringUtils.randomAlphanumeric(16), - storageEngineList.get(storageEngineNum - 1 - i).getId(), + storageEngineList.get((storageEngineNum - 1 + i) % storageEngineNum).getId(), masterId, false)); } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/policy/naive/Sampler.java b/core/src/main/java/cn/edu/tsinghua/iginx/policy/naive/Sampler.java index 711d8cec51..431d9fce16 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/policy/naive/Sampler.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/policy/naive/Sampler.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.policy.naive; import java.util.*; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/policy/simple/ColumnCalDO.java b/core/src/main/java/cn/edu/tsinghua/iginx/policy/simple/ColumnCalDO.java index 10cb853cb2..2afb07a2fa 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/policy/simple/ColumnCalDO.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/policy/simple/ColumnCalDO.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.policy.simple; import lombok.Data; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/policy/simple/FragmentCreator.java b/core/src/main/java/cn/edu/tsinghua/iginx/policy/simple/FragmentCreator.java index b894a25fc5..0525865bfe 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/policy/simple/FragmentCreator.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/policy/simple/FragmentCreator.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.policy.simple; import cn.edu.tsinghua.iginx.conf.Config; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/policy/simple/SimplePolicy.java b/core/src/main/java/cn/edu/tsinghua/iginx/policy/simple/SimplePolicy.java index 4661c366ef..324c5bfd01 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/policy/simple/SimplePolicy.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/policy/simple/SimplePolicy.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.policy.simple; import cn.edu.tsinghua.iginx.conf.Config; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/policy/test/KeyRangeTestPolicy.java b/core/src/main/java/cn/edu/tsinghua/iginx/policy/test/KeyRangeTestPolicy.java index 71e211f8f8..482423aaa4 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/policy/test/KeyRangeTestPolicy.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/policy/test/KeyRangeTestPolicy.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + /** 该Policy是为了测试而写的,不要在生产环境中使用 该Policy会生成KeyRange不同的Fragment,用于测试。 */ package cn.edu.tsinghua.iginx.policy.test; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/resource/QueryResourceManager.java b/core/src/main/java/cn/edu/tsinghua/iginx/resource/QueryResourceManager.java index cf415bb1e5..e7b3d635ca 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/resource/QueryResourceManager.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/resource/QueryResourceManager.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.resource; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/resource/ResourceManager.java b/core/src/main/java/cn/edu/tsinghua/iginx/resource/ResourceManager.java index 0e03d54114..faf2befe35 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/resource/ResourceManager.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/resource/ResourceManager.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.resource; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/resource/system/DefaultSystemMetricsService.java b/core/src/main/java/cn/edu/tsinghua/iginx/resource/system/DefaultSystemMetricsService.java index 2a48dfe77e..b9b4dcaf18 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/resource/system/DefaultSystemMetricsService.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/resource/system/DefaultSystemMetricsService.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.resource.system; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/resource/system/SystemMetricsService.java b/core/src/main/java/cn/edu/tsinghua/iginx/resource/system/SystemMetricsService.java index 9def741047..3a44a03d03 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/resource/system/SystemMetricsService.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/resource/system/SystemMetricsService.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.resource.system; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/MetricsResource.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/MetricsResource.java index 5def01525b..6d8c5bb66b 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/MetricsResource.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/MetricsResource.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.rest; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/RestServer.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/RestServer.java index df7e7b40ad..cf0f43ceff 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/RestServer.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/RestServer.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.rest; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/RestSession.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/RestSession.java index 65ae5d9fb0..b5fe2068fe 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/RestSession.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/RestSession.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.rest; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/RestUtils.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/RestUtils.java index d820d59a16..6f50593233 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/RestUtils.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/RestUtils.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.rest; import cn.edu.tsinghua.iginx.session.SessionQueryDataSet; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/bean/Annotation.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/bean/Annotation.java index 8e2e870380..e75f9a654b 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/bean/Annotation.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/bean/Annotation.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.rest.bean; import com.fasterxml.jackson.databind.JsonNode; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/bean/AnnotationLimit.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/bean/AnnotationLimit.java index 8be17ac846..609157764b 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/bean/AnnotationLimit.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/bean/AnnotationLimit.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.rest.bean; import java.util.*; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/bean/Metric.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/bean/Metric.java index b1aaeee04d..b13c927186 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/bean/Metric.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/bean/Metric.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.rest.bean; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/bean/Query.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/bean/Query.java index 123f5327e1..0af8e5fcb1 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/bean/Query.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/bean/Query.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.rest.bean; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/bean/QueryMetric.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/bean/QueryMetric.java index 194b797783..07a3b5dfbe 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/bean/QueryMetric.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/bean/QueryMetric.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.rest.bean; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/bean/QueryResult.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/bean/QueryResult.java index f750f151e0..f05ce37fb3 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/bean/QueryResult.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/bean/QueryResult.java @@ -1,23 +1,23 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.rest.bean; +import static cn.edu.tsinghua.iginx.engine.shared.Constants.*; import static cn.edu.tsinghua.iginx.rest.RestUtils.TOP_KEY; import cn.edu.tsinghua.iginx.metadata.DefaultMetaManager; @@ -146,7 +146,15 @@ private String nameToString(int pos) { if (queryAggregators.get(pos).getType() == QueryAggregatorType.SAVE_AS) { return String.format("\"name\": \"%s\"", queryAggregators.get(pos).getMetric_name()); } else { - return String.format("\"name\": \"%s\"", queryMetrics.get(pos).getName()); + // downsample query, window_start & window_end columns are attached + if (queryMetrics.get(pos).getAggregators() != null + && queryMetrics.get(pos).getAggregators().size() != 0) { + return String.format( + "\"names\": [\"%s\", \"%s\", \"%s\"]", + WINDOW_START_COL, WINDOW_END_COL, queryMetrics.get(pos).getName()); + } else { + return String.format("\"name\": \"%s\"", queryMetrics.get(pos).getName()); + } } } @@ -232,6 +240,7 @@ private String sampleSizeToString(int pos) { private Map> getTagsFromPaths(List paths) { Map> ret = new TreeMap<>(); for (String path : paths) { + if (RESERVED_COLS.contains(path)) continue; int firstBrace = path.indexOf("{"); int lastBrace = path.indexOf("}"); if (firstBrace == -1 || lastBrace == -1) { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/bean/QueryResultDataset.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/bean/QueryResultDataset.java index bc222318b4..60d93181fc 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/bean/QueryResultDataset.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/bean/QueryResultDataset.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.rest.bean; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/insert/DataPointsParser.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/insert/DataPointsParser.java index df9ecc6310..44a5e4146f 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/insert/DataPointsParser.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/insert/DataPointsParser.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.rest.insert; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/insert/InsertWorker.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/insert/InsertWorker.java index 05a871932c..2beaa32fbc 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/insert/InsertWorker.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/insert/InsertWorker.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.rest.insert; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryExecutor.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryExecutor.java index 37a89456c7..81559fb414 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryExecutor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryExecutor.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.rest.query; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryParser.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryParser.java index 7926134a61..459b88ee69 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryParser.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/QueryParser.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.rest.query; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/Filter.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/Filter.java index 4f5da3f17d..b351d78ad5 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/Filter.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/Filter.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.rest.query.aggregator; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregator.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregator.java index 16a47ad6a7..98cf57913e 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregator.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregator.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.rest.query.aggregator; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorAvg.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorAvg.java index 11105b3096..b102b17241 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorAvg.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorAvg.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.rest.query.aggregator; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorCount.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorCount.java index e949c4f047..0b6b6a4399 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorCount.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorCount.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.rest.query.aggregator; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorDev.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorDev.java index 029cb5c61e..3c77712706 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorDev.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorDev.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.rest.query.aggregator; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorDiff.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorDiff.java index 4406bc193b..ee72c3ca5c 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorDiff.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorDiff.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.rest.query.aggregator; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorDiv.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorDiv.java index b9b6b83da4..574d7654cd 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorDiv.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorDiv.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.rest.query.aggregator; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorFilter.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorFilter.java index 462c7b6888..ad2225dc83 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorFilter.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorFilter.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.rest.query.aggregator; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorFirst.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorFirst.java index 48b73c40da..a09b6d2fc6 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorFirst.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorFirst.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.rest.query.aggregator; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorLast.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorLast.java index 1526a0babf..4cc112a0bb 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorLast.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorLast.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.rest.query.aggregator; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorMax.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorMax.java index 50c065ef99..e122ac759f 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorMax.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorMax.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.rest.query.aggregator; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorMin.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorMin.java index e919b3c66b..8ab41b0817 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorMin.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorMin.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.rest.query.aggregator; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorNone.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorNone.java index 171a358bae..51961d0a48 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorNone.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorNone.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.rest.query.aggregator; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorPercentile.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorPercentile.java index 9684a198a2..4daf1cd00f 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorPercentile.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorPercentile.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.rest.query.aggregator; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorRate.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorRate.java index 33c2f068e9..26385e635b 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorRate.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorRate.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.rest.query.aggregator; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorSampler.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorSampler.java index a5c2d0f9cd..111a60d9db 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorSampler.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorSampler.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.rest.query.aggregator; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorSaveAs.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorSaveAs.java index 74c5924790..e21df27f77 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorSaveAs.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorSaveAs.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.rest.query.aggregator; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorSum.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorSum.java index 78c34532da..c82489b3cd 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorSum.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorSum.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.rest.query.aggregator; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorType.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorType.java index 82da56ddad..e756c418d2 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorType.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryAggregatorType.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.rest.query.aggregator; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryShowColumns.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryShowColumns.java index 2272bf3f54..0ab1083800 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryShowColumns.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/query/aggregator/QueryShowColumns.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.rest.query.aggregator; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/IginXSqlVisitor.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/IginXSqlVisitor.java index dee8b0c6e1..80fb8b8d4b 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/IginXSqlVisitor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/IginXSqlVisitor.java @@ -1,5 +1,25 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql; +import static cn.edu.tsinghua.iginx.constant.GlobalConstant.KEY_MAX_VAL; +import static cn.edu.tsinghua.iginx.constant.GlobalConstant.KEY_MIN_VAL; import static cn.edu.tsinghua.iginx.engine.shared.function.FunctionUtils.isCanUseSetQuantifierFunction; import static cn.edu.tsinghua.iginx.engine.shared.operator.MarkJoin.MARK_PREFIX; import static cn.edu.tsinghua.iginx.sql.statement.select.SelectStatement.markJoinCount; @@ -527,6 +547,13 @@ public Statement visitAddStorageEngineStatement(AddStorageEngineStatementContext return addStorageEngineStatement; } + @Override + public Statement visitAlterEngineStatement(SqlParser.AlterEngineStatementContext ctx) { + long engineId = Long.parseLong(ctx.engineId.getText()); + Map newParams = parseExtra(ctx.params); + return new AlterEngineStatement(engineId, newParams); + } + @Override public Statement visitShowColumnsStatement(ShowColumnsStatementContext ctx) { return parseShowColumnsOptions(ctx.showColumnsOptions()); @@ -1068,27 +1095,31 @@ private void parseSpecialClause(SpecialClauseContext ctx, UnarySelectStatement s private void parseDownsampleClause( DownsampleClauseContext ctx, UnarySelectStatement selectStatement) { long precision = parseAggLen(ctx.aggLen(0)); - Pair timeInterval = parseTimeInterval(ctx.timeInterval()); - selectStatement.setStartKey(timeInterval.k); - selectStatement.setEndKey(timeInterval.v); + Pair timeInterval = + ctx.timeInterval() == null ? null : parseTimeInterval(ctx.timeInterval()); + long startKey = timeInterval == null ? KEY_MIN_VAL : timeInterval.k; + long endKey = timeInterval == null ? KEY_MAX_VAL : timeInterval.v; + selectStatement.setStartKey(startKey); + selectStatement.setEndKey(endKey); selectStatement.setPrecision(precision); selectStatement.setSlideDistance(precision); selectStatement.setHasDownsample(true); - if (ctx.STEP() != null) { + if (ctx.SLIDE() != null) { long distance = parseAggLen(ctx.aggLen(1)); selectStatement.setSlideDistance(distance); } // merge value filter and group time range filter - KeyFilter startKey = new KeyFilter(Op.GE, timeInterval.k); - KeyFilter endKey = new KeyFilter(Op.L, timeInterval.v); + KeyFilter startKeyFilter = new KeyFilter(Op.GE, startKey); + KeyFilter endKeyFilter = new KeyFilter(Op.L, endKey); Filter mergedFilter; if (selectStatement.hasValueFilter()) { mergedFilter = new AndFilter( - new ArrayList<>(Arrays.asList(selectStatement.getFilter(), startKey, endKey))); + new ArrayList<>( + Arrays.asList(selectStatement.getFilter(), startKeyFilter, endKeyFilter))); } else { - mergedFilter = new AndFilter(new ArrayList<>(Arrays.asList(startKey, endKey))); + mergedFilter = new AndFilter(new ArrayList<>(Arrays.asList(startKeyFilter, endKeyFilter))); selectStatement.setHasValueFilter(true); } selectStatement.setFilter(mergedFilter); diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/SQLConstant.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/SQLConstant.java index cfbdd1e72f..7c0093d987 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/SQLConstant.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/SQLConstant.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql; public class SQLConstant { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/SQLParseError.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/SQLParseError.java index 0fc2b3b2b7..3d936031ad 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/SQLParseError.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/SQLParseError.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql; import org.antlr.v4.runtime.BaseErrorListener; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/exception/SQLParserException.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/exception/SQLParserException.java index 3c986d351d..d86e7f1b41 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/exception/SQLParserException.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/exception/SQLParserException.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.exception; public class SQLParserException extends RuntimeException { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/AddStorageEngineStatement.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/AddStorageEngineStatement.java index 8a12eb5946..e3e433feac 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/AddStorageEngineStatement.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/AddStorageEngineStatement.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement; import cn.edu.tsinghua.iginx.IginxWorker; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/AlterEngineStatement.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/AlterEngineStatement.java new file mode 100644 index 0000000000..a8af8b1270 --- /dev/null +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/AlterEngineStatement.java @@ -0,0 +1,50 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cn.edu.tsinghua.iginx.sql.statement; + +import cn.edu.tsinghua.iginx.IginxWorker; +import cn.edu.tsinghua.iginx.engine.shared.RequestContext; +import cn.edu.tsinghua.iginx.engine.shared.Result; +import cn.edu.tsinghua.iginx.engine.shared.exception.StatementExecutionException; +import cn.edu.tsinghua.iginx.thrift.AlterStorageEngineReq; +import cn.edu.tsinghua.iginx.thrift.Status; +import java.util.Map; + +public class AlterEngineStatement extends SystemStatement { + + private final long engineId; + + private final IginxWorker worker = IginxWorker.getInstance(); + + private final Map newParams; + + public AlterEngineStatement(long engineId, Map newParams) { + this.statementType = StatementType.ALTER_STORAGE_ENGINE; + this.engineId = engineId; + this.newParams = newParams; + } + + @Override + public void execute(RequestContext ctx) throws StatementExecutionException { + AlterStorageEngineReq req = new AlterStorageEngineReq(ctx.getSessionId(), engineId, newParams); + Status status = worker.alterStorageEngine(req); + + Result result = new Result(status); + ctx.setResult(result); + } +} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/CancelJobStatement.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/CancelJobStatement.java index 8cb032ed6d..51b22e74d6 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/CancelJobStatement.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/CancelJobStatement.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement; import cn.edu.tsinghua.iginx.IginxWorker; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ClearDataStatement.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ClearDataStatement.java index e941b4f09c..7d74986336 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ClearDataStatement.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ClearDataStatement.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement; public class ClearDataStatement extends DataStatement { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/CommitTransformJobStatement.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/CommitTransformJobStatement.java index 7b0d89f7ca..216c894581 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/CommitTransformJobStatement.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/CommitTransformJobStatement.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement; import cn.edu.tsinghua.iginx.engine.shared.RequestContext; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/CompactStatement.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/CompactStatement.java index 754367b47a..31b5ff9671 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/CompactStatement.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/CompactStatement.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement; import cn.edu.tsinghua.iginx.compaction.CompactionManager; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/CountPointsStatement.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/CountPointsStatement.java index 5ad50b1ba9..b74a2b0147 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/CountPointsStatement.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/CountPointsStatement.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement; public class CountPointsStatement extends DataStatement { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/DataStatement.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/DataStatement.java index cf84f7a607..ff16efcc46 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/DataStatement.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/DataStatement.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement; public abstract class DataStatement extends Statement {} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/DeleteColumnsStatement.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/DeleteColumnsStatement.java index 07d76b6bc9..5b6c9f26d5 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/DeleteColumnsStatement.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/DeleteColumnsStatement.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement; import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/DeleteStatement.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/DeleteStatement.java index 120e50b8a3..f12db217a4 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/DeleteStatement.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/DeleteStatement.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement; import cn.edu.tsinghua.iginx.engine.logical.utils.LogicalFilterUtils; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/DropTaskStatement.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/DropTaskStatement.java index 7f45240994..5dbcae57ed 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/DropTaskStatement.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/DropTaskStatement.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement; import cn.edu.tsinghua.iginx.IginxWorker; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ExportFileFromSelectStatement.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ExportFileFromSelectStatement.java index f88559d8b8..96323a8e68 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ExportFileFromSelectStatement.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ExportFileFromSelectStatement.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement; import cn.edu.tsinghua.iginx.engine.shared.file.FileType; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/InsertFromCsvStatement.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/InsertFromCsvStatement.java index 6fb684f414..d7b65e444d 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/InsertFromCsvStatement.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/InsertFromCsvStatement.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement; import cn.edu.tsinghua.iginx.engine.shared.file.read.ImportFile; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/InsertFromSelectStatement.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/InsertFromSelectStatement.java index 475bb6cf3f..4d8d80f445 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/InsertFromSelectStatement.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/InsertFromSelectStatement.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement; import cn.edu.tsinghua.iginx.sql.statement.select.SelectStatement; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/InsertStatement.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/InsertStatement.java index 8ed4d95987..7728b8e4fa 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/InsertStatement.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/InsertStatement.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement; import cn.edu.tsinghua.iginx.engine.shared.data.write.RawData; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/RegisterTaskStatement.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/RegisterTaskStatement.java index f8461b9145..69d7d43a04 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/RegisterTaskStatement.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/RegisterTaskStatement.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement; import cn.edu.tsinghua.iginx.IginxWorker; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/RemoveHistoryDataSourceStatement.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/RemoveHistoryDataSourceStatement.java index 9306783112..cc962ea43c 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/RemoveHistoryDataSourceStatement.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/RemoveHistoryDataSourceStatement.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement; import cn.edu.tsinghua.iginx.IginxWorker; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/SetConfigStatement.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/SetConfigStatement.java index bd8d110911..4c7a474985 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/SetConfigStatement.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/SetConfigStatement.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement; import cn.edu.tsinghua.iginx.conf.Config; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/SetRulesStatement.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/SetRulesStatement.java index db13fb3cca..f7a9235e60 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/SetRulesStatement.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/SetRulesStatement.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement; import cn.edu.tsinghua.iginx.IginxWorker; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ShowClusterInfoStatement.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ShowClusterInfoStatement.java index df902eda64..fa5dd1ce85 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ShowClusterInfoStatement.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ShowClusterInfoStatement.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement; import cn.edu.tsinghua.iginx.IginxWorker; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ShowColumnsStatement.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ShowColumnsStatement.java index ebc69d3b20..19eea0ebc9 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ShowColumnsStatement.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ShowColumnsStatement.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement; import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ShowConfigStatement.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ShowConfigStatement.java index a5221df0eb..b69ef3da0b 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ShowConfigStatement.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ShowConfigStatement.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement; import cn.edu.tsinghua.iginx.conf.Config; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ShowEligibleJobStatement.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ShowEligibleJobStatement.java index 2d38f30995..d877c44c9f 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ShowEligibleJobStatement.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ShowEligibleJobStatement.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement; import cn.edu.tsinghua.iginx.IginxWorker; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ShowJobStatusStatement.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ShowJobStatusStatement.java index c0e5b0280d..06348630bc 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ShowJobStatusStatement.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ShowJobStatusStatement.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement; import cn.edu.tsinghua.iginx.IginxWorker; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ShowRegisterTaskStatement.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ShowRegisterTaskStatement.java index d8932c0615..e02e2e75ac 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ShowRegisterTaskStatement.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ShowRegisterTaskStatement.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement; import cn.edu.tsinghua.iginx.IginxWorker; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ShowReplicationStatement.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ShowReplicationStatement.java index a28d3074cd..a9ccf435d0 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ShowReplicationStatement.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ShowReplicationStatement.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement; import cn.edu.tsinghua.iginx.conf.ConfigDescriptor; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ShowRulesStatement.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ShowRulesStatement.java index 75e59a5bfb..9198ddc19c 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ShowRulesStatement.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ShowRulesStatement.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement; import cn.edu.tsinghua.iginx.IginxWorker; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ShowSessionIDStatement.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ShowSessionIDStatement.java index 25a50e5211..a3c0647afa 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ShowSessionIDStatement.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/ShowSessionIDStatement.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement; import cn.edu.tsinghua.iginx.IginxWorker; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/Statement.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/Statement.java index d5ff6af936..ee0a81c76f 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/Statement.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/Statement.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement; public abstract class Statement { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/StatementType.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/StatementType.java index 8f1d5ef319..236820ea2a 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/StatementType.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/StatementType.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement; public enum StatementType { @@ -10,6 +28,7 @@ public enum StatementType { EXPORT_CSV_FROM_SELECT, EXPORT_STREAM_FROM_SELECT, ADD_STORAGE_ENGINE, + ALTER_STORAGE_ENGINE, SHOW_REPLICATION, COUNT_POINTS, CLEAR_DATA, diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/SystemStatement.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/SystemStatement.java index df29cc0a6d..a95d887adf 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/SystemStatement.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/SystemStatement.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement; import cn.edu.tsinghua.iginx.engine.shared.RequestContext; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/frompart/CteFromPart.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/frompart/CteFromPart.java index 1f09dba85e..75840a9e5a 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/frompart/CteFromPart.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/frompart/CteFromPart.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement.frompart; import static cn.edu.tsinghua.iginx.engine.shared.Constants.ALL_PATH_SUFFIX; @@ -6,11 +24,10 @@ import cn.edu.tsinghua.iginx.engine.shared.operator.Operator; import cn.edu.tsinghua.iginx.sql.statement.frompart.join.JoinCondition; import cn.edu.tsinghua.iginx.sql.statement.select.CommonTableExpression; +import cn.edu.tsinghua.iginx.utils.Pair; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.List; -import java.util.Map; public class CteFromPart implements FromPart { @@ -42,10 +59,9 @@ public boolean hasAlias() { } @Override - public Map getAliasMap() { - Map aliasMap = new HashMap<>(); - aliasMap.put(cte.getName() + ALL_PATH_SUFFIX, alias + ALL_PATH_SUFFIX); - return aliasMap; + public List> getAliasList() { + return Collections.singletonList( + new Pair<>(cte.getName() + ALL_PATH_SUFFIX, alias + ALL_PATH_SUFFIX)); } @Override diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/frompart/FromPart.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/frompart/FromPart.java index 2897f1cac6..4cbb2d5bc5 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/frompart/FromPart.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/frompart/FromPart.java @@ -1,14 +1,32 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement.frompart; import cn.edu.tsinghua.iginx.sql.statement.frompart.join.JoinCondition; +import cn.edu.tsinghua.iginx.utils.Pair; import java.util.List; -import java.util.Map; public interface FromPart { FromPartType getType(); - Map getAliasMap(); + List> getAliasList(); boolean hasAlias(); diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/frompart/FromPartType.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/frompart/FromPartType.java index bfe9e57ffb..e74891f72a 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/frompart/FromPartType.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/frompart/FromPartType.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement.frompart; public enum FromPartType { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/frompart/PathFromPart.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/frompart/PathFromPart.java index eee60ac70a..5a808ae6eb 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/frompart/PathFromPart.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/frompart/PathFromPart.java @@ -1,13 +1,30 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement.frompart; import static cn.edu.tsinghua.iginx.engine.shared.Constants.ALL_PATH_SUFFIX; import cn.edu.tsinghua.iginx.engine.shared.Constants; import cn.edu.tsinghua.iginx.sql.statement.frompart.join.JoinCondition; +import cn.edu.tsinghua.iginx.utils.Pair; import java.util.Collections; -import java.util.HashMap; import java.util.List; -import java.util.Map; public class PathFromPart implements FromPart { @@ -35,10 +52,8 @@ public FromPartType getType() { } @Override - public Map getAliasMap() { - Map aliasMap = new HashMap<>(); - aliasMap.put(path + ALL_PATH_SUFFIX, alias + ALL_PATH_SUFFIX); - return aliasMap; + public List> getAliasList() { + return Collections.singletonList(new Pair<>(path + ALL_PATH_SUFFIX, alias + ALL_PATH_SUFFIX)); } @Override diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/frompart/ShowColumnsFromPart.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/frompart/ShowColumnsFromPart.java index 27f07ad587..2a8b66d6a4 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/frompart/ShowColumnsFromPart.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/frompart/ShowColumnsFromPart.java @@ -1,14 +1,31 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement.frompart; import cn.edu.tsinghua.iginx.engine.shared.Constants; import cn.edu.tsinghua.iginx.sql.statement.ShowColumnsStatement; import cn.edu.tsinghua.iginx.sql.statement.frompart.join.JoinCondition; +import cn.edu.tsinghua.iginx.utils.Pair; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.HashMap; import java.util.List; -import java.util.Map; public class ShowColumnsFromPart implements FromPart { @@ -30,11 +47,11 @@ public ShowColumnsStatement getShowColumnsStatement() { } @Override - public Map getAliasMap() { - Map aliasMap = new HashMap<>(); - aliasMap.put("path", alias + ".path"); - aliasMap.put("type", alias + ".type"); - return aliasMap; + public List> getAliasList() { + List> aliasList = new ArrayList<>(2); + aliasList.add(new Pair<>("path", alias + ".path")); + aliasList.add(new Pair<>("type", alias + ".type")); + return aliasList; } @Override diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/frompart/SubQueryFromPart.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/frompart/SubQueryFromPart.java index 866b4b8019..e3e75433d7 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/frompart/SubQueryFromPart.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/frompart/SubQueryFromPart.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement.frompart; import cn.edu.tsinghua.iginx.engine.shared.Constants; @@ -5,8 +23,8 @@ import cn.edu.tsinghua.iginx.sql.statement.select.BinarySelectStatement; import cn.edu.tsinghua.iginx.sql.statement.select.SelectStatement; import cn.edu.tsinghua.iginx.sql.statement.select.UnarySelectStatement; +import cn.edu.tsinghua.iginx.utils.Pair; import java.util.List; -import java.util.Map; public class SubQueryFromPart implements FromPart { @@ -42,8 +60,8 @@ public FromPartType getType() { } @Override - public Map getAliasMap() { - return subQuery.getSubQueryAliasMap(alias); + public List> getAliasList() { + return subQuery.getSubQueryAliasList(alias); } @Override diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/frompart/join/JoinCondition.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/frompart/join/JoinCondition.java index 382c94aba8..5e41d38f14 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/frompart/join/JoinCondition.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/frompart/join/JoinCondition.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement.frompart.join; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/frompart/join/JoinType.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/frompart/join/JoinType.java index 0e3deaefc1..54969f25e6 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/frompart/join/JoinType.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/frompart/join/JoinType.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement.frompart.join; public enum JoinType { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/select/BinarySelectStatement.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/select/BinarySelectStatement.java index a92c46156c..b7dbe5b082 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/select/BinarySelectStatement.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/select/BinarySelectStatement.java @@ -1,11 +1,29 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement.select; import cn.edu.tsinghua.iginx.engine.shared.expr.Expression; import cn.edu.tsinghua.iginx.engine.shared.operator.type.OperatorType; +import cn.edu.tsinghua.iginx.utils.Pair; import java.util.ArrayList; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; public class BinarySelectStatement extends SelectStatement { @@ -75,7 +93,7 @@ public void initFreeVariables() { } @Override - public Map getSubQueryAliasMap(String alias) { - return leftQuery.getSubQueryAliasMap(alias); + public List> getSubQueryAliasList(String alias) { + return leftQuery.getSubQueryAliasList(alias); } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/select/CommonTableExpression.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/select/CommonTableExpression.java index c8b5f486d0..e08925a4d6 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/select/CommonTableExpression.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/select/CommonTableExpression.java @@ -1,12 +1,30 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement.select; import cn.edu.tsinghua.iginx.engine.shared.expr.Expression; import cn.edu.tsinghua.iginx.engine.shared.operator.Operator; import cn.edu.tsinghua.iginx.sql.SQLConstant; +import cn.edu.tsinghua.iginx.utils.Pair; +import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.List; -import java.util.Map; public class CommonTableExpression { @@ -48,18 +66,18 @@ public void setRoot(Operator root) { this.root = root; } - public Map getAliasMap() { + public List> getAliasList() { if (columns.isEmpty()) { - return statement.getSubQueryAliasMap(name); + return statement.getSubQueryAliasList(name); } else { - Map aliasMap = new HashMap<>(); + List> aliasList = new ArrayList<>(columns.size()); for (int i = 0; i < columns.size(); i++) { Expression expression = statement.getExpressions().get(i); String originName = expression.hasAlias() ? expression.getAlias() : expression.getColumnName(); - aliasMap.put(originName, name + SQLConstant.DOT + columns.get(i)); + aliasList.add(new Pair<>(originName, name + SQLConstant.DOT + columns.get(i))); } - return aliasMap; + return aliasList; } } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/select/SelectStatement.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/select/SelectStatement.java index e47eb95730..2c3e4ce3c3 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/select/SelectStatement.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/select/SelectStatement.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement.select; import cn.edu.tsinghua.iginx.engine.shared.expr.Expression; @@ -5,10 +23,10 @@ import cn.edu.tsinghua.iginx.sql.statement.StatementType; import cn.edu.tsinghua.iginx.sql.statement.select.subclause.LimitClause; import cn.edu.tsinghua.iginx.sql.statement.select.subclause.OrderByClause; +import cn.edu.tsinghua.iginx.utils.Pair; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.Set; public abstract class SelectStatement extends DataStatement { @@ -117,7 +135,7 @@ public void addFreeVariable(String freeVariable) { public abstract void initFreeVariables(); - public abstract Map getSubQueryAliasMap(String alias); + public abstract List> getSubQueryAliasList(String alias); public enum SelectStatementType { UNARY, diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/select/UnarySelectStatement.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/select/UnarySelectStatement.java index 16c4280dda..4e2821435f 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/select/UnarySelectStatement.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/select/UnarySelectStatement.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement.select; import static cn.edu.tsinghua.iginx.sql.SQLConstant.DOT; @@ -20,8 +38,18 @@ import cn.edu.tsinghua.iginx.sql.statement.frompart.SubQueryFromPart; import cn.edu.tsinghua.iginx.sql.statement.select.subclause.*; import cn.edu.tsinghua.iginx.thrift.AggregateType; +import cn.edu.tsinghua.iginx.utils.Pair; import cn.edu.tsinghua.iginx.utils.StringUtils; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; public class UnarySelectStatement extends SelectStatement { @@ -63,8 +91,7 @@ public UnarySelectStatement(List paths, long startKey, long endKey) { } // aggregate query - public UnarySelectStatement( - List paths, long startKey, long endKey, AggregateType aggregateType) { + public UnarySelectStatement(List paths, AggregateType aggregateType) { this(false); if (aggregateType == AggregateType.LAST || aggregateType == AggregateType.FIRST) { @@ -82,7 +109,11 @@ public UnarySelectStatement( }); setHasFunc(true); + } + public UnarySelectStatement( + List paths, long startKey, long endKey, AggregateType aggregateType) { + this(paths, aggregateType); this.setFromSession(startKey, endKey); } @@ -404,48 +435,57 @@ public void setQueryType(QueryType queryType) { @Override public List getExpressions() { - List expressions = new ArrayList<>(); - expressions.addAll(selectClause.getExpressions()); - return expressions; + return new ArrayList<>(selectClause.getExpressions()); } public void addSelectClauseExpression(Expression expression) { selectClause.addExpression(expression); } - public Map getSelectAliasMap() { - Map aliasMap = new HashMap<>(); + public List> getSelectAliasList() { + List> aliasList = new ArrayList<>(); + AtomicBoolean hasAlias = new AtomicBoolean(false); getExpressions() .forEach( expression -> { if (expression.hasAlias()) { - aliasMap.put(expression.getColumnName(), expression.getAlias()); + aliasList.add(new Pair<>(expression.getColumnName(), expression.getAlias())); + hasAlias.set(true); + } else { + aliasList.add(new Pair<>(expression.getColumnName(), expression.getColumnName())); } }); - return aliasMap; + return hasAlias.get() ? aliasList : Collections.emptyList(); } @Override - public Map getSubQueryAliasMap(String alias) { - Map aliasMap = new HashMap<>(); + public List> getSubQueryAliasList(String alias) { + List> aliasList = new ArrayList<>(); getExpressions() .forEach( expression -> { if (expression.hasAlias()) { - aliasMap.put(expression.getAlias(), alias + DOT + expression.getAlias()); + aliasList.add( + new Pair<>(expression.getAlias(), alias + DOT + expression.getAlias())); } else { if (expression.getType().equals(Expression.ExpressionType.Binary) || expression.getType().equals(Expression.ExpressionType.Unary)) { - aliasMap.put( - expression.getColumnName(), - alias + DOT + L_PARENTHESES + expression.getColumnName() + R_PARENTHESES); + aliasList.add( + new Pair<>( + expression.getColumnName(), + alias + + DOT + + L_PARENTHESES + + expression.getColumnName() + + R_PARENTHESES)); } else { - aliasMap.put( - expression.getColumnName(), alias + DOT + expression.getColumnName()); + aliasList.add( + new Pair<>( + expression.getColumnName(), alias + DOT + expression.getColumnName())); } } }); - return aliasMap; + return aliasList; } public boolean needRowTransform() { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/select/subclause/FromClause.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/select/subclause/FromClause.java index 87b53cbd69..8b076818f9 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/select/subclause/FromClause.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/select/subclause/FromClause.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement.select.subclause; import cn.edu.tsinghua.iginx.sql.statement.frompart.FromPart; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/select/subclause/GroupByClause.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/select/subclause/GroupByClause.java index ac85f4ebe0..017c1b82de 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/select/subclause/GroupByClause.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/select/subclause/GroupByClause.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement.select.subclause; import cn.edu.tsinghua.iginx.sql.statement.select.UnarySelectStatement.QueryType; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/select/subclause/HavingClause.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/select/subclause/HavingClause.java index fb7bbf6106..4effb9ce21 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/select/subclause/HavingClause.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/select/subclause/HavingClause.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement.select.subclause; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/select/subclause/LimitClause.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/select/subclause/LimitClause.java index 6bd6803d7b..9dd5398185 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/select/subclause/LimitClause.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/select/subclause/LimitClause.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement.select.subclause; public class LimitClause { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/select/subclause/OrderByClause.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/select/subclause/OrderByClause.java index 09ddd44771..4ec70e4605 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/select/subclause/OrderByClause.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/select/subclause/OrderByClause.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement.select.subclause; import java.util.ArrayList; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/select/subclause/SelectClause.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/select/subclause/SelectClause.java index b61e2f309b..bb07872366 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/select/subclause/SelectClause.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/select/subclause/SelectClause.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement.select.subclause; import cn.edu.tsinghua.iginx.engine.shared.expr.Expression; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/select/subclause/WhereClause.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/select/subclause/WhereClause.java index 9b992ad56e..00434d9da6 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/select/subclause/WhereClause.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/select/subclause/WhereClause.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.statement.select.subclause; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/utils/ExpressionUtils.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/utils/ExpressionUtils.java index a242cc2fa4..fd77a34777 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/utils/ExpressionUtils.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/utils/ExpressionUtils.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql.utils; import cn.edu.tsinghua.iginx.engine.shared.expr.BinaryExpression; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/statistics/AbstractStageStatisticsCollector.java b/core/src/main/java/cn/edu/tsinghua/iginx/statistics/AbstractStageStatisticsCollector.java index 098d24197f..024b685226 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/statistics/AbstractStageStatisticsCollector.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/statistics/AbstractStageStatisticsCollector.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.statistics; import cn.edu.tsinghua.iginx.engine.shared.RequestContext; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/statistics/ExecuteStatisticsCollector.java b/core/src/main/java/cn/edu/tsinghua/iginx/statistics/ExecuteStatisticsCollector.java index e8b49fc60f..dd4a880e30 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/statistics/ExecuteStatisticsCollector.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/statistics/ExecuteStatisticsCollector.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.statistics; import cn.edu.tsinghua.iginx.engine.shared.Result; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/statistics/IExecuteStatisticsCollector.java b/core/src/main/java/cn/edu/tsinghua/iginx/statistics/IExecuteStatisticsCollector.java index 1286c089ef..9a6c7883b2 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/statistics/IExecuteStatisticsCollector.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/statistics/IExecuteStatisticsCollector.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.statistics; import cn.edu.tsinghua.iginx.engine.shared.processor.PostExecuteProcessor; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/statistics/ILogicalStatisticsCollector.java b/core/src/main/java/cn/edu/tsinghua/iginx/statistics/ILogicalStatisticsCollector.java index 9c969da0dc..78a2f0b4a5 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/statistics/ILogicalStatisticsCollector.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/statistics/ILogicalStatisticsCollector.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.statistics; import cn.edu.tsinghua.iginx.engine.shared.processor.PostLogicalProcessor; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/statistics/IParseStatisticsCollector.java b/core/src/main/java/cn/edu/tsinghua/iginx/statistics/IParseStatisticsCollector.java index 6ce9bd25ff..ae1dc8b36d 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/statistics/IParseStatisticsCollector.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/statistics/IParseStatisticsCollector.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.statistics; import cn.edu.tsinghua.iginx.engine.shared.processor.PostParseProcessor; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/statistics/IPhysicalStatisticsCollector.java b/core/src/main/java/cn/edu/tsinghua/iginx/statistics/IPhysicalStatisticsCollector.java index 20e2996f9b..685b4eabc3 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/statistics/IPhysicalStatisticsCollector.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/statistics/IPhysicalStatisticsCollector.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.statistics; import cn.edu.tsinghua.iginx.engine.shared.processor.PostPhysicalProcessor; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/statistics/IStatisticsCollector.java b/core/src/main/java/cn/edu/tsinghua/iginx/statistics/IStatisticsCollector.java index f1a98d3295..0c3de7a791 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/statistics/IStatisticsCollector.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/statistics/IStatisticsCollector.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.statistics; public interface IStatisticsCollector diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/statistics/LogicalStatisticsCollector.java b/core/src/main/java/cn/edu/tsinghua/iginx/statistics/LogicalStatisticsCollector.java index 31d782fda2..430f892a87 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/statistics/LogicalStatisticsCollector.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/statistics/LogicalStatisticsCollector.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.statistics; import cn.edu.tsinghua.iginx.engine.shared.processor.PostLogicalProcessor; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/statistics/ParseStatisticsCollector.java b/core/src/main/java/cn/edu/tsinghua/iginx/statistics/ParseStatisticsCollector.java index 977ae3894d..ccb31eca68 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/statistics/ParseStatisticsCollector.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/statistics/ParseStatisticsCollector.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.statistics; import cn.edu.tsinghua.iginx.engine.shared.processor.PostParseProcessor; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/statistics/PhysicalStatisticsCollector.java b/core/src/main/java/cn/edu/tsinghua/iginx/statistics/PhysicalStatisticsCollector.java index 23d9ac0d77..f2c2719348 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/statistics/PhysicalStatisticsCollector.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/statistics/PhysicalStatisticsCollector.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.statistics; import cn.edu.tsinghua.iginx.engine.shared.processor.PostPhysicalProcessor; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/statistics/Statistics.java b/core/src/main/java/cn/edu/tsinghua/iginx/statistics/Statistics.java index 599d771516..b93d32d907 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/statistics/Statistics.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/statistics/Statistics.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.statistics; import cn.edu.tsinghua.iginx.engine.shared.RequestContext; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/statistics/StatisticsCollector.java b/core/src/main/java/cn/edu/tsinghua/iginx/statistics/StatisticsCollector.java index ba63ce0568..afd481f870 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/statistics/StatisticsCollector.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/statistics/StatisticsCollector.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.statistics; import cn.edu.tsinghua.iginx.conf.ConfigDescriptor; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/api/Checker.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/api/Checker.java index 02839241e8..ef01135f0f 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/transform/api/Checker.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/api/Checker.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.transform.api; import cn.edu.tsinghua.iginx.transform.pojo.Job; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/api/Driver.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/api/Driver.java index ed883d305b..d759876153 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/transform/api/Driver.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/api/Driver.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.transform.api; import cn.edu.tsinghua.iginx.transform.driver.IPCWorker; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/api/Reader.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/api/Reader.java index 0fe283f03b..a5559c65d9 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/transform/api/Reader.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/api/Reader.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.transform.api; import cn.edu.tsinghua.iginx.transform.data.BatchData; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/api/Runner.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/api/Runner.java index 79b96848a4..2d8a0a617d 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/transform/api/Runner.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/api/Runner.java @@ -1,12 +1,33 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.transform.api; import cn.edu.tsinghua.iginx.transform.exception.TransformException; +import org.quartz.SchedulerException; public interface Runner { - void start() throws TransformException; + void start() throws TransformException, SchedulerException; void run() throws TransformException; void close() throws TransformException; + + boolean scheduled(); } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/api/Stage.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/api/Stage.java index 86b6692ed0..f5fec0f5a8 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/transform/api/Stage.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/api/Stage.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.transform.api; import cn.edu.tsinghua.iginx.thrift.DataFlowType; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/api/Writer.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/api/Writer.java index 5b4e5f4cec..82ae54a2fe 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/transform/api/Writer.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/api/Writer.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.transform.api; import cn.edu.tsinghua.iginx.transform.data.BatchData; @@ -6,4 +24,7 @@ public interface Writer { void writeBatch(BatchData batchData) throws WriteBatchException; + + // reset state for next scheduled run + void reset(); } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/ArrowReader.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/ArrowReader.java index c828003c91..9dc490580c 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/ArrowReader.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/ArrowReader.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.transform.data; import cn.edu.tsinghua.iginx.engine.shared.data.read.Header; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/ArrowWriter.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/ArrowWriter.java index 79227d09a7..05eb2c596a 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/ArrowWriter.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/ArrowWriter.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.transform.data; import cn.edu.tsinghua.iginx.conf.Config; @@ -68,4 +86,7 @@ public void writeBatch(BatchData batchData) throws WriteBatchException { throw new WriteBatchException("ArrowWriter fail to write batch", e); } } + + @Override + public void reset() {} } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/BatchData.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/BatchData.java index 18efc3dfb5..f4a473d8ab 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/BatchData.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/BatchData.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.transform.data; import cn.edu.tsinghua.iginx.engine.shared.data.read.Header; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/CollectionWriter.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/CollectionWriter.java index ab874d94d5..cfdacf5dbf 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/CollectionWriter.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/CollectionWriter.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.transform.data; import cn.edu.tsinghua.iginx.engine.shared.data.read.Row; @@ -19,4 +37,9 @@ public void write(BatchData batchData) { public BatchData getCollectedData() { return collectedData; } + + @Override + public void reset() { + collectedData = null; + } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/ExportWriter.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/ExportWriter.java index 7b54bf2974..72f3d7ae05 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/ExportWriter.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/ExportWriter.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.transform.data; import cn.edu.tsinghua.iginx.transform.api.Writer; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/Exporter.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/Exporter.java index 2e16b31f69..52ca86a676 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/Exporter.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/Exporter.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.transform.data; import cn.edu.tsinghua.iginx.transform.utils.Mutex; @@ -5,4 +23,7 @@ public interface Exporter { Mutex getMutex(); + + // reset state for next scheduled run + void reset(); } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/FileAppendWriter.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/FileAppendWriter.java index 3b45d467b5..110a992f9d 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/FileAppendWriter.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/FileAppendWriter.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.transform.data; import cn.edu.tsinghua.iginx.auth.FilePermissionManager; @@ -93,4 +111,9 @@ private void writeFile(String fileName, String content) { LOGGER.error("unexpected error: ", e); } } + + @Override + public void reset() { + hasWriteHeader = false; + } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/IginXWriter.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/IginXWriter.java index ecff9a8edb..f83fbec8a8 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/IginXWriter.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/IginXWriter.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.transform.data; import cn.edu.tsinghua.iginx.engine.ContextBuilder; @@ -77,4 +95,7 @@ private String reformatPath(String path) { path = path.replaceAll("[}]", "]"); return path; } + + @Override + public void reset() {} } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/LogWriter.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/LogWriter.java index e26e6f8f2f..0520bd2526 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/LogWriter.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/LogWriter.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.transform.data; import cn.edu.tsinghua.iginx.constant.GlobalConstant; @@ -32,4 +50,9 @@ public void write(BatchData batchData) { LOGGER.info(row.toCSVTypeString()); }); } + + @Override + public void reset() { + hasWriteHeader = false; + } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/PemjaReader.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/PemjaReader.java index e547260133..ebe432a54a 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/PemjaReader.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/PemjaReader.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.transform.data; import cn.edu.tsinghua.iginx.engine.shared.data.read.Field; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/PemjaWriter.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/PemjaWriter.java index c1223f8675..311da56331 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/PemjaWriter.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/PemjaWriter.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.transform.data; import cn.edu.tsinghua.iginx.transform.api.Writer; @@ -16,4 +34,7 @@ public PemjaWriter(PemjaWorker worker) { public void writeBatch(BatchData batchData) throws WriteBatchException { worker.process(batchData); } + + @Override + public void reset() {} } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/RowStreamReader.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/RowStreamReader.java index e4237c9f56..cf7d7cc77c 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/RowStreamReader.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/RowStreamReader.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.transform.data; import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/SplitReader.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/SplitReader.java index 1100c588a1..465a5f58db 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/SplitReader.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/SplitReader.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.transform.data; import cn.edu.tsinghua.iginx.engine.shared.data.read.Header; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/driver/IPCWorker.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/driver/IPCWorker.java index c3476aab24..cb6a563ce0 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/transform/driver/IPCWorker.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/driver/IPCWorker.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.transform.driver; import cn.edu.tsinghua.iginx.conf.Config; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/driver/PemjaDriver.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/driver/PemjaDriver.java index f63356241a..ac9329561d 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/transform/driver/PemjaDriver.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/driver/PemjaDriver.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.transform.driver; import cn.edu.tsinghua.iginx.conf.Config; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/driver/PemjaWorker.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/driver/PemjaWorker.java index 9112bfba7b..06b3b31e5c 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/transform/driver/PemjaWorker.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/driver/PemjaWorker.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.transform.driver; import static cn.edu.tsinghua.iginx.transform.utils.Constants.UDF_CLASS; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/driver/PythonDriver.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/driver/PythonDriver.java index a22f8ceca5..7ca9fe8b58 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/transform/driver/PythonDriver.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/driver/PythonDriver.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.transform.driver; import cn.edu.tsinghua.iginx.conf.Config; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/exception/CreateWorkerException.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/exception/CreateWorkerException.java index 9f18171b78..ffc40d43b6 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/transform/exception/CreateWorkerException.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/exception/CreateWorkerException.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.transform.exception; public class CreateWorkerException extends TransformException { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/exception/TransformException.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/exception/TransformException.java index 056a2bb996..3e384da994 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/transform/exception/TransformException.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/exception/TransformException.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.transform.exception; public class TransformException extends Exception { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/exception/UnknownArgumentException.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/exception/UnknownArgumentException.java index b366e370b4..344eff4550 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/transform/exception/UnknownArgumentException.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/exception/UnknownArgumentException.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.transform.exception; public class UnknownArgumentException extends TransformException { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/exception/UnknownDataFlowException.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/exception/UnknownDataFlowException.java index 95d325bd21..71949376b1 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/transform/exception/UnknownDataFlowException.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/exception/UnknownDataFlowException.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.transform.exception; import cn.edu.tsinghua.iginx.thrift.DataFlowType; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/exception/WriteBatchException.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/exception/WriteBatchException.java index 6e194f71e0..9de96f644d 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/transform/exception/WriteBatchException.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/exception/WriteBatchException.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.transform.exception; public class WriteBatchException extends TransformException { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/exec/BatchStageRunner.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/exec/BatchStageRunner.java index 85fb21fa9a..6cab1ff374 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/transform/exec/BatchStageRunner.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/exec/BatchStageRunner.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.transform.exec; import cn.edu.tsinghua.iginx.transform.api.Runner; @@ -57,8 +75,17 @@ public void run() throws WriteBatchException { // wait for py work finish writing. mutex.lock(); + + mutex.unlock(); + writer.reset(); } @Override public void close() {} + + // schedule config would be set at higher level + @Override + public boolean scheduled() { + return false; + } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/exec/JobRunner.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/exec/JobRunner.java index 6871c2686f..bea851de4b 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/transform/exec/JobRunner.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/exec/JobRunner.java @@ -1,16 +1,38 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.transform.exec; import cn.edu.tsinghua.iginx.thrift.DataFlowType; import cn.edu.tsinghua.iginx.thrift.JobState; import cn.edu.tsinghua.iginx.transform.api.Runner; import cn.edu.tsinghua.iginx.transform.api.Stage; -import cn.edu.tsinghua.iginx.transform.exception.TransformException; import cn.edu.tsinghua.iginx.transform.exception.UnknownDataFlowException; import cn.edu.tsinghua.iginx.transform.pojo.BatchStage; import cn.edu.tsinghua.iginx.transform.pojo.Job; import cn.edu.tsinghua.iginx.transform.pojo.StreamStage; +import cn.edu.tsinghua.iginx.transform.pojo.TransformJobFinishListener; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import org.quartz.*; +import org.quartz.impl.StdSchedulerFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -21,13 +43,15 @@ public class JobRunner implements Runner { private final List runnerList; + private Scheduler scheduler; + public JobRunner(Job job) { this.job = job; this.runnerList = new ArrayList<>(); } @Override - public void start() throws UnknownDataFlowException { + public void start() throws UnknownDataFlowException, SchedulerException { for (Stage stage : job.getStageList()) { DataFlowType dataFlowType = stage.getStageType(); switch (dataFlowType) { @@ -42,41 +66,54 @@ public void start() throws UnknownDataFlowException { throw new UnknownDataFlowException(dataFlowType); } } + scheduler = StdSchedulerFactory.getDefaultScheduler(); + + JobDetail jobDetail = JobBuilder.newJob(ScheduledJob.class).build(); + + jobDetail.getJobDataMap().put("runnerList", runnerList); + jobDetail.getJobDataMap().put("job", job); + + Trigger trigger = job.getTrigger(); + LOGGER.info( + "Trigger details: StartTime={}, EndTime={}, NextFireTime={}", + trigger.getStartTime(), + trigger.getEndTime(), + trigger.getNextFireTime()); + + scheduler.scheduleJob(jobDetail, trigger); } @Override public void run() { - job.setState(JobState.JOB_RUNNING); - try { - for (Runner runner : runnerList) { - runner.start(); - runner.run(); - runner.close(); - } - // we don't need this.close() because all children runners are closed. - if (job.getActive().compareAndSet(true, false)) { - job.setState(JobState.JOB_FINISHED); - job.setException(null); - } - } catch (TransformException e) { - LOGGER.error("Fail to run transform job id={}, because", job.getJobId(), e); - if (job.getActive().compareAndSet(true, false)) { - job.setState(JobState.JOB_FAILING); - job.setException(e); - close(); - job.setState(JobState.JOB_FAILED); - } - } + // idle: waiting for scheduler to fire jobs + job.setState(JobState.JOB_IDLE); + ExecutorService executor = Executors.newSingleThreadExecutor(); + executor.submit( + // 新起一个线程 + () -> { + try { + scheduler.getListenerManager().addTriggerListener(new TransformJobFinishListener()); + LOGGER.info("Starting scheduler..."); + scheduler.start(); // 启动调度器 + LOGGER.info("Scheduler started"); + } catch (SchedulerException e) { + LOGGER.error("Failed to start scheduler", e); + job.setState(JobState.JOB_FAILED); + } + }); } @Override public void close() { try { - for (Runner runner : runnerList) { - runner.close(); - } - } catch (TransformException e) { + scheduler.shutdown(); + } catch (SchedulerException e) { LOGGER.error("Fail to close Transform job runner id={}, because", job.getJobId(), e); } } + + @Override + public boolean scheduled() { + return job.isScheduled(); + } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/exec/JobValidationChecker.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/exec/JobValidationChecker.java index a37367e03b..4cef4756f9 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/transform/exec/JobValidationChecker.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/exec/JobValidationChecker.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.transform.exec; import cn.edu.tsinghua.iginx.thrift.DataFlowType; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/exec/ScheduledJob.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/exec/ScheduledJob.java new file mode 100644 index 0000000000..f6cae81aab --- /dev/null +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/exec/ScheduledJob.java @@ -0,0 +1,71 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cn.edu.tsinghua.iginx.transform.exec; + +import cn.edu.tsinghua.iginx.thrift.JobState; +import cn.edu.tsinghua.iginx.transform.api.Runner; +import cn.edu.tsinghua.iginx.transform.exception.TransformException; +import cn.edu.tsinghua.iginx.transform.pojo.Job; +import java.util.List; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; +import org.quartz.SchedulerException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ScheduledJob implements org.quartz.Job { + private static final Logger LOGGER = LoggerFactory.getLogger(ScheduledJob.class); + + @Override + public void execute(JobExecutionContext context) throws JobExecutionException { + Job job = (Job) context.getMergedJobDataMap().get("job"); + List runnerList = (List) context.getMergedJobDataMap().get("runnerList"); + + job.setState(JobState.JOB_RUNNING); + job.getActive().compareAndSet(false, true); + try { + for (Runner runner : runnerList) { + runner.start(); + runner.run(); + runner.close(); + } + if (job.getActive().compareAndSet(true, false)) { + // wait for next execution + job.setState(JobState.JOB_IDLE); + job.setException(null); + } + // if a trigger has finished all execution, TransformJobFinishListener will handle work left. + } catch (TransformException | SchedulerException e) { + job.setState(JobState.JOB_FAILING); + job.setException(e); + try { + for (Runner runner : runnerList) { + runner.close(); + } + } catch (TransformException closeException) { + LOGGER.error("can't close job: {}", job.getJobId()); + } + JobExecutionException e2 = new JobExecutionException(e); + // Quartz will automatically unschedule + // all triggers associated with this job + // so that it does not run again + e2.setUnscheduleAllTriggers(true); + throw e2; + } + } +} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/exec/StreamStageRunner.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/exec/StreamStageRunner.java index 8e51a5135d..b47eeb3bfe 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/transform/exec/StreamStageRunner.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/exec/StreamStageRunner.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.transform.exec; import cn.edu.tsinghua.iginx.conf.Config; @@ -103,6 +121,10 @@ public void run() throws WriteBatchException { // wait for last batch finished. mutex.lock(); + + // unlock for further scheduled runs + mutex.unlock(); + writer.reset(); } @Override @@ -111,4 +133,10 @@ public void close() { reader.close(); } } + + // schedule config would be set at higher level + @Override + public boolean scheduled() { + return false; + } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/exec/TaskManager.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/exec/TaskManager.java index 506583a435..63b4998901 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/transform/exec/TaskManager.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/exec/TaskManager.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.transform.exec; public class TaskManager {} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/exec/TransformJobManager.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/exec/TransformJobManager.java index 63fc1ca2d2..6a5402a570 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/transform/exec/TransformJobManager.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/exec/TransformJobManager.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.transform.exec; import cn.edu.tsinghua.iginx.conf.Config; @@ -7,6 +25,7 @@ import cn.edu.tsinghua.iginx.thrift.JobState; import cn.edu.tsinghua.iginx.thrift.TaskType; import cn.edu.tsinghua.iginx.transform.api.Checker; +import cn.edu.tsinghua.iginx.transform.exception.TransformException; import cn.edu.tsinghua.iginx.transform.pojo.Job; import cn.edu.tsinghua.iginx.transform.pojo.PythonTask; import cn.edu.tsinghua.iginx.transform.pojo.Task; @@ -89,20 +108,15 @@ private void process(Job job) throws Exception { runner.start(); jobRunnerMap.put(job.getJobId(), runner); runner.run(); - jobRunnerMap.remove(job.getJobId()); // since we will retry, we can't do this in finally } catch (Exception e) { LOGGER.error("Fail to process transform job id={}, because", job.getJobId(), e); throw e; - } finally { - // TODO: is it legal to retry after runner.close()??? - // TODO: - // we don't need to close runner for FINISHED or FAILED jobs - // can we move runner.close() into catch clause? - runner.close(); } // TODO: should we set end time and log time cost for failed jobs? - job.setEndTime(System.currentTimeMillis()); - LOGGER.info("Job id={} cost {} ms.", job.getJobId(), job.getEndTime() - job.getStartTime()); + if (!runner.scheduled()) { + job.setEndTime(System.currentTimeMillis()); + LOGGER.info("Job id={} cost {} ms.", job.getJobId(), job.getEndTime() - job.getStartTime()); + } } public boolean cancel(long jobId) { @@ -129,22 +143,40 @@ public boolean cancel(long jobId) { switch (job.getState()) { // won't be null case JOB_RUNNING: case JOB_CREATED: - break; // continue execution + // atomic guard + if (!job.getActive().compareAndSet(true, false)) { + return false; + } + case JOB_IDLE: + // reorder as Normal run: [set-ING,] close, set-ED, remove[, set end time, log time cost]. + job.setState(JobState.JOB_CLOSING); + runner.close(); + job.setState(JobState.JOB_CLOSED); + jobRunnerMap.remove(jobId); + job.setEndTime(System.currentTimeMillis()); + LOGGER.info("Job id={} cost {} ms.", job.getJobId(), job.getEndTime() - job.getStartTime()); + return true; default: return false; } - // atomic guard - if (!job.getActive().compareAndSet(true, false)) { - return false; + } + + public void removeFinishedScheduleJob(long jobID) throws TransformException { + Job job = jobMap.get(jobID); + if (job == null) { + throw new TransformException("No job with id: " + jobID + " exists."); + } + JobRunner runner = jobRunnerMap.get(jobID); + if (runner == null) { + throw new TransformException("No job runner with id: " + jobID + " exists."); + } + if (job.getState() == JobState.JOB_FINISHED) { + jobRunnerMap.remove(jobID); + runner.close(); + return; } - // reorder as Normal run: [set-ING,] close, set-ED, remove[, set end time, log time cost]. - job.setState(JobState.JOB_CLOSING); - runner.close(); - job.setState(JobState.JOB_CLOSED); - jobRunnerMap.remove(jobId); - job.setEndTime(System.currentTimeMillis()); - LOGGER.info("Job id={} cost {} ms.", job.getJobId(), job.getEndTime() - job.getStartTime()); - return true; + throw new TransformException( + "Job with id: " + jobID + "did not finish correctly. Current state: " + job.getState()); } public JobState queryJobState(long jobId) { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/pojo/BatchStage.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/pojo/BatchStage.java index 436daf29e4..4a8f7189af 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/transform/pojo/BatchStage.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/pojo/BatchStage.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.transform.pojo; import cn.edu.tsinghua.iginx.thrift.DataFlowType; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/pojo/IginXTask.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/pojo/IginXTask.java index 3c7a287577..f0ce7d4f2e 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/transform/pojo/IginXTask.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/pojo/IginXTask.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.transform.pojo; import cn.edu.tsinghua.iginx.thrift.TaskInfo; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/pojo/Job.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/pojo/Job.java index d2f71812d9..877dc9e1e3 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/transform/pojo/Job.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/pojo/Job.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.transform.pojo; import cn.edu.tsinghua.iginx.thrift.*; @@ -9,6 +27,8 @@ import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; import lombok.Data; +import org.quartz.Trigger; +import org.quartz.TriggerBuilder; @Data public class Job { @@ -30,11 +50,15 @@ public class Job { private final List taskList; private final List stageList; + private boolean scheduled = false; + private String scheduleStr = null; + private final Trigger trigger; + public Job(long id, CommitTransformJobReq req) { jobId = id; sessionId = req.getSessionId(); state = JobState.JOB_CREATED; - active = new AtomicBoolean(true); + active = new AtomicBoolean(false); exportType = req.getExportType(); if (exportType.equals(ExportType.File)) { @@ -79,13 +103,21 @@ public Job(long id, CommitTransformJobReq req) { stage = new StreamStage(sessionId, stage, new ArrayList<>(stageTasks), writer); stageList.add(stage); } + if (req.isSetSchedule()) { + trigger = JobScheduleTriggerMaker.getTrigger(req.getSchedule()); + scheduled = true; + scheduleStr = req.getSchedule(); + } else { + // no schedule information provided. job will be fired instantly + trigger = TriggerBuilder.newTrigger().startNow().build(); + } } public Job(long id, long sessionId, JobFromYAML jobFromYAML) { this.jobId = id; this.sessionId = sessionId; this.state = JobState.JOB_CREATED; - active = new AtomicBoolean(true); + active = new AtomicBoolean(false); String exportType = jobFromYAML.getExportType().toLowerCase().trim(); if (exportType.equals("file")) { @@ -133,6 +165,14 @@ public Job(long id, long sessionId, JobFromYAML jobFromYAML) { stage = new StreamStage(sessionId, stage, new ArrayList<>(stageTasks), writer); stageList.add(stage); } + + if (jobFromYAML.getSchedule() != null && !jobFromYAML.getSchedule().isEmpty()) { + trigger = JobScheduleTriggerMaker.getTrigger(jobFromYAML.getSchedule()); + scheduled = true; + scheduleStr = jobFromYAML.getSchedule(); + } else { + trigger = TriggerBuilder.newTrigger().startNow().build(); + } } @Override @@ -158,6 +198,7 @@ public String toString() { + taskList + ", stageList=" + stageList + + (scheduleStr != null ? ", schedule string=" + scheduleStr : "") + '}'; } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/pojo/JobScheduleTriggerMaker.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/pojo/JobScheduleTriggerMaker.java new file mode 100644 index 0000000000..cd56a39e9e --- /dev/null +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/pojo/JobScheduleTriggerMaker.java @@ -0,0 +1,395 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cn.edu.tsinghua.iginx.transform.pojo; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import javax.validation.constraints.NotNull; +import org.quartz.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class JobScheduleTriggerMaker { + + private static final Logger LOGGER = LoggerFactory.getLogger(JobScheduleTriggerMaker.class); + private static final SimpleDateFormat DATE_TIME_FORMAT = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + private static final SimpleDateFormat TIME_FORMAT = new SimpleDateFormat("HH:mm:ss"); + private static final String weekdayRegex = + "(mon|tue|wed|thu|fri|sat|sun|monday|tuesday|wednesday|thursday|friday|saturday|sunday)"; + + private static final Pattern everyPattern = + Pattern.compile( + "(?i)^every\\s+(\\d+)\\s+(second|minute|hour|day|month|year)(?:\\s+starts\\s+'([^']+)')?(?:\\s+ends\\s+'([^']+)')?$"); + + private static final Pattern everyWeekdayPattern = + Pattern.compile(String.format("(?i)^every ((?:%s,?)+)$", weekdayRegex)); + + private static final Pattern afterPattern = + Pattern.compile("(?i)^after\\s+(\\d+)\\s+(second|minute|hour|day|month|year)$"); + + private static final Pattern atPattern = + Pattern.compile("(?i)^at\\s+'((?:\\d{4}-\\d{2}-\\d{2}\\s+)?\\d{2}:\\d{2}:\\d{2})'$"); + + private static String errMsg; + + private static Date now; + + private enum INTERVAL_ENUM { + SECOND, + MINUTE, + HOUR, + DAY, + MONTH, + YEAR; + + public static INTERVAL_ENUM matcher(String unit) { + unit = unit.trim().toLowerCase(); + switch (unit) { + case "second": + return SECOND; + case "minute": + return MINUTE; + case "hour": + return HOUR; + case "day": + return DAY; + case "month": + return MONTH; + case "year": + return YEAR; + default: + return null; + } + } + } + + /** + * 根据调度字符串生成 Quartz Trigger。调度字符串大致可分为以下四种: 1. every 3 second/minute/hour/day/month/year + * 每隔3秒/分/小时/天/月/年执行一次,可以添加开始时间和结束时间,两个时间必须用单引号包围,例如: every 10 minute starts '2024-02-03 12:00:00' + * ends '2024-02-04 12:00:00' 2. after 3 second/minute/hour/day/month/year 在3秒/分/小时/天/月/年后执行一次; 3. + * at (yyyy-MM-dd)? HH:mm:ss 在指定时间执行 4. (* * * * * *) cron格式的字符串 + * + * @param jobSchedule 调度字符串,控制任务执行的时间 + * @return 返回在指定时间触发的Quartz触发器 + */ + public static Trigger getTrigger(@NotNull String jobSchedule) { + // corn in quartz is not case-sensitive + jobSchedule = jobSchedule.trim().toLowerCase(); + now = new Date(); + if (jobSchedule.isEmpty()) { + throw new IllegalArgumentException("Job schedule indicator string is empty."); + } + if (jobSchedule.startsWith("every")) { + if (jobSchedule.matches(String.valueOf(everyWeekdayPattern))) { + return everyWeeklyTrigger(jobSchedule); + } + return everyTrigger(jobSchedule); + } else if (jobSchedule.startsWith("after")) { + return afterTrigger(jobSchedule); + } else if (jobSchedule.startsWith("at")) { + return atTrigger(jobSchedule); + } else if (jobSchedule.matches("\\(.*\\)")) { + return cronTrigger(jobSchedule); + } + + throw new IllegalArgumentException("Invalid time format: " + jobSchedule); + } + + private static Trigger everyTrigger(String jobSchedule) { + Matcher nomalEverymatcher = everyPattern.matcher(jobSchedule); + TriggerBuilder triggerBuilder = TriggerBuilder.newTrigger(); + + if (nomalEverymatcher.matches()) { + int intervalValue = Integer.parseInt(nomalEverymatcher.group(1)); + INTERVAL_ENUM intervalUnit = INTERVAL_ENUM.matcher(nomalEverymatcher.group(2)); + if (intervalUnit == null) { + LOGGER.error( + "Error parsing interval unit {}. Available: second, minute, hour, day.", + nomalEverymatcher.group(2)); + throw new IllegalArgumentException( + "Error parsing interval unit " + + nomalEverymatcher.group(2) + + ". Available: second, minute, hour, day."); + } + String starts = nomalEverymatcher.group(3); + String ends = nomalEverymatcher.group(4); + + switch (intervalUnit) { + case SECOND: + triggerBuilder.withSchedule( + SimpleScheduleBuilder.simpleSchedule() + .withIntervalInSeconds(intervalValue) + .repeatForever()); + break; + case MINUTE: + triggerBuilder.withSchedule( + SimpleScheduleBuilder.simpleSchedule() + .withIntervalInMinutes(intervalValue) + .repeatForever()); + break; + case HOUR: + triggerBuilder.withSchedule( + SimpleScheduleBuilder.simpleSchedule() + .withIntervalInHours(intervalValue) + .repeatForever()); + break; + case DAY: + triggerBuilder.withSchedule( + SimpleScheduleBuilder.simpleSchedule() + .withIntervalInHours(intervalValue * 24) + .repeatForever()); + break; + case MONTH: + triggerBuilder.withSchedule( + CalendarIntervalScheduleBuilder.calendarIntervalSchedule() + .withIntervalInMonths(intervalValue)); + break; + case YEAR: + triggerBuilder.withSchedule( + CalendarIntervalScheduleBuilder.calendarIntervalSchedule() + .withIntervalInYears(intervalValue)); + break; + } + + Date startDate = null, endDate = null; + + if (starts != null) { + try { + startDate = parseDate(starts); + if (startDate.before(now)) { + errMsg = String.format("Start Time %s is before current time.", starts); + LOGGER.error(errMsg); + throw new IllegalArgumentException(errMsg); + } + triggerBuilder.startAt(startDate); + } catch (ParseException e) { + errMsg = String.format("Error parsing start time %s. Please refer to manual.", starts); + LOGGER.error(errMsg, e); + throw new IllegalArgumentException(errMsg); + } + } else { + triggerBuilder.startAt(now); + } + + if (ends != null) { + try { + endDate = parseDate(ends); + if (startDate != null && endDate.before(startDate)) { + errMsg = String.format("End Time %s is before start time %s.", ends, starts); + LOGGER.error(errMsg); + throw new IllegalArgumentException(errMsg); + } + if (endDate.before(now)) { + errMsg = String.format("End Time %s is before current time.", ends); + LOGGER.error(errMsg); + throw new IllegalArgumentException(errMsg); + } + triggerBuilder.endAt(endDate); + } catch (ParseException e) { + errMsg = String.format("Error parsing end time %s. Please refer to manual.", ends); + LOGGER.error(errMsg); + throw new IllegalArgumentException(errMsg); + } + } + } else { + errMsg = + String.format("Error parsing schedule string %s. Please refer to manual.", jobSchedule); + LOGGER.error(errMsg); + throw new IllegalArgumentException(errMsg); + } + return triggerBuilder.build(); + } + + private static Trigger everyWeeklyTrigger(String jobSchedule) { + Matcher matcher = everyWeekdayPattern.matcher(jobSchedule); + if (matcher.matches()) { + String daysString = matcher.group(1); + String[] daysArray = daysString.split(","); + Set daysOfWeek = new HashSet<>(); + + for (String day : daysArray) { + day = day.trim().toLowerCase(Locale.ROOT); + switch (day) { + case "mon": + case "monday": + daysOfWeek.add(DateBuilder.MONDAY); + break; + case "tue": + case "tuesday": + daysOfWeek.add(DateBuilder.TUESDAY); + break; + case "wed": + case "wednesday": + daysOfWeek.add(DateBuilder.WEDNESDAY); + break; + case "thu": + case "thursday": + daysOfWeek.add(DateBuilder.THURSDAY); + break; + case "fri": + case "friday": + daysOfWeek.add(DateBuilder.FRIDAY); + break; + case "sat": + case "saturday": + daysOfWeek.add(DateBuilder.SATURDAY); + break; + case "sun": + case "sunday": + daysOfWeek.add(DateBuilder.SUNDAY); + break; + default: + throw new IllegalArgumentException("Invalid day of the week: " + day); + } + } + return TriggerBuilder.newTrigger() + .withSchedule( + DailyTimeIntervalScheduleBuilder.dailyTimeIntervalSchedule() + .onDaysOfTheWeek(daysOfWeek.toArray(new Integer[0]))) + .build(); + } + throw new IllegalArgumentException("Invalid weekly format: " + jobSchedule); + } + + private static Trigger afterTrigger(String jobSchedule) { + Matcher afterMatcher = afterPattern.matcher(jobSchedule); + TriggerBuilder triggerBuilder = TriggerBuilder.newTrigger(); + + if (afterMatcher.matches()) { + int intervalValue = Integer.parseInt(afterMatcher.group(1)); + INTERVAL_ENUM intervalUnit = INTERVAL_ENUM.matcher(afterMatcher.group(2)); + if (intervalUnit == null) { + LOGGER.error( + "Error parsing interval unit {}. Available: second, minute, hour, day.", + afterMatcher.group(2)); + throw new IllegalArgumentException( + "Error parsing interval unit " + + afterMatcher.group(2) + + ". Available: second, minute, hour, day."); + } + + switch (intervalUnit) { + case SECOND: + triggerBuilder.startAt( + DateBuilder.futureDate(intervalValue, DateBuilder.IntervalUnit.SECOND)); + break; + case MINUTE: + triggerBuilder.startAt( + DateBuilder.futureDate(intervalValue, DateBuilder.IntervalUnit.MINUTE)); + break; + case HOUR: + triggerBuilder.startAt( + DateBuilder.futureDate(intervalValue, DateBuilder.IntervalUnit.HOUR)); + break; + case DAY: + triggerBuilder.startAt( + DateBuilder.futureDate(intervalValue, DateBuilder.IntervalUnit.DAY)); + break; + case MONTH: + triggerBuilder.startAt( + DateBuilder.futureDate(intervalValue, DateBuilder.IntervalUnit.MONTH)); + break; + case YEAR: + triggerBuilder.startAt( + DateBuilder.futureDate(intervalValue, DateBuilder.IntervalUnit.YEAR)); + break; + } + } else { + LOGGER.error("Error parsing schedule string {}. Please refer to manual.", jobSchedule); + throw new IllegalArgumentException( + "Error parsing schedule string " + jobSchedule + ". Please refer to manual."); + } + return triggerBuilder.build(); + } + + private static Trigger atTrigger(String jobSchedule) { + Matcher atMatcher = atPattern.matcher(jobSchedule); + + if (atMatcher.matches()) { + String atTime = atMatcher.group(1); + try { + Date atDate = parseDate(atTime); + if (atDate.before(now)) { + errMsg = + String.format("Trying to trigger a job at %s which is before current time", atTime); + LOGGER.error(errMsg); + throw new IllegalArgumentException(errMsg); + } + return TriggerBuilder.newTrigger() + .startAt(atDate) + .withSchedule( + SimpleScheduleBuilder.simpleSchedule() + .withMisfireHandlingInstructionFireNow() + .withRepeatCount(0)) + .build(); + } catch (ParseException e) { + errMsg = String.format("Error parsing time %s. Please refer to manual.", atTime); + LOGGER.error(errMsg); + throw new IllegalArgumentException(errMsg); + } + } else { + LOGGER.error("Error parsing schedule string {}. Please refer to manual.", jobSchedule); + throw new IllegalArgumentException( + "Error parsing schedule string " + jobSchedule + ". Please refer to manual."); + } + } + + private static Trigger cronTrigger(String jobSchedule) { + String cronExpression = jobSchedule.substring(1, jobSchedule.length() - 1); + if (cronExpression.isEmpty()) { + errMsg = "Cron string is empty. Please provide a valid corn expression."; + LOGGER.error(errMsg); + throw new IllegalArgumentException(errMsg); + } + if (!CronExpression.isValidExpression(cronExpression)) { + errMsg = + String.format( + "Cron string (%s) is not valid. Please provide a valid cron expression.", + cronExpression); + LOGGER.error(errMsg); + throw new IllegalArgumentException(errMsg); + } + return TriggerBuilder.newTrigger() + .withSchedule(CronScheduleBuilder.cronSchedule(cronExpression)) + .build(); + } + + /** + * 解析日期“HH:mm:ss”格式或者"yyyy-MM-dd HH:mm:ss"格式 + * + * @param dateString 日期字符串 + * @return 解析得到的Date类型变量 + * @throws ParseException + */ + private static Date parseDate(String dateString) throws ParseException { + if (dateString == null || dateString.isEmpty()) { + return null; + } + if (dateString.length() == 8) { // 'HH:mm:ss' format + // Set the date part to the current date + String today = new SimpleDateFormat("yyyy-MM-dd").format(now); + dateString = today + " " + dateString; + } + // 'yyyy-MM-dd HH:mm:ss' format + return DATE_TIME_FORMAT.parse(dateString); + } +} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/pojo/PythonTask.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/pojo/PythonTask.java index 5dd94ba237..42b10a44ca 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/transform/pojo/PythonTask.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/pojo/PythonTask.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.transform.pojo; import cn.edu.tsinghua.iginx.thrift.DataFlowType; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/pojo/StreamStage.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/pojo/StreamStage.java index 6a8bba7829..8cf2a4e530 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/transform/pojo/StreamStage.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/pojo/StreamStage.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.transform.pojo; import cn.edu.tsinghua.iginx.thrift.DataFlowType; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/pojo/Task.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/pojo/Task.java index 218a96af48..7026da763a 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/transform/pojo/Task.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/pojo/Task.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.transform.pojo; import cn.edu.tsinghua.iginx.thrift.DataFlowType; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/pojo/TaskFactory.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/pojo/TaskFactory.java index 72e4141811..77dd82f434 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/transform/pojo/TaskFactory.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/pojo/TaskFactory.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.transform.pojo; import cn.edu.tsinghua.iginx.thrift.TaskInfo; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/pojo/TransformJobFinishListener.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/pojo/TransformJobFinishListener.java new file mode 100644 index 0000000000..fda033d779 --- /dev/null +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/pojo/TransformJobFinishListener.java @@ -0,0 +1,70 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cn.edu.tsinghua.iginx.transform.pojo; + +import cn.edu.tsinghua.iginx.thrift.JobState; +import cn.edu.tsinghua.iginx.transform.exception.TransformException; +import cn.edu.tsinghua.iginx.transform.exec.TransformJobManager; +import org.quartz.JobExecutionContext; +import org.quartz.Trigger; +import org.quartz.listeners.TriggerListenerSupport; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TransformJobFinishListener extends TriggerListenerSupport { + + private static final Logger LOGGER = LoggerFactory.getLogger(TransformJobFinishListener.class); + + @Override + public String getName() { + return "JobFinishListener"; + } + + @Override + public void triggerFired(Trigger trigger, JobExecutionContext context) { + // 触发器被触发时的操作 + LOGGER.info( + "Trigger fired: {}, job:{}", + trigger.getKey(), + ((Job) context.getMergedJobDataMap().get("job")).getJobId()); + } + + @Override + public void triggerMisfired(Trigger trigger) { + // 触发器错过触发时的操作 + LOGGER.warn("Trigger misfired: {}", trigger.getKey()); + } + + @Override + public void triggerComplete( + Trigger trigger, + JobExecutionContext context, + Trigger.CompletedExecutionInstruction triggerInstructionCode) { + if (trigger.getNextFireTime() == null) { + // 触发器的所有执行结束 + Job job = (Job) context.getMergedJobDataMap().get("job"); + job.setState(JobState.JOB_FINISHED); + LOGGER.info("Job {} has completed all executions.", job.getJobId()); + try { + TransformJobManager.getInstance().removeFinishedScheduleJob(job.getJobId()); + } catch (TransformException e) { + throw new RuntimeException(e); + } + } + } +} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/utils/Constants.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/utils/Constants.java index d5a0abdbe9..53d3eb49d9 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/transform/utils/Constants.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/utils/Constants.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.transform.utils; import java.util.HashMap; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/utils/Mutex.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/utils/Mutex.java index 0a796e57d0..2d88bf1585 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/transform/utils/Mutex.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/utils/Mutex.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.transform.utils; import org.slf4j.Logger; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/utils/RedirectLogger.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/utils/RedirectLogger.java index e9fe35f067..bf4916983c 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/transform/utils/RedirectLogger.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/utils/RedirectLogger.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.transform.utils; import java.io.InputStream; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/utils/TypeUtils.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/utils/TypeUtils.java index 2d1191461e..7c62485dea 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/transform/utils/TypeUtils.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/utils/TypeUtils.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.transform.utils; import cn.edu.tsinghua.iginx.thrift.DataType; diff --git a/core/src/main/resources/file-permission.properties b/core/src/main/resources/file-permission.properties index e95a5a5592..a8b8148fa5 100644 --- a/core/src/main/resources/file-permission.properties +++ b/core/src/main/resources/file-permission.properties @@ -1,160 +1,23 @@ -# This file is used to configure the accessType of the files in IGinX. # +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University # -# Configuration format: -# ===================== +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. # -# Note: +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. # -# 1. If there is conflict between key, the first one will be used. -# 2. Charset should be UTF-8. -# 3. The configuration file is in the format of `properties`. -# 4. The key-value pair is separated by `=`. -# 5. The file is case-sensitive. -# -# Variable: -# -# The value of the key can contain variables, which will be replaced by the -# real value. Including: -# -# * `${}`: get the value of this configuration file. -# * `${env:}`: get the value of the environment variable. -# * `${sys:}`: get the value of the system property. -# -# more details: -# https://commons.apache.org/proper/commons-configuration/userguide/howto_basicfeatures.html#Variable_Interpolation -# -# -# Refresh Interval: -# ================= -# -# IGinX will refresh the accessType configuration file at a regular interval. -# The refresh interval can be configured by the `refreshInterval` key. -# The value is the interval in milliseconds. The default value is 5000. -# -# -# File access rule: -# ================= -# -# The file accessType rule is used to configure the accessType of the files -# in IGinX. The rule can be configured to allow or deny the file to be -# read/write/execute. The rule can be applied to the specific user and module. -# -# Rule Format: -# -# The rule is represented by 4 configuration items with the same prefix: -# -# ..include = -# ..read = true/false -# ..write = true/false -# ..execute = true/false -# -# where: -# * `` is the user name that the rule applies to. Specific users: -# + `default` means the default user. -# - All users first match the rules of specific users, if no match, -# then match the rules of the default user. -# + `root` means the root user. -# * `` is the name of the rule. -# + The name of the rule is used to distinguish different rules. -# * `include` means the file path pattern that the rule applies to. -# + this property is required. -# * `read/write/execute` means whether file can be read/written/executed. -# + this property is optional. -# + if read not set, the rule will be ignored when checking the read. -# Write and execute are the same. -# * `` is the pattern of the absolute file path. It can be: -# - `glob:`: matches the file path with the glob pattern. -# - `regex:`: matches the file path with the regex pattern. -# more details: -# https://docs.oracle.com/javase/8/docs/api/java/nio/file/FileSystem.html#getPathMatcher-java.lang.String- -# -# Rule Name: -# -# The rule name can be any string, but it should be unique for the same user. -# The rule name is used to distinguish different rules. Different modules -# determine the subset of rules used by filter. In details: -# -# * the default module uses rules starting with "default". -# + the rules of the default module is applied to all modules. -# * the udf module uses rules starting with "udf" -# * the transform module uses rules starting with "transformer". -# * the filesystem driver uses rules starting with "filesystem". -# -# Rule Application Order: -# 1. For the same user, the rules are applied in the order of the configuration -# file. The first rule that matches the file path will be used to determine -# whether the file can be read/written/executed. -# 2. If there is no read/write/execute for specific file when checking the rule, -# try to apply the next rule. -# 3. If there is no rule that matches the file path, the default user will -# be used. -# 4. If there is no rule that matches the file path for the default user, the -# default accessType will be used. Default accessType is `true`. -# -# Example: -# -# root.defaultRule.include=glob:** -# root.defaultRule.read=true -# root.defaultRule.write=true -# root.defaultRule.execute=true -# -# default.defaultRule.include=glob:**.parquet -# default.defaultRule.read=true -# -# default.defaultRule.include=glob:** -# default.defaultRule.read=false -# default.defaultRule.write=false -# default.defaultRule.execute=false -# -# in this example: all users can read all files. Only the root user can write -# and execute all files. The other users only can read the parquet files. -# -# -# Important Note: -# ============ -# -# Only the default user is supported currently. -# --------------------------------------------- -# -# Although the configuration file supports setting different permissions for -# different users, currently only the default user is supported, and the -# configuration of other users will be ignored. +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . # -# the refresh interval of the accessType configuration file in milliseconds refreshInterval=1000 - -# match all files for root user and default module -#root.defaultRule.include=glob:** -#root.defaultRule.read=true -#root.defaultRule.write=true -#root.defaultRule.execute=true - -# match iginx internal python files for default user and udf & transform module -default.transformerRule.include=glob:${env:IGINX_HOME}${sys:file.separator}udf_funcs${sys:file.separator}**.py -default.transformerRule.read=false -default.transformerRule.write=true -default.transformerRule.execute=true -default.udfTheRule.include=${default.transformerRule.include} -default.udfTheRule.read=${default.transformerRule.read} -default.udfTheRule.write=${default.transformerRule.write} -default.udfTheRule.execute=${default.transformerRule.execute} - -# match iginx internal system files for default user and default module -default.defaultFirstRule.include=glob:${env:IGINX_HOME}${sys:file.separator}**.{jar,properties,sh,bat} -default.defaultFirstRule.read=false -default.defaultFirstRule.write=false -default.defaultFirstRule.execute=false - -# match log files for default user and default module -default.default2ndRule.include=glob:${sys:user.dir}${sys:file.separator}logs${sys:file.separator}*.{log,log.gz} -default.default2ndRule.read=true -default.default2ndRule.write=false -default.default2ndRule.execute=false - -# match all files for default user and module -default.defaultLastRule.include=glob:** -default.defaultLastRule.read=true -default.defaultLastRule.write=true -default.defaultLastRule.execute=true +default.defaultRule.include=glob:** +default.defaultRule.read=true +default.defaultRule.write=true +default.defaultRule.execute=true \ No newline at end of file diff --git a/core/src/main/resources/log4j2.properties b/core/src/main/resources/log4j2.properties index cc24c56f20..78ec9c18c3 100644 --- a/core/src/main/resources/log4j2.properties +++ b/core/src/main/resources/log4j2.properties @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + # Update log configuration from file every 30 seconds monitorInterval=30 @@ -8,28 +26,11 @@ appender.console.type=Console appender.console.target=SYSTEM_ERR appender.console.layout.type=PatternLayout appender.console.layout.pattern=%d %highlight{%-5p} [%t] - [%C.%M:%L] %m%n%ex -#rolling-file -appender.rolling.name=RollingFileAppender -appender.rolling.type=RollingFile -appender.rolling.fileName=logs/iginx-latest.log -appender.rolling.filePattern=logs/iginx-%d{yyyy-MM-dd}-%i.log.gz -appender.rolling.layout.type=PatternLayout -appender.rolling.layout.pattern=%d %-5p [%t] - [%C.%M:%L] %m%n%ex -appender.rolling.policies.type=Policies -appender.rolling.policies.time.type=TimeBasedTriggeringPolicy -appender.rolling.policies.time.interval=1 -appender.rolling.policies.time.modulate=true -appender.rolling.policies.size.type=SizeBasedTriggeringPolicy -appender.rolling.policies.size.size=100MB -appender.rolling.policies.startup.type=OnStartupTriggeringPolicy -appender.rolling.strategy.type=DefaultRolloverStrategy -appender.rolling.strategy.max=30 # Define Loggers #root rootLogger.level=warn rootLogger.appenderRef.console.ref=ConsoleAppender -rootLogger.appenderRef.rolling.ref=RollingFileAppender #iginx logger.iginx.name=cn.edu.tsinghua.iginx logger.iginx.level=info diff --git a/core/src/test/java/cn/edu/tsinghua/iginx/auth/FilePermissionManagerTest.java b/core/src/test/java/cn/edu/tsinghua/iginx/auth/FilePermissionManagerTest.java index 849af83d5c..703ff28fcb 100644 --- a/core/src/test/java/cn/edu/tsinghua/iginx/auth/FilePermissionManagerTest.java +++ b/core/src/test/java/cn/edu/tsinghua/iginx/auth/FilePermissionManagerTest.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.auth; import static cn.edu.tsinghua.iginx.auth.utils.FilePermissionRuleNameFilters.defaultRules; diff --git a/core/src/test/java/cn/edu/tsinghua/iginx/compaction/LowAccessFragmentCompactionTest.java b/core/src/test/java/cn/edu/tsinghua/iginx/compaction/LowAccessFragmentCompactionTest.java index 22c479f6de..d7c9a1d1d7 100644 --- a/core/src/test/java/cn/edu/tsinghua/iginx/compaction/LowAccessFragmentCompactionTest.java +++ b/core/src/test/java/cn/edu/tsinghua/iginx/compaction/LowAccessFragmentCompactionTest.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.compaction; import static org.junit.Assert.assertEquals; diff --git a/core/src/test/java/cn/edu/tsinghua/iginx/compaction/LowWriteFragmentCompactionTest.java b/core/src/test/java/cn/edu/tsinghua/iginx/compaction/LowWriteFragmentCompactionTest.java index b0e2019418..7163a81e81 100644 --- a/core/src/test/java/cn/edu/tsinghua/iginx/compaction/LowWriteFragmentCompactionTest.java +++ b/core/src/test/java/cn/edu/tsinghua/iginx/compaction/LowWriteFragmentCompactionTest.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.compaction; import static org.junit.Assert.assertEquals; diff --git a/core/src/test/java/cn/edu/tsinghua/iginx/compaction/PhysicalEngineMock.java b/core/src/test/java/cn/edu/tsinghua/iginx/compaction/PhysicalEngineMock.java index 0a4ea4f510..a5e9c375bf 100644 --- a/core/src/test/java/cn/edu/tsinghua/iginx/compaction/PhysicalEngineMock.java +++ b/core/src/test/java/cn/edu/tsinghua/iginx/compaction/PhysicalEngineMock.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.compaction; import cn.edu.tsinghua.iginx.engine.physical.PhysicalEngine; diff --git a/core/src/test/java/cn/edu/tsinghua/iginx/conf/FilePermissionConfigTest.java b/core/src/test/java/cn/edu/tsinghua/iginx/conf/FilePermissionConfigTest.java index df0ea46eac..2c59afaa35 100644 --- a/core/src/test/java/cn/edu/tsinghua/iginx/conf/FilePermissionConfigTest.java +++ b/core/src/test/java/cn/edu/tsinghua/iginx/conf/FilePermissionConfigTest.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.conf; import static org.junit.Assert.*; diff --git a/core/src/test/java/cn/edu/tsinghua/iginx/engine/logical/utils/LogicalFilterUtilsTest.java b/core/src/test/java/cn/edu/tsinghua/iginx/engine/logical/utils/LogicalFilterUtilsTest.java index 32ac842d6a..4ed699fe19 100644 --- a/core/src/test/java/cn/edu/tsinghua/iginx/engine/logical/utils/LogicalFilterUtilsTest.java +++ b/core/src/test/java/cn/edu/tsinghua/iginx/engine/logical/utils/LogicalFilterUtilsTest.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.logical.utils; import static org.junit.Assert.assertEquals; diff --git a/core/src/test/java/cn/edu/tsinghua/iginx/engine/logical/utils/PathUtilsTest.java b/core/src/test/java/cn/edu/tsinghua/iginx/engine/logical/utils/PathUtilsTest.java index 6feb53be8f..f181be96f5 100644 --- a/core/src/test/java/cn/edu/tsinghua/iginx/engine/logical/utils/PathUtilsTest.java +++ b/core/src/test/java/cn/edu/tsinghua/iginx/engine/logical/utils/PathUtilsTest.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.logical.utils; import static org.junit.Assert.assertEquals; diff --git a/core/src/test/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/AbstractOperatorMemoryExecutorTest.java b/core/src/test/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/AbstractOperatorMemoryExecutorTest.java index 4e383f69d6..7c08858257 100644 --- a/core/src/test/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/AbstractOperatorMemoryExecutorTest.java +++ b/core/src/test/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/AbstractOperatorMemoryExecutorTest.java @@ -1,23 +1,24 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.memory.execute; +import static cn.edu.tsinghua.iginx.engine.shared.Constants.WINDOW_END_COL; +import static cn.edu.tsinghua.iginx.engine.shared.Constants.WINDOW_START_COL; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -2263,9 +2264,11 @@ public void testDownsample() throws PhysicalException { Header targetHeader = stream.getHeader(); assertTrue(targetHeader.hasKey()); - assertEquals(1, targetHeader.getFields().size()); - assertEquals("avg(a.a.b)", targetHeader.getFields().get(0).getFullName()); - assertEquals(DataType.DOUBLE, targetHeader.getFields().get(0).getType()); + assertEquals(3, targetHeader.getFields().size()); + assertEquals(WINDOW_START_COL, targetHeader.getFields().get(0).getFullName()); + assertEquals(WINDOW_END_COL, targetHeader.getFields().get(1).getFullName()); + assertEquals("avg(a.a.b)", targetHeader.getFields().get(2).getFullName()); + assertEquals(DataType.DOUBLE, targetHeader.getFields().get(2).getType()); int index = 0; while (stream.hasNext()) { @@ -2276,7 +2279,7 @@ public void testDownsample() throws PhysicalException { sum += (int) table.getRow(index + cnt).getValue("a.a.b"); cnt++; } - assertEquals(sum * 1.0 / cnt, (double) targetRow.getValue(0), 0.01); + assertEquals(sum * 1.0 / cnt, (double) targetRow.getValue(2), 0.01); index += cnt; } assertEquals(table.getRowSize(), index); diff --git a/core/src/test/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/naive/NaiveOperatorMemoryExecutorTest.java b/core/src/test/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/naive/NaiveOperatorMemoryExecutorTest.java index 43116ae09c..33b17da65d 100644 --- a/core/src/test/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/naive/NaiveOperatorMemoryExecutorTest.java +++ b/core/src/test/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/naive/NaiveOperatorMemoryExecutorTest.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.memory.execute.naive; diff --git a/core/src/test/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/StreamOperatorMemoryExecutorTest.java b/core/src/test/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/StreamOperatorMemoryExecutorTest.java index b59fca864b..d64f43bc5f 100644 --- a/core/src/test/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/StreamOperatorMemoryExecutorTest.java +++ b/core/src/test/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/stream/StreamOperatorMemoryExecutorTest.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.physical.memory.execute.stream; diff --git a/core/src/test/java/cn/edu/tsinghua/iginx/engine/physical/storage/utils/ColumnKeyTranslatorTest.java b/core/src/test/java/cn/edu/tsinghua/iginx/engine/physical/storage/utils/ColumnKeyTranslatorTest.java index edb1a30b64..bd10ccf245 100644 --- a/core/src/test/java/cn/edu/tsinghua/iginx/engine/physical/storage/utils/ColumnKeyTranslatorTest.java +++ b/core/src/test/java/cn/edu/tsinghua/iginx/engine/physical/storage/utils/ColumnKeyTranslatorTest.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.engine.physical.storage.utils; import cn.edu.tsinghua.iginx.engine.physical.storage.domain.ColumnKey; diff --git a/core/src/test/java/cn/edu/tsinghua/iginx/engine/shared/function/system/MaxTest.java b/core/src/test/java/cn/edu/tsinghua/iginx/engine/shared/function/system/MaxTest.java index 857b6f61f5..c4b3fedc91 100644 --- a/core/src/test/java/cn/edu/tsinghua/iginx/engine/shared/function/system/MaxTest.java +++ b/core/src/test/java/cn/edu/tsinghua/iginx/engine/shared/function/system/MaxTest.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.engine.shared.function.system; diff --git a/core/src/test/java/cn/edu/tsinghua/iginx/metadata/entity/ColumnsIntervalTest.java b/core/src/test/java/cn/edu/tsinghua/iginx/metadata/entity/ColumnsIntervalTest.java index eb37113512..2653ca2033 100644 --- a/core/src/test/java/cn/edu/tsinghua/iginx/metadata/entity/ColumnsIntervalTest.java +++ b/core/src/test/java/cn/edu/tsinghua/iginx/metadata/entity/ColumnsIntervalTest.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.metadata.entity; import static org.junit.Assert.*; diff --git a/core/src/test/java/cn/edu/tsinghua/iginx/metadata/entity/ExtraParamTest.java b/core/src/test/java/cn/edu/tsinghua/iginx/metadata/entity/ExtraParamTest.java index ee28df15c5..b8ea99bd67 100644 --- a/core/src/test/java/cn/edu/tsinghua/iginx/metadata/entity/ExtraParamTest.java +++ b/core/src/test/java/cn/edu/tsinghua/iginx/metadata/entity/ExtraParamTest.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.metadata.entity; import static cn.edu.tsinghua.iginx.metadata.utils.StorageEngineUtils.checkEmbeddedStorageExtraParams; diff --git a/core/src/test/java/cn/edu/tsinghua/iginx/notice/EmailNotifierTest.java b/core/src/test/java/cn/edu/tsinghua/iginx/notice/EmailNotifierTest.java index be4e4dbc77..f8b8ccdab8 100644 --- a/core/src/test/java/cn/edu/tsinghua/iginx/notice/EmailNotifierTest.java +++ b/core/src/test/java/cn/edu/tsinghua/iginx/notice/EmailNotifierTest.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.notice; import static org.junit.Assert.*; diff --git a/core/src/test/java/cn/edu/tsinghua/iginx/optimizer/TreePrinter.java b/core/src/test/java/cn/edu/tsinghua/iginx/optimizer/TreePrinter.java deleted file mode 100644 index 1264db082d..0000000000 --- a/core/src/test/java/cn/edu/tsinghua/iginx/optimizer/TreePrinter.java +++ /dev/null @@ -1,13 +0,0 @@ -package cn.edu.tsinghua.iginx.optimizer; - -import cn.edu.tsinghua.iginx.engine.shared.operator.Operator; -import cn.edu.tsinghua.iginx.engine.shared.operator.visitor.TreeInfoVisitor; - -public class TreePrinter { - - public static String getTreeInfo(Operator root) { - TreeInfoVisitor visitor = new TreeInfoVisitor(); - root.accept(visitor); - return visitor.getTreeInfo(); - } -} diff --git a/core/src/test/java/cn/edu/tsinghua/iginx/rest/ParseTest.java b/core/src/test/java/cn/edu/tsinghua/iginx/rest/ParseTest.java index 820d758641..f5e3b2e962 100644 --- a/core/src/test/java/cn/edu/tsinghua/iginx/rest/ParseTest.java +++ b/core/src/test/java/cn/edu/tsinghua/iginx/rest/ParseTest.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.rest; import static org.junit.Assert.assertEquals; diff --git a/core/src/test/java/cn/edu/tsinghua/iginx/sql/FilterVisitorTest.java b/core/src/test/java/cn/edu/tsinghua/iginx/sql/FilterVisitorTest.java index 20a028826d..00e07e5681 100644 --- a/core/src/test/java/cn/edu/tsinghua/iginx/sql/FilterVisitorTest.java +++ b/core/src/test/java/cn/edu/tsinghua/iginx/sql/FilterVisitorTest.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.AndFilter; diff --git a/core/src/test/java/cn/edu/tsinghua/iginx/sql/ParseTest.java b/core/src/test/java/cn/edu/tsinghua/iginx/sql/ParseTest.java index 3ccbc50bc0..023c056d7d 100644 --- a/core/src/test/java/cn/edu/tsinghua/iginx/sql/ParseTest.java +++ b/core/src/test/java/cn/edu/tsinghua/iginx/sql/ParseTest.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql; import static org.junit.Assert.assertEquals; @@ -70,7 +88,7 @@ public void testParseInsertWithSubQuery() { @Test public void testParseSelect() { String selectStr = - "SELECT SUM(c), SUM(d), SUM(e), COUNT(f), COUNT(g) FROM a.b WHERE 100 < key and key < 1000 or d == \"abc\" or \"666\" <= c or (e < 10 and not (f < 10)) OVER (RANGE 10 IN [200, 300));"; + "SELECT SUM(c), SUM(d), SUM(e), COUNT(f), COUNT(g) FROM a.b WHERE 100 < key and key < 1000 or d == \"abc\" or \"666\" <= c or (e < 10 and not (f < 10)) OVER WINDOW (size 10 IN [200, 300));"; UnarySelectStatement statement = (UnarySelectStatement) TestUtils.buildStatement(selectStr); assertTrue(statement.hasFunc()); @@ -122,7 +140,7 @@ public void testFilter() { @Test public void testParseGroupBy() { - String selectStr = "SELECT MAX(c) FROM a.b OVER (RANGE 10 IN [100, 1000));"; + String selectStr = "SELECT MAX(c) FROM a.b OVER WINDOW (size 10 IN [100, 1000));"; UnarySelectStatement statement = (UnarySelectStatement) TestUtils.buildStatement(selectStr); assertEquals(100, statement.getStartKey()); assertEquals(1000, statement.getEndKey()); @@ -148,7 +166,7 @@ public void testParseSpecialClause() { assertEquals(5, statement.getOffset()); assertEquals(10, statement.getLimit()); - String groupBy = "SELECT max(a) FROM test OVER (RANGE 5 IN (10, 120])"; + String groupBy = "SELECT max(a) FROM test OVER WINDOW (size 5 IN (10, 120])"; statement = (UnarySelectStatement) TestUtils.buildStatement(groupBy); assertEquals(11, statement.getStartKey()); @@ -156,7 +174,7 @@ public void testParseSpecialClause() { assertEquals(5L, statement.getPrecision()); String groupByAndLimit = - "SELECT max(a) FROM test OVER (RANGE 10 IN (10, 120)) LIMIT 5 OFFSET 2;"; + "SELECT max(a) FROM test OVER WINDOW (size 10 IN (10, 120)) LIMIT 5 OFFSET 2;"; statement = (UnarySelectStatement) TestUtils.buildStatement(groupByAndLimit); assertEquals(11, statement.getStartKey()); assertEquals(120, statement.getEndKey()); @@ -273,7 +291,7 @@ public void testParseTimeWithUnit() { assertEquals(expectedTimes, insertStatement.getKeys()); String queryStr = - "SELECT AVG(c) FROM a.b WHERE c > 10 AND c < 1ms OVER (RANGE 10 IN [1s, 2s));"; + "SELECT AVG(c) FROM a.b WHERE c > 10 AND c < 1ms OVER WINDOW (size 10 IN [1s, 2s));"; UnarySelectStatement selectStatement = (UnarySelectStatement) TestUtils.buildStatement(queryStr); diff --git a/core/src/test/java/cn/edu/tsinghua/iginx/sql/TestUtils.java b/core/src/test/java/cn/edu/tsinghua/iginx/sql/TestUtils.java index a3a2d81bc4..08cd7d2532 100644 --- a/core/src/test/java/cn/edu/tsinghua/iginx/sql/TestUtils.java +++ b/core/src/test/java/cn/edu/tsinghua/iginx/sql/TestUtils.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.sql; import cn.edu.tsinghua.iginx.engine.shared.operator.BinaryOperator; diff --git a/core/src/test/java/cn/edu/tsinghua/iginx/transform/TriggerTest.java b/core/src/test/java/cn/edu/tsinghua/iginx/transform/TriggerTest.java new file mode 100644 index 0000000000..c008dc85c5 --- /dev/null +++ b/core/src/test/java/cn/edu/tsinghua/iginx/transform/TriggerTest.java @@ -0,0 +1,354 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cn.edu.tsinghua.iginx.transform; + +import static org.junit.Assert.*; + +import cn.edu.tsinghua.iginx.transform.pojo.JobScheduleTriggerMaker; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Date; +import java.util.concurrent.TimeUnit; +import org.junit.Test; +import org.quartz.CronTrigger; +import org.quartz.DateBuilder; +import org.quartz.Trigger; +import org.quartz.impl.triggers.CalendarIntervalTriggerImpl; +import org.quartz.impl.triggers.SimpleTriggerImpl; + +public class TriggerTest { + + private static final SimpleDateFormat DATE_TIME_FORMAT = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + private static final SimpleDateFormat TIME_FORMAT = new SimpleDateFormat("HH:mm:ss"); + + private Trigger make(String schedule) { + return JobScheduleTriggerMaker.getTrigger(schedule); + } + + @Test + public void testEveryTrigger() { + String schedule = "every 3 second"; + Trigger trigger = make(schedule); + assertNotNull(trigger); + assertTrue(trigger instanceof SimpleTriggerImpl); + assertEquals(3000L, ((SimpleTriggerImpl) trigger).getRepeatInterval()); + + // minute + schedule = "every 10 minute"; + trigger = make(schedule); + assertNotNull(trigger); + assertTrue(trigger instanceof SimpleTriggerImpl); + assertEquals(600000L, ((SimpleTriggerImpl) trigger).getRepeatInterval()); + + // hour + schedule = "every 2 hour"; + trigger = make(schedule); + assertNotNull(trigger); + assertTrue(trigger instanceof SimpleTriggerImpl); + assertEquals(7200000L, ((SimpleTriggerImpl) trigger).getRepeatInterval()); + + // day + schedule = "every 1 day"; + trigger = make(schedule); + assertNotNull(trigger); + assertTrue(trigger instanceof SimpleTriggerImpl); + assertEquals(86400000L, ((SimpleTriggerImpl) trigger).getRepeatInterval()); + + // month + schedule = "every 1 month"; + trigger = make(schedule); + assertNotNull(trigger); + assertTrue(trigger instanceof CalendarIntervalTriggerImpl); + assertEquals(1, ((CalendarIntervalTriggerImpl) trigger).getRepeatInterval()); + assertEquals( + DateBuilder.IntervalUnit.MONTH, + ((CalendarIntervalTriggerImpl) trigger).getRepeatIntervalUnit()); + + // year + schedule = "every 1 year"; + trigger = make(schedule); + assertNotNull(trigger); + assertTrue(trigger instanceof CalendarIntervalTriggerImpl); + assertEquals(1, ((CalendarIntervalTriggerImpl) trigger).getRepeatInterval()); + assertEquals( + DateBuilder.IntervalUnit.YEAR, + ((CalendarIntervalTriggerImpl) trigger).getRepeatIntervalUnit()); + } + + @Test + public void testEveryTriggerWithDateTimeBounds() throws ParseException { + String schedule = "every 10 minute starts '2099-02-03 12:00:00' ends '2099-02-04 12:00:00'"; + Date expectedStartTime = DATE_TIME_FORMAT.parse("2099-02-03 12:00:00"); + Date expectedEndTime = DATE_TIME_FORMAT.parse("2099-02-04 12:00:00"); + + Trigger trigger = make(schedule); + assertNotNull(trigger); + assertTrue(trigger instanceof SimpleTriggerImpl); + assertEquals(600000L, ((SimpleTriggerImpl) trigger).getRepeatInterval()); + assertEquals(expectedStartTime, trigger.getStartTime()); + assertEquals(expectedEndTime, trigger.getEndTime()); + } + + @Test + public void testEveryTriggerWithTimeBounds() throws ParseException, InterruptedException { + String schedule = "every 5 second starts '23:59:53' ends '23:59:59'"; + + Calendar now = Calendar.getInstance(); + String today = new SimpleDateFormat("yyyy-MM-dd").format(new Date()); + + // wait till next day if startTime-3s is before current time, -3s to make sure we have enough + // time + // +5 to make sure it's next day + if (now.after(DATE_TIME_FORMAT.parse(today + " " + "23:59:50"))) { + long toNextDay = 24 * 60 * 60 * 1000 - now.getTimeInMillis() + 5; + TimeUnit.MILLISECONDS.sleep(toNextDay); + } + today = new SimpleDateFormat("yyyy-MM-dd").format(new Date()); + Date expectedStartTime = DATE_TIME_FORMAT.parse(today + " " + "23:59:53"); + Date expectedEndTime = DATE_TIME_FORMAT.parse(today + " " + "23:59:59"); + + Trigger trigger = make(schedule); + assertNotNull(trigger); + assertTrue(trigger instanceof SimpleTriggerImpl); + assertEquals(5000L, ((SimpleTriggerImpl) trigger).getRepeatInterval()); + assertEquals(expectedStartTime, trigger.getStartTime()); + assertEquals(expectedEndTime, trigger.getEndTime()); + } + + @Test + public void testEveryTriggerWithStartDateTimeBound() throws ParseException, InterruptedException { + // with date + String schedule = "every 10 minute starts '2099-02-03 12:00:00'"; + Date expectedStartTime = DATE_TIME_FORMAT.parse("2099-02-03 12:00:00"); + + Trigger trigger = make(schedule); + assertNotNull(trigger); + assertTrue(trigger instanceof SimpleTriggerImpl); + assertEquals(600000L, ((SimpleTriggerImpl) trigger).getRepeatInterval()); + assertEquals(expectedStartTime, trigger.getStartTime()); + assertNull(trigger.getEndTime()); + + // no date, may need to wait till next day + schedule = "every 5 second starts '23:59:58'"; + Calendar now = Calendar.getInstance(); + String today = new SimpleDateFormat("yyyy-MM-dd").format(new Date()); + if (now.after(DATE_TIME_FORMAT.parse(today + " 23:59:55"))) { + long toNextDay = 24 * 60 * 60 * 1000 - now.getTimeInMillis() + 5; + TimeUnit.MILLISECONDS.sleep(toNextDay); + } + today = new SimpleDateFormat("yyyy-MM-dd").format(new Date()); + expectedStartTime = DATE_TIME_FORMAT.parse(today + " " + "23:59:58"); + + trigger = make(schedule); + assertNotNull(trigger); + assertTrue(trigger instanceof SimpleTriggerImpl); + assertEquals(5000L, ((SimpleTriggerImpl) trigger).getRepeatInterval()); + assertEquals(expectedStartTime, trigger.getStartTime()); + assertNull(trigger.getEndTime()); + } + + @Test + public void testEveryTriggerWithEndDateTimeBound() throws ParseException, InterruptedException { + // with date + String schedule = "every 10 minute ends '2099-02-03 12:00:00'"; + Date expectedEndTime = DATE_TIME_FORMAT.parse("2099-02-03 12:00:00"); + + Trigger trigger = make(schedule); + assertNotNull(trigger); + assertTrue(trigger instanceof SimpleTriggerImpl); + assertEquals(600000L, ((SimpleTriggerImpl) trigger).getRepeatInterval()); + assertEquals(expectedEndTime, trigger.getEndTime()); + + long currentTime = System.currentTimeMillis(); + long triggerStartTime = trigger.getStartTime().getTime(); + + // Allow a tolerance of 1000 milliseconds, 500 would fail + long tolerance = 1000L; + + assertTrue( + "The trigger start time is not within the expected tolerance range.", + Math.abs(triggerStartTime - currentTime) <= tolerance); + + // no date, may need to wait till next day + schedule = "every 5 second ends '23:59:57'"; + Calendar now = Calendar.getInstance(); + String today = new SimpleDateFormat("yyyy-MM-dd").format(new Date()); + if (now.after(DATE_TIME_FORMAT.parse(today + " 23:59:57"))) { + long toNextDay = 24 * 60 * 60 * 1000 - now.getTimeInMillis() + 5; + TimeUnit.MILLISECONDS.sleep(toNextDay); + } + today = new SimpleDateFormat("yyyy-MM-dd").format(new Date()); + expectedEndTime = DATE_TIME_FORMAT.parse(today + " " + "23:59:57"); + + trigger = make(schedule); + assertNotNull(trigger); + assertTrue(trigger instanceof SimpleTriggerImpl); + assertEquals(5000L, ((SimpleTriggerImpl) trigger).getRepeatInterval()); + assertEquals(expectedEndTime, trigger.getEndTime()); + + currentTime = System.currentTimeMillis(); + triggerStartTime = trigger.getStartTime().getTime(); + + assertTrue( + "The trigger start time is not within the expected tolerance range.", + Math.abs(triggerStartTime - currentTime) <= tolerance); + } + + @Test + public void testAfterTrigger() { + // second + String schedule = "after 3 second"; + long currentTime = System.currentTimeMillis(); + Trigger trigger = make(schedule); + assertNotNull(trigger); + assertTrue(trigger instanceof SimpleTriggerImpl); + long triggerStartTime = trigger.getStartTime().getTime(); + long expectedStartTime = currentTime + 3000L; + + // Allow a tolerance of 1000 milliseconds + long tolerance = 1000L; + assertTrue( + "The trigger start time is not within the expected tolerance range.", + Math.abs(triggerStartTime - expectedStartTime) <= tolerance); + + // minute + schedule = "after 3 minute"; + currentTime = System.currentTimeMillis(); + trigger = make(schedule); + assertNotNull(trigger); + assertTrue(trigger instanceof SimpleTriggerImpl); + triggerStartTime = trigger.getStartTime().getTime(); + expectedStartTime = currentTime + 3 * 60 * 1000L; + assertTrue( + "The trigger start time is not within the expected tolerance range.", + Math.abs(triggerStartTime - expectedStartTime) <= tolerance); + + // hour + schedule = "after 3 hour"; + currentTime = System.currentTimeMillis(); + trigger = make(schedule); + assertNotNull(trigger); + assertTrue(trigger instanceof SimpleTriggerImpl); + triggerStartTime = trigger.getStartTime().getTime(); + expectedStartTime = currentTime + 3 * 60 * 60 * 1000L; + assertTrue( + "The trigger start time is not within the expected tolerance range.", + Math.abs(triggerStartTime - expectedStartTime) <= tolerance); + + // day + schedule = "after 3 day"; + currentTime = System.currentTimeMillis(); + trigger = make(schedule); + assertNotNull(trigger); + assertTrue(trigger instanceof SimpleTriggerImpl); + triggerStartTime = trigger.getStartTime().getTime(); + expectedStartTime = currentTime + 3 * 60 * 60 * 24 * 1000L; + assertTrue( + "The trigger start time is not within the expected tolerance range.", + Math.abs(triggerStartTime - expectedStartTime) <= tolerance); + + // month + schedule = "after 3 month"; + currentTime = System.currentTimeMillis(); + trigger = make(schedule); + assertNotNull(trigger); + assertTrue(trigger instanceof SimpleTriggerImpl); + + // Calculate the expected start time by adding 3 months to the current time + Calendar expectedStartCalendar = Calendar.getInstance(); + expectedStartCalendar.setTimeInMillis(currentTime); + int currentHour = expectedStartCalendar.get(Calendar.HOUR_OF_DAY); + int currentMinute = expectedStartCalendar.get(Calendar.MINUTE); + int currentSecond = expectedStartCalendar.get(Calendar.SECOND); + expectedStartCalendar.add(Calendar.MONTH, 3); + expectedStartCalendar.set(Calendar.HOUR_OF_DAY, currentHour); + expectedStartCalendar.set(Calendar.MINUTE, currentMinute); + expectedStartCalendar.set(Calendar.SECOND, currentSecond); + expectedStartTime = expectedStartCalendar.getTimeInMillis(); + triggerStartTime = trigger.getStartTime().getTime(); + assertTrue( + "The trigger start time is not within the expected tolerance range.", + Math.abs(triggerStartTime - expectedStartTime) <= tolerance); + + // year + schedule = "after 3 year"; + currentTime = System.currentTimeMillis(); + trigger = make(schedule); + assertNotNull(trigger); + assertTrue(trigger instanceof SimpleTriggerImpl); + + // Calculate the expected start time by adding 3 years to the current time + expectedStartCalendar = Calendar.getInstance(); + expectedStartCalendar.setTimeInMillis(currentTime); + currentHour = expectedStartCalendar.get(Calendar.HOUR_OF_DAY); + currentMinute = expectedStartCalendar.get(Calendar.MINUTE); + currentSecond = expectedStartCalendar.get(Calendar.SECOND); + expectedStartCalendar.add(Calendar.YEAR, 3); + expectedStartCalendar.set(Calendar.HOUR_OF_DAY, currentHour); + expectedStartCalendar.set(Calendar.MINUTE, currentMinute); + expectedStartCalendar.set(Calendar.SECOND, currentSecond); + expectedStartTime = expectedStartCalendar.getTimeInMillis(); + triggerStartTime = trigger.getStartTime().getTime(); + assertTrue( + "The trigger start time is not within the expected tolerance range.", + Math.abs(triggerStartTime - expectedStartTime) <= tolerance); + } + + @Test + public void testAtTrigger() throws ParseException, InterruptedException { + // with date + String schedule = "at '2099-12-31 23:59:59'"; + Trigger trigger = make(schedule); + Date atDate = DATE_TIME_FORMAT.parse("2099-12-31 23:59:59"); + assertNotNull(trigger); + assertTrue(trigger instanceof SimpleTriggerImpl); + assertEquals(atDate, trigger.getStartTime()); + + // no date, may need to wait till next day + schedule = "at '23:59:57'"; + Calendar now = Calendar.getInstance(); + String today = new SimpleDateFormat("yyyy-MM-dd").format(new Date()); + if (now.after(DATE_TIME_FORMAT.parse(today + " 23:59:57"))) { + long toNextDay = 24 * 60 * 60 * 1000 - now.getTimeInMillis() + 5; + TimeUnit.MILLISECONDS.sleep(toNextDay); + } + today = new SimpleDateFormat("yyyy-MM-dd").format(new Date()); + atDate = DATE_TIME_FORMAT.parse(today + " " + "23:59:57"); + trigger = make(schedule); + assertNotNull(trigger); + assertTrue(trigger instanceof SimpleTriggerImpl); + assertEquals(atDate, trigger.getStartTime()); + } + + @Test + public void testCronTrigger() { + String schedule = "(0 0/5 14,18 * * ?)"; + Trigger trigger = make(schedule); + assertNotNull(trigger); + assertTrue(trigger instanceof CronTrigger); + assertEquals("0 0/5 14,18 * * ?", ((CronTrigger) trigger).getCronExpression()); + } + + @Test + public void testInvalidSchedule() { + String schedule = "invalid schedule"; + assertThrows(IllegalArgumentException.class, () -> make(schedule)); + } +} diff --git a/core/src/test/java/cn/edu/tsinghua/iginx/transform/YamlReadTest.java b/core/src/test/java/cn/edu/tsinghua/iginx/transform/YamlReadTest.java index 00271a7a95..c4a0b065b4 100644 --- a/core/src/test/java/cn/edu/tsinghua/iginx/transform/YamlReadTest.java +++ b/core/src/test/java/cn/edu/tsinghua/iginx/transform/YamlReadTest.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.transform; diff --git a/dataSource/filesystem/pom.xml b/dataSource/filesystem/pom.xml new file mode 100644 index 0000000000..b864642567 --- /dev/null +++ b/dataSource/filesystem/pom.xml @@ -0,0 +1,47 @@ + + + + 4.0.0 + + + cn.edu.tsinghua + iginx-driver + ${revision} + + + filesystem + IGinX FileSystem + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + org.apache.maven.plugins + maven-antrun-plugin + + + + + diff --git a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/FileSystemStorage.java b/dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/FileSystemStorage.java similarity index 81% rename from dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/FileSystemStorage.java rename to dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/FileSystemStorage.java index 9c32e4fd9d..110daada32 100644 --- a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/FileSystemStorage.java +++ b/dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/FileSystemStorage.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.filesystem; @@ -27,11 +26,15 @@ import cn.edu.tsinghua.iginx.engine.physical.storage.domain.Column; import cn.edu.tsinghua.iginx.engine.physical.storage.domain.DataArea; import cn.edu.tsinghua.iginx.engine.physical.task.TaskExecuteResult; -import cn.edu.tsinghua.iginx.engine.shared.operator.*; +import cn.edu.tsinghua.iginx.engine.shared.operator.Delete; +import cn.edu.tsinghua.iginx.engine.shared.operator.Insert; +import cn.edu.tsinghua.iginx.engine.shared.operator.Project; +import cn.edu.tsinghua.iginx.engine.shared.operator.Select; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.AndFilter; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.KeyFilter; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Op; +import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; import cn.edu.tsinghua.iginx.filesystem.exec.Executor; import cn.edu.tsinghua.iginx.filesystem.exec.LocalExecutor; import cn.edu.tsinghua.iginx.filesystem.exec.RemoteExecutor; @@ -43,6 +46,7 @@ import cn.edu.tsinghua.iginx.utils.Pair; import java.util.Arrays; import java.util.List; +import java.util.Set; import org.apache.thrift.transport.TTransportException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -149,8 +153,9 @@ public TaskExecuteResult executeDelete(Delete delete, DataArea dataArea) { } @Override - public List getColumns() throws PhysicalException { - return executor.getColumnsOfStorageUnit(WILDCARD); + public List getColumns(Set patterns, TagFilter tagFilter) + throws PhysicalException { + return executor.getColumnsOfStorageUnit(WILDCARD, patterns, tagFilter); } @Override @@ -160,7 +165,7 @@ public Pair getBoundaryOfStorage(String prefix) } @Override - public synchronized void release() throws PhysicalException { + public synchronized void release() { executor.close(); if (thread != null) { thread.interrupt(); diff --git a/dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exception/FileSystemTaskExecuteFailureException.java b/dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exception/FileSystemTaskExecuteFailureException.java new file mode 100644 index 0000000000..c4d49acb99 --- /dev/null +++ b/dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exception/FileSystemTaskExecuteFailureException.java @@ -0,0 +1,31 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cn.edu.tsinghua.iginx.filesystem.exception; + +import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalTaskExecuteFailureException; + +public class FileSystemTaskExecuteFailureException extends PhysicalTaskExecuteFailureException { + + public FileSystemTaskExecuteFailureException(String message) { + super(message); + } + + public FileSystemTaskExecuteFailureException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exception/FilesystemException.java b/dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exception/FilesystemException.java new file mode 100644 index 0000000000..67965145aa --- /dev/null +++ b/dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exception/FilesystemException.java @@ -0,0 +1,36 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.filesystem.exception; + +import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; + +public class FilesystemException extends PhysicalException { + + public FilesystemException(String message) { + super(message); + } + + public FilesystemException(String message, Throwable cause) { + super(message, cause); + } + + public FilesystemException(Throwable cause) { + super(cause); + } +} diff --git a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/Executor.java b/dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/Executor.java similarity index 57% rename from dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/Executor.java rename to dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/Executor.java index 5c59a40c50..f83dc526fc 100644 --- a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/Executor.java +++ b/dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/Executor.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.filesystem.exec; import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; @@ -11,6 +29,7 @@ import cn.edu.tsinghua.iginx.metadata.entity.KeyInterval; import cn.edu.tsinghua.iginx.utils.Pair; import java.util.List; +import java.util.Set; public interface Executor { @@ -26,7 +45,8 @@ TaskExecuteResult executeProjectTask( TaskExecuteResult executeDeleteTask( List paths, List keyRanges, TagFilter tagFilter, String storageUnit); - List getColumnsOfStorageUnit(String storageUnit) throws PhysicalException; + List getColumnsOfStorageUnit( + String storageUnit, Set patterns, TagFilter tagFilter) throws PhysicalException; Pair getBoundaryOfStorage(String dataPrefix) throws PhysicalException; diff --git a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/FileSystemManager.java b/dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/FileSystemManager.java similarity index 91% rename from dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/FileSystemManager.java rename to dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/FileSystemManager.java index 436fca62d2..8e96d87998 100644 --- a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/FileSystemManager.java +++ b/dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/FileSystemManager.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.filesystem.exec; import static cn.edu.tsinghua.iginx.engine.logical.utils.PathUtils.MAX_CHAR; @@ -16,6 +34,7 @@ import cn.edu.tsinghua.iginx.filesystem.tools.MemoryPool; import cn.edu.tsinghua.iginx.thrift.DataType; import cn.edu.tsinghua.iginx.utils.Pair; +import cn.edu.tsinghua.iginx.utils.StringUtils; import java.io.File; import java.io.IOException; import java.nio.file.*; @@ -373,8 +392,16 @@ public FileVisitResult postVisitDirectory(Path dir, IOException exc) { .collect(Collectors.toList()); } - public List getAllFiles(File dir, boolean containsEmptyDir) { + public List getTargetFiles( + File dir, String root, String storageUnit, List patterns, boolean containsEmptyDir) { dir = FilePathUtils.normalize(dir, FileAccessType.READ); + List pathRegexList = new ArrayList<>(patterns.size()); + String suffix = storageUnit == null ? "" : "\\d+"; // 末尾匹配数字 + patterns.forEach( + p -> { + String pathPattern = FilePathUtils.toFilePath(root, storageUnit, p); + pathRegexList.add(StringUtils.reformatPath(pathPattern) + suffix); + }); List res = new ArrayList<>(); try { @@ -387,12 +414,18 @@ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) if (containsEmptyDir && isDirEmpty(dir)) { res.add(dir.toFile()); } + try (DirectoryStream stream = + Files.newDirectoryStream( + dir, + path -> + path.toFile().isFile() && FilePathUtils.matches(path, pathRegexList))) { + stream.forEach(path -> res.add(path.toFile())); + } return FileVisitResult.CONTINUE; } @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { - res.add(file.toFile()); return FileVisitResult.CONTINUE; } diff --git a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/LocalExecutor.java b/dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/LocalExecutor.java similarity index 86% rename from dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/LocalExecutor.java rename to dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/LocalExecutor.java index e7510fb036..03966d577c 100644 --- a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/LocalExecutor.java +++ b/dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/LocalExecutor.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.filesystem.exec; import static cn.edu.tsinghua.iginx.engine.logical.utils.LogicalFilterUtils.getKeyRangesFromFilter; @@ -7,6 +25,7 @@ import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; import cn.edu.tsinghua.iginx.engine.physical.memory.execute.stream.EmptyRowStream; import cn.edu.tsinghua.iginx.engine.physical.storage.domain.Column; +import cn.edu.tsinghua.iginx.engine.physical.storage.utils.TagKVUtils; import cn.edu.tsinghua.iginx.engine.physical.task.TaskExecuteResult; import cn.edu.tsinghua.iginx.engine.shared.KeyRange; import cn.edu.tsinghua.iginx.engine.shared.data.read.RowStream; @@ -34,6 +53,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Set; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -49,9 +69,9 @@ public class LocalExecutor implements Executor { private String prefix; - private boolean hasData; + private final boolean hasData; - private FileSystemManager fileSystemManager; + private final FileSystemManager fileSystemManager; public LocalExecutor(boolean isReadOnly, boolean hasData, Map extraParams) { String dir = extraParams.get(Constant.INIT_INFO_DIR); @@ -309,11 +329,17 @@ public TaskExecuteResult executeDeleteTask( } @Override - public List getColumnsOfStorageUnit(String storageUnit) throws PhysicalException { + public List getColumnsOfStorageUnit( + String storageUnit, Set patterns, TagFilter tagFilter) throws PhysicalException { List columns = new ArrayList<>(); + List patternList = new ArrayList<>(patterns); + if (patternList.isEmpty()) { + patternList.add("*"); + } if (root != null) { File directory = new File(FilePathUtils.toIginxPath(root, storageUnit, null)); - for (File file : fileSystemManager.getAllFiles(directory, false)) { + for (File file : + fileSystemManager.getTargetFiles(directory, root, storageUnit, patternList, false)) { FileMeta meta = fileSystemManager.getFileMeta(file); if (meta == null) { throw new PhysicalException( @@ -321,31 +347,30 @@ public List getColumnsOfStorageUnit(String storageUnit) throws PhysicalE "encounter error when getting columns of storage unit because file meta %s is null", file.getAbsolutePath())); } - columns.add( - new Column( - FilePathUtils.convertAbsolutePathToPath(root, file.getAbsolutePath(), storageUnit), - meta.getDataType(), - meta.getTags(), - false)); + // get columns by tag filter + if (tagFilter != null && !TagKVUtils.match(meta.getTags(), tagFilter)) { + continue; + } + String columnPath = + FilePathUtils.convertAbsolutePathToPath(root, file.getAbsolutePath(), storageUnit); + columns.add(new Column(columnPath, meta.getDataType(), meta.getTags(), false)); } } - if (hasData && dummyRoot != null) { - for (File file : fileSystemManager.getAllFiles(new File(realDummyRoot), true)) { - columns.add( - new Column( - FilePathUtils.convertAbsolutePathToPath( - dummyRoot, file.getAbsolutePath(), storageUnit), - DataType.BINARY, - null, - true)); + // get columns from dummy storage unit + if (hasData && dummyRoot != null && tagFilter == null) { + for (File file : + fileSystemManager.getTargetFiles( + new File(realDummyRoot), dummyRoot, null, patternList, true)) { + String dummyPath = + FilePathUtils.convertAbsolutePathToPath(dummyRoot, file.getAbsolutePath(), storageUnit); + columns.add(new Column(dummyPath, DataType.BINARY, null, true)); } } return columns; } @Override - public Pair getBoundaryOfStorage(String dataPrefix) - throws PhysicalException { + public Pair getBoundaryOfStorage(String dataPrefix) { KeyInterval keyInterval = KeyInterval.getDefaultKeyInterval(); ColumnsInterval columnsInterval; diff --git a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/RemoteExecutor.java b/dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/RemoteExecutor.java similarity index 86% rename from dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/RemoteExecutor.java rename to dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/RemoteExecutor.java index 32c9c31087..e209067994 100644 --- a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/RemoteExecutor.java +++ b/dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/RemoteExecutor.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.filesystem.exec; import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; @@ -13,10 +31,25 @@ import cn.edu.tsinghua.iginx.engine.shared.data.write.DataView; import cn.edu.tsinghua.iginx.engine.shared.data.write.RawDataType; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; -import cn.edu.tsinghua.iginx.engine.shared.operator.tag.*; +import cn.edu.tsinghua.iginx.engine.shared.operator.tag.AndTagFilter; +import cn.edu.tsinghua.iginx.engine.shared.operator.tag.BasePreciseTagFilter; +import cn.edu.tsinghua.iginx.engine.shared.operator.tag.BaseTagFilter; +import cn.edu.tsinghua.iginx.engine.shared.operator.tag.OrTagFilter; +import cn.edu.tsinghua.iginx.engine.shared.operator.tag.PreciseTagFilter; +import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; import cn.edu.tsinghua.iginx.filesystem.exception.FilesystemException; -import cn.edu.tsinghua.iginx.filesystem.thrift.*; +import cn.edu.tsinghua.iginx.filesystem.thrift.DeleteReq; +import cn.edu.tsinghua.iginx.filesystem.thrift.FSHeader; +import cn.edu.tsinghua.iginx.filesystem.thrift.FSKeyRange; +import cn.edu.tsinghua.iginx.filesystem.thrift.FSRawData; import cn.edu.tsinghua.iginx.filesystem.thrift.FileSystemService.Client; +import cn.edu.tsinghua.iginx.filesystem.thrift.GetBoundaryOfStorageResp; +import cn.edu.tsinghua.iginx.filesystem.thrift.GetColumnsOfStorageUnitResp; +import cn.edu.tsinghua.iginx.filesystem.thrift.InsertReq; +import cn.edu.tsinghua.iginx.filesystem.thrift.ProjectReq; +import cn.edu.tsinghua.iginx.filesystem.thrift.ProjectResp; +import cn.edu.tsinghua.iginx.filesystem.thrift.RawTagFilter; +import cn.edu.tsinghua.iginx.filesystem.thrift.Status; import cn.edu.tsinghua.iginx.filesystem.thrift.TagFilterType; import cn.edu.tsinghua.iginx.filesystem.tools.FilterTransformer; import cn.edu.tsinghua.iginx.metadata.entity.ColumnsInterval; @@ -32,6 +65,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; import org.apache.thrift.TException; import org.apache.thrift.protocol.TBinaryProtocol; @@ -207,11 +241,13 @@ public TaskExecuteResult executeDeleteTask( } @Override - public List getColumnsOfStorageUnit(String storageUnit) throws PhysicalException { + public List getColumnsOfStorageUnit( + String storageUnit, Set patterns, TagFilter tagFilter) throws PhysicalException { try { TTransport transport = thriftConnPool.borrowTransport(); Client client = new Client(new TBinaryProtocol(transport)); - GetColumnsOfStorageUnitResp resp = client.getColumnsOfStorageUnit(storageUnit); + GetColumnsOfStorageUnitResp resp = + client.getColumnsOfStorageUnit(storageUnit, patterns, constructRawTagFilter(tagFilter)); thriftConnPool.returnTransport(transport); List columns = new ArrayList<>(); resp.getPathList() @@ -254,6 +290,9 @@ public void close() { private RawTagFilter constructRawTagFilter(TagFilter tagFilter) { RawTagFilter filter = null; + if (tagFilter == null) { + return null; + } switch (tagFilter.getType()) { case Base: { diff --git a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/file/DefaultFileOperator.java b/dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/file/DefaultFileOperator.java similarity index 95% rename from dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/file/DefaultFileOperator.java rename to dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/file/DefaultFileOperator.java index d4a5b50e6f..dd74903eb9 100644 --- a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/file/DefaultFileOperator.java +++ b/dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/file/DefaultFileOperator.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.filesystem.file; import static cn.edu.tsinghua.iginx.filesystem.shared.Constant.*; diff --git a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/file/IFileOperator.java b/dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/file/IFileOperator.java similarity index 57% rename from dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/file/IFileOperator.java rename to dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/file/IFileOperator.java index 3294dfa815..8b12f5e267 100644 --- a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/file/IFileOperator.java +++ b/dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/file/IFileOperator.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.filesystem.file; import cn.edu.tsinghua.iginx.filesystem.file.entity.FileMeta; diff --git a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/file/entity/FileMeta.java b/dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/file/entity/FileMeta.java similarity index 57% rename from dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/file/entity/FileMeta.java rename to dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/file/entity/FileMeta.java index fb115db556..64ed8a3fb4 100644 --- a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/file/entity/FileMeta.java +++ b/dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/file/entity/FileMeta.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.filesystem.file.entity; import static cn.edu.tsinghua.iginx.filesystem.shared.Constant.MAGIC_NUMBER; diff --git a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/query/entity/FileSystemHistoryQueryRowStream.java b/dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/query/entity/FileSystemHistoryQueryRowStream.java similarity index 85% rename from dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/query/entity/FileSystemHistoryQueryRowStream.java rename to dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/query/entity/FileSystemHistoryQueryRowStream.java index e5550132fc..e7c2a969fc 100644 --- a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/query/entity/FileSystemHistoryQueryRowStream.java +++ b/dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/query/entity/FileSystemHistoryQueryRowStream.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.filesystem.query.entity; import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; diff --git a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/query/entity/FileSystemQueryRowStream.java b/dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/query/entity/FileSystemQueryRowStream.java similarity index 83% rename from dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/query/entity/FileSystemQueryRowStream.java rename to dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/query/entity/FileSystemQueryRowStream.java index 6639fa9cc2..53d6475f19 100644 --- a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/query/entity/FileSystemQueryRowStream.java +++ b/dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/query/entity/FileSystemQueryRowStream.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.filesystem.query.entity; import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; diff --git a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/query/entity/FileSystemResultTable.java b/dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/query/entity/FileSystemResultTable.java similarity index 59% rename from dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/query/entity/FileSystemResultTable.java rename to dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/query/entity/FileSystemResultTable.java index c1e2f659dd..cc349b5f35 100644 --- a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/query/entity/FileSystemResultTable.java +++ b/dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/query/entity/FileSystemResultTable.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.filesystem.query.entity; import cn.edu.tsinghua.iginx.thrift.DataType; diff --git a/dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/query/entity/Record.java b/dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/query/entity/Record.java new file mode 100644 index 0000000000..27686a6018 --- /dev/null +++ b/dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/query/entity/Record.java @@ -0,0 +1,51 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.filesystem.query.entity; + +import cn.edu.tsinghua.iginx.engine.shared.data.Value; +import cn.edu.tsinghua.iginx.thrift.DataType; + +public class Record { + + private final long key; + + private final Value value; + + public Record(long key, DataType dataType, Object rawData) { + this.key = key; + this.value = new Value(dataType, rawData); + } + + public Record(long key, Object rawData) { + this.key = key; + this.value = new Value(rawData); + } + + public Object getRawData() { + return value.getValue(); + } + + public long getKey() { + return key; + } + + public DataType getDataType() { + return value.getDataType(); + } +} diff --git a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/server/FileSystemServer.java b/dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/server/FileSystemServer.java similarity index 74% rename from dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/server/FileSystemServer.java rename to dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/server/FileSystemServer.java index 4b0d15a869..749a4653d0 100644 --- a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/server/FileSystemServer.java +++ b/dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/server/FileSystemServer.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.filesystem.server; import cn.edu.tsinghua.iginx.conf.Config; diff --git a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/server/FileSystemWorker.java b/dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/server/FileSystemWorker.java similarity index 88% rename from dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/server/FileSystemWorker.java rename to dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/server/FileSystemWorker.java index 73383ae362..91be1d5005 100644 --- a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/server/FileSystemWorker.java +++ b/dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/server/FileSystemWorker.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.filesystem.server; import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; @@ -19,9 +37,13 @@ import cn.edu.tsinghua.iginx.utils.DataTypeUtils; import cn.edu.tsinghua.iginx.utils.Pair; import java.nio.ByteBuffer; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; -import org.apache.thrift.TException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,7 +70,7 @@ public FileSystemWorker(Executor executor) { } @Override - public ProjectResp executeProject(ProjectReq req) throws TException { + public ProjectResp executeProject(ProjectReq req) { TagFilter tagFilter = resolveRawTagFilter(req.getTagFilter()); TaskExecuteResult result = executor.executeProjectTask( @@ -121,7 +143,7 @@ public ProjectResp executeProject(ProjectReq req) throws TException { } @Override - public Status executeInsert(InsertReq req) throws TException { + public Status executeInsert(InsertReq req) { FSRawData fsRawData = req.getRawData(); RawDataType rawDataType = strToRawDataType(fsRawData.getRawDataType()); if (rawDataType == null) { @@ -176,7 +198,7 @@ public Status executeInsert(InsertReq req) throws TException { } @Override - public Status executeDelete(DeleteReq req) throws TException { + public Status executeDelete(DeleteReq req) { TagFilter tagFilter = resolveRawTagFilter(req.getTagFilter()); // null keyRanges means delete key @@ -198,10 +220,12 @@ public Status executeDelete(DeleteReq req) throws TException { } @Override - public GetColumnsOfStorageUnitResp getColumnsOfStorageUnit(String storageUnit) throws TException { + public GetColumnsOfStorageUnitResp getColumnsOfStorageUnit( + String storageUnit, Set patterns, RawTagFilter tagFilter) { List ret = new ArrayList<>(); try { - List columns = executor.getColumnsOfStorageUnit(storageUnit); + List columns = + executor.getColumnsOfStorageUnit(storageUnit, patterns, resolveRawTagFilter(tagFilter)); columns.forEach( column -> { FSColumn fsColumn = @@ -221,7 +245,7 @@ public GetColumnsOfStorageUnitResp getColumnsOfStorageUnit(String storageUnit) t } @Override - public GetBoundaryOfStorageResp getBoundaryOfStorage(String prefix) throws TException { + public GetBoundaryOfStorageResp getBoundaryOfStorage(String prefix) { try { Pair pair = executor.getBoundaryOfStorage(prefix); GetBoundaryOfStorageResp resp = new GetBoundaryOfStorageResp(SUCCESS); diff --git a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/shared/Constant.java b/dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/shared/Constant.java similarity index 58% rename from dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/shared/Constant.java rename to dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/shared/Constant.java index 7cc5afefa7..28b28644ce 100644 --- a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/shared/Constant.java +++ b/dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/shared/Constant.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.filesystem.shared; import java.nio.charset.Charset; diff --git a/dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/shared/FileType.java b/dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/shared/FileType.java new file mode 100644 index 0000000000..0691c8b659 --- /dev/null +++ b/dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/shared/FileType.java @@ -0,0 +1,26 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.filesystem.shared; + +public enum FileType { + IGINX_FILE, + NORMAL_FILE, + UNKNOWN_FILE, + DIR +} diff --git a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/tools/FilePathUtils.java b/dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/tools/FilePathUtils.java similarity index 68% rename from dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/tools/FilePathUtils.java rename to dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/tools/FilePathUtils.java index 95a04f610d..df3a10bb46 100644 --- a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/tools/FilePathUtils.java +++ b/dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/tools/FilePathUtils.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.filesystem.tools; import static cn.edu.tsinghua.iginx.filesystem.shared.Constant.*; @@ -7,8 +25,10 @@ import cn.edu.tsinghua.iginx.auth.utils.FilePermissionRuleNameFilters; import java.io.File; import java.nio.file.Path; +import java.util.List; import java.util.Optional; import java.util.function.Predicate; +import java.util.regex.Pattern; public class FilePathUtils { @@ -102,4 +122,35 @@ public static String convertAbsolutePathToPath(String root, String filePath, Str return res.substring(0, res.length() - 1); } } + + public static String toFilePath(String root, String storageUnit, String path) { + if (path == null) { + return root; + } + StringBuilder target = new StringBuilder(root); + if (storageUnit != null) { + target.append(storageUnit).append(SEPARATOR); + } + String[] parts = path.split("\\."); + StringBuilder res = new StringBuilder(); + for (String s : parts) { + s = s.replace("\\", "."); + res.append(s).append(SEPARATOR); + } + target.append(res.substring(0, res.length() - 1)); + if (storageUnit != null) { + target.append(FILE_EXTENSION); + } + return target.toString(); + } + + public static boolean matches(Path path, List regexList) { + String filePath = path.toAbsolutePath().toString(); + for (String regex : regexList) { + if (Pattern.matches(regex, filePath)) { + return true; + } + } + return false; + } } diff --git a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/tools/FilterTransformer.java b/dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/tools/FilterTransformer.java similarity index 90% rename from dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/tools/FilterTransformer.java rename to dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/tools/FilterTransformer.java index bbc33239c4..fb42abaff0 100644 --- a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/tools/FilterTransformer.java +++ b/dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/tools/FilterTransformer.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.filesystem.tools; import static cn.edu.tsinghua.iginx.engine.shared.operator.filter.Op.*; diff --git a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/tools/LimitedSizeMap.java b/dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/tools/LimitedSizeMap.java similarity index 54% rename from dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/tools/LimitedSizeMap.java rename to dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/tools/LimitedSizeMap.java index 298c6ba840..068d501a56 100644 --- a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/tools/LimitedSizeMap.java +++ b/dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/tools/LimitedSizeMap.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.filesystem.tools; import java.util.LinkedHashMap; diff --git a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/tools/MemoryPool.java b/dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/tools/MemoryPool.java similarity index 65% rename from dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/tools/MemoryPool.java rename to dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/tools/MemoryPool.java index 62af9b6a95..77ee08d6df 100644 --- a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/tools/MemoryPool.java +++ b/dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/tools/MemoryPool.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.filesystem.tools; import java.util.Queue; diff --git a/dataSource/influxdb/pom.xml b/dataSource/influxdb/pom.xml new file mode 100644 index 0000000000..4d598e043e --- /dev/null +++ b/dataSource/influxdb/pom.xml @@ -0,0 +1,54 @@ + + + + 4.0.0 + + + cn.edu.tsinghua + iginx-driver + ${revision} + + + influxdb + IGinX InfluxDB + + + + com.influxdb + influxdb-client-java + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + org.apache.maven.plugins + maven-antrun-plugin + + + + + diff --git a/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/InfluxDBStorage.java b/dataSource/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/InfluxDBStorage.java similarity index 88% rename from dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/InfluxDBStorage.java rename to dataSource/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/InfluxDBStorage.java index ece91da52a..5d2a6f4d06 100644 --- a/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/InfluxDBStorage.java +++ b/dataSource/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/InfluxDBStorage.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.influxdb; @@ -27,6 +26,7 @@ import cn.edu.tsinghua.iginx.engine.physical.storage.IStorage; import cn.edu.tsinghua.iginx.engine.physical.storage.domain.Column; import cn.edu.tsinghua.iginx.engine.physical.storage.domain.DataArea; +import cn.edu.tsinghua.iginx.engine.physical.storage.utils.TagKVUtils; import cn.edu.tsinghua.iginx.engine.physical.task.TaskExecuteResult; import cn.edu.tsinghua.iginx.engine.shared.KeyRange; import cn.edu.tsinghua.iginx.engine.shared.data.write.BitmapView; @@ -37,7 +37,15 @@ import cn.edu.tsinghua.iginx.engine.shared.operator.Insert; import cn.edu.tsinghua.iginx.engine.shared.operator.Project; import cn.edu.tsinghua.iginx.engine.shared.operator.Select; -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.*; +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.AndFilter; +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.BoolFilter; +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.FilterType; +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.NotFilter; +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Op; +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.OrFilter; +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.PathFilter; +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.ValueFilter; import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilterType; import cn.edu.tsinghua.iginx.influxdb.exception.InfluxDBException; @@ -48,7 +56,9 @@ import cn.edu.tsinghua.iginx.influxdb.tools.FilterTransformer; import cn.edu.tsinghua.iginx.influxdb.tools.SchemaTransformer; import cn.edu.tsinghua.iginx.influxdb.tools.TagFilterUtils; -import cn.edu.tsinghua.iginx.metadata.entity.*; +import cn.edu.tsinghua.iginx.metadata.entity.ColumnsInterval; +import cn.edu.tsinghua.iginx.metadata.entity.KeyInterval; +import cn.edu.tsinghua.iginx.metadata.entity.StorageEngineMeta; import cn.edu.tsinghua.iginx.thrift.DataType; import cn.edu.tsinghua.iginx.thrift.StorageEngineType; import cn.edu.tsinghua.iginx.utils.Pair; @@ -66,7 +76,12 @@ import java.time.Instant; import java.time.OffsetDateTime; import java.time.ZoneId; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -95,6 +110,9 @@ public class InfluxDBStorage implements IStorage { private static final String SHOW_TIME_SERIES = "from(bucket:\"%s\") |> range(start: time(v: 0), stop: time(v: 9223372036854775807)) |> filter(fn: (r) => (r._measurement =~ /.*/ and r._field =~ /.+/)) |> first()"; + private static final String SHOW_TIME_SERIES_BY_PATTERN = + "from(bucket:\"%s\") |> range(start: time(v: 0), stop: time(v: 9223372036854775807)) |> filter(fn: (r) => (r._measurement =~ /%s/ and r._field =~ /%s/)) |> first()"; + private final StorageEngineMeta meta; private final InfluxDBClient client; @@ -113,7 +131,7 @@ public InfluxDBStorage(StorageEngineMeta meta) throws StorageInitializationExcep throw new StorageInitializationException("unexpected database: " + meta.getStorageEngine()); } if (!testConnection()) { - throw new StorageInitializationException("cannot connect to " + meta.toString()); + throw new StorageInitializationException("cannot connect to " + meta); } Map extraParams = meta.getExtraParams(); String url = extraParams.getOrDefault("url", "http://localhost:8086/"); @@ -230,9 +248,8 @@ private String findExtremeRecordPath( } @Override - public List getColumns() { + public List getColumns(Set patterns, TagFilter tagFilter) { List timeseries = new ArrayList<>(); - for (Bucket bucket : client.getBucketsApi().findBucketsByOrgName(organization.getName())) { // get all the bucket // query all the series by querying all the data with first() @@ -242,13 +259,87 @@ public List getColumns() { boolean isDummy = meta.isHasData() && (meta.getDataPrefix() == null - || bucket.getName().startsWith(meta.getDataPrefix())); + || bucket + .getName() + .startsWith( + meta.getDataPrefix().substring(0, meta.getDataPrefix().indexOf(".")))); if (bucket.getType() == Bucket.TypeEnum.SYSTEM || (!isUnit && !isDummy)) { continue; } - String statement = String.format(SHOW_TIME_SERIES, bucket.getName()); - List tables = client.getQueryApi().query(statement, organization.getId()); + List tables = new ArrayList<>(); + String measPattern, fieldPattern, statement, bucketPattern; + // + List> patternPairs = new ArrayList<>(); + + if (patterns == null + || patterns.size() == 0 + || patterns.contains("*") + || patterns.contains("*.*")) { + statement = String.format(SHOW_TIME_SERIES, bucket.getName()); + tables = client.getQueryApi().query(statement, organization.getId()); + } else { + boolean thisBucketIsQueried = false; + for (String p : patterns) { + if (isDummy && !isUnit) { + bucketPattern = p.substring(0, p.indexOf(".")); + // dummy path starts with . + if (!Pattern.matches(StringUtils.reformatPath(bucketPattern), bucket.getName())) { + continue; + } + // * can match multiple layers. + if (p.startsWith("*.")) { + // match one layer first + p = p.substring(2); + if (p.contains(".")) { + // pattern *.xx.xx + patternPairs.add( + new Pair<>( + StringUtils.reformatPath(p.substring(0, p.indexOf("."))), + StringUtils.reformatPath(p.substring(p.indexOf(".") + 1)))); + } + // match multiple layers later. + p = "*.*." + p; + } + // remove . part from pattern + p = p.substring(p.indexOf(".") + 1); + } + thisBucketIsQueried = true; + + if (p.startsWith("*.")) { + // match one layer first + patternPairs.add( + new Pair<>( + StringUtils.reformatPath(p.substring(0, p.indexOf("."))), + StringUtils.reformatPath(p.substring(p.indexOf(".") + 1)))); + // match multiple layers + patternPairs.add( + new Pair<>(StringUtils.reformatPath("*"), StringUtils.reformatPath(p))); + } else if (p.equals("*")) { + patternPairs.add( + new Pair<>(StringUtils.reformatPath("*"), StringUtils.reformatPath("*"))); + } else if (p.contains(".")) { + patternPairs.add( + new Pair<>( + StringUtils.reformatPath(p.substring(0, p.indexOf("."))), + StringUtils.reformatPath(p.substring(p.indexOf(".") + 1)))); + } + for (Pair patternPair : patternPairs) { + measPattern = patternPair.k; + fieldPattern = patternPair.v; + // query time series based on pattern + statement = + String.format( + SHOW_TIME_SERIES_BY_PATTERN, bucket.getName(), measPattern, fieldPattern); + LOGGER.info("executing column query: {}", statement); + tables.addAll(client.getQueryApi().query(statement, organization.getId())); + } + } + // if bucket is dummy && all patterns do not match(.*), move on to next bucket + if (!thisBucketIsQueried) { + continue; + } + } for (FluxTable table : tables) { List column = table.getColumns(); @@ -263,6 +354,13 @@ public List getColumns() { String val = (String) table.getRecords().get(0).getValues().get(key); tag.put(key, val); } + if (isDummy && !isUnit) { + path = bucket.getName() + "." + path; + } + // get columns by tag filter + if (tagFilter != null && !TagKVUtils.match(tag, tagFilter)) { + continue; + } DataType dataType; switch (column.get(5).getDataType()) { // the index 1 is the type of the data @@ -289,10 +387,7 @@ public List getColumns() { LOGGER.warn("DataType don't match and default is String"); break; } - if (isDummy && !isUnit) { - path = bucket.getName() + "." + path; - } - timeseries.add(new Column(path, dataType, tag)); + timeseries.add(new Column(path, dataType, tag, isDummy)); } } @@ -300,7 +395,7 @@ public List getColumns() { } @Override - public void release() throws PhysicalException { + public void release() { client.close(); } diff --git a/dataSource/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/exception/InfluxDBException.java b/dataSource/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/exception/InfluxDBException.java new file mode 100644 index 0000000000..7b29d0e133 --- /dev/null +++ b/dataSource/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/exception/InfluxDBException.java @@ -0,0 +1,36 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.influxdb.exception; + +import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; + +public class InfluxDBException extends PhysicalException { + + public InfluxDBException(String message) { + super(message); + } + + public InfluxDBException(String message, Throwable cause) { + super(message, cause); + } + + public InfluxDBException(Throwable cause) { + super(cause); + } +} diff --git a/dataSource/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/exception/InfluxDBTaskExecuteFailureException.java b/dataSource/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/exception/InfluxDBTaskExecuteFailureException.java new file mode 100644 index 0000000000..2cc741fff8 --- /dev/null +++ b/dataSource/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/exception/InfluxDBTaskExecuteFailureException.java @@ -0,0 +1,31 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cn.edu.tsinghua.iginx.influxdb.exception; + +import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalTaskExecuteFailureException; + +public class InfluxDBTaskExecuteFailureException extends PhysicalTaskExecuteFailureException { + + public InfluxDBTaskExecuteFailureException(String message) { + super(message); + } + + public InfluxDBTaskExecuteFailureException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/query/entity/InfluxDBHistoryQueryRowStream.java b/dataSource/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/query/entity/InfluxDBHistoryQueryRowStream.java similarity index 86% rename from dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/query/entity/InfluxDBHistoryQueryRowStream.java rename to dataSource/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/query/entity/InfluxDBHistoryQueryRowStream.java index cbb85dcf6f..46d5fb6adf 100644 --- a/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/query/entity/InfluxDBHistoryQueryRowStream.java +++ b/dataSource/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/query/entity/InfluxDBHistoryQueryRowStream.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.influxdb.query.entity; diff --git a/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/query/entity/InfluxDBQueryRowStream.java b/dataSource/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/query/entity/InfluxDBQueryRowStream.java similarity index 92% rename from dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/query/entity/InfluxDBQueryRowStream.java rename to dataSource/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/query/entity/InfluxDBQueryRowStream.java index 913db4e56b..c6eeefa5b2 100644 --- a/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/query/entity/InfluxDBQueryRowStream.java +++ b/dataSource/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/query/entity/InfluxDBQueryRowStream.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.influxdb.query.entity; diff --git a/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/query/entity/InfluxDBSchema.java b/dataSource/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/query/entity/InfluxDBSchema.java similarity index 68% rename from dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/query/entity/InfluxDBSchema.java rename to dataSource/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/query/entity/InfluxDBSchema.java index 86d6e0b95b..7dec918481 100644 --- a/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/query/entity/InfluxDBSchema.java +++ b/dataSource/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/query/entity/InfluxDBSchema.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.influxdb.query.entity; diff --git a/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/tools/DataTypeTransformer.java b/dataSource/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/tools/DataTypeTransformer.java similarity index 51% rename from dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/tools/DataTypeTransformer.java rename to dataSource/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/tools/DataTypeTransformer.java index a56dd54e6f..fe4d60953a 100644 --- a/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/tools/DataTypeTransformer.java +++ b/dataSource/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/tools/DataTypeTransformer.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.influxdb.tools; import static cn.edu.tsinghua.iginx.thrift.DataType.BINARY; diff --git a/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/tools/FilterTransformer.java b/dataSource/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/tools/FilterTransformer.java similarity index 82% rename from dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/tools/FilterTransformer.java rename to dataSource/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/tools/FilterTransformer.java index ba080f671e..f4c4891e0c 100644 --- a/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/tools/FilterTransformer.java +++ b/dataSource/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/tools/FilterTransformer.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.influxdb.tools; diff --git a/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/tools/SchemaTransformer.java b/dataSource/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/tools/SchemaTransformer.java similarity index 78% rename from dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/tools/SchemaTransformer.java rename to dataSource/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/tools/SchemaTransformer.java index 217b9c6783..cf652fa953 100644 --- a/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/tools/SchemaTransformer.java +++ b/dataSource/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/tools/SchemaTransformer.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.influxdb.tools; import static cn.edu.tsinghua.iginx.influxdb.tools.DataTypeTransformer.fromInfluxDB; diff --git a/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/tools/TagFilterUtils.java b/dataSource/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/tools/TagFilterUtils.java similarity index 84% rename from dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/tools/TagFilterUtils.java rename to dataSource/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/tools/TagFilterUtils.java index aa478f6253..d75f35efbe 100644 --- a/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/tools/TagFilterUtils.java +++ b/dataSource/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/tools/TagFilterUtils.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.influxdb.tools; import cn.edu.tsinghua.iginx.engine.shared.operator.tag.*; diff --git a/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/tools/TimeUtils.java b/dataSource/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/tools/TimeUtils.java similarity index 50% rename from dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/tools/TimeUtils.java rename to dataSource/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/tools/TimeUtils.java index 10a3f9e018..c1a036ff34 100644 --- a/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/tools/TimeUtils.java +++ b/dataSource/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/tools/TimeUtils.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.influxdb.tools; import java.time.Instant; diff --git a/dataSources/iotdb12/pom.xml b/dataSource/iotdb12/pom.xml similarity index 53% rename from dataSources/iotdb12/pom.xml rename to dataSource/iotdb12/pom.xml index 79d58ebd33..d471263235 100644 --- a/dataSources/iotdb12/pom.xml +++ b/dataSource/iotdb12/pom.xml @@ -1,4 +1,23 @@ + 4.0.0 @@ -12,13 +31,6 @@ iotdb12 IGinX IoTDB12 - - .. - iotdb12 - 8 - 8 - - org.apache.iotdb @@ -61,46 +73,9 @@ org.apache.maven.plugins maven-assembly-plugin - - org.apache.maven.plugins - maven-dependency-plugin - 2.10 - - - copy-dependencies - - copy-dependencies - - package - - ../../core/target/iginx-core-${project.version}/driver/iotdb12/ - provided - - - - org.apache.maven.plugins maven-antrun-plugin - 1.7 - - - copy-native-libraries - - run - - package - - - - - - - - - - - diff --git a/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/IoTDBStorage.java b/dataSource/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/IoTDBStorage.java similarity index 89% rename from dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/IoTDBStorage.java rename to dataSource/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/IoTDBStorage.java index 74be91517d..0215cb19b5 100644 --- a/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/IoTDBStorage.java +++ b/dataSource/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/IoTDBStorage.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.iotdb; @@ -39,7 +38,13 @@ import cn.edu.tsinghua.iginx.engine.shared.operator.Insert; import cn.edu.tsinghua.iginx.engine.shared.operator.Project; import cn.edu.tsinghua.iginx.engine.shared.operator.Select; -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.*; +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.AndFilter; +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.KeyFilter; +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.NotFilter; +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Op; +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.OrFilter; +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.ValueFilter; import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; import cn.edu.tsinghua.iginx.iotdb.exception.IoTDBException; import cn.edu.tsinghua.iginx.iotdb.exception.IoTDBTaskExecuteFailureException; @@ -47,12 +52,22 @@ import cn.edu.tsinghua.iginx.iotdb.tools.DataViewWrapper; import cn.edu.tsinghua.iginx.iotdb.tools.FilterTransformer; import cn.edu.tsinghua.iginx.iotdb.tools.TagKVUtils; -import cn.edu.tsinghua.iginx.metadata.entity.*; +import cn.edu.tsinghua.iginx.metadata.entity.ColumnsInterval; +import cn.edu.tsinghua.iginx.metadata.entity.KeyInterval; +import cn.edu.tsinghua.iginx.metadata.entity.StorageEngineMeta; import cn.edu.tsinghua.iginx.thrift.DataType; import cn.edu.tsinghua.iginx.thrift.StorageEngineType; import cn.edu.tsinghua.iginx.utils.Pair; import cn.edu.tsinghua.iginx.utils.StringUtils; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -96,7 +111,11 @@ public class IoTDBStorage implements IStorage { private static final String DELETE_TIMESERIES_CLAUSE = "DELETE TIMESERIES %s"; - private static final String SHOW_TIMESERIES = "SHOW TIMESERIES"; + private static final String SHOW_TIMESERIES_DUMMY = "SHOW TIMESERIES root.%s"; + + private static final String SHOW_TIMESERIES = "SHOW TIMESERIES root.*.%s"; + + private static final String SHOW_TIMESERIES_ALL = "SHOW TIMESERIES"; private static final String DOES_NOT_EXISTED = "does not exist"; @@ -114,7 +133,7 @@ public IoTDBStorage(StorageEngineMeta meta) throws StorageInitializationExceptio throw new StorageInitializationException("unexpected database: " + meta.getStorageEngine()); } if (!testConnection()) { - throw new StorageInitializationException("cannot connect to " + meta.toString()); + throw new StorageInitializationException("cannot connect to " + meta); } sessionPool = createSessionPool(); } @@ -155,7 +174,7 @@ public Pair getBoundaryOfStorage(String dataPrefix ColumnsInterval columnsInterval; try { if (dataPrefix == null || dataPrefix.isEmpty()) { - dataSet = sessionPool.executeQueryStatement(SHOW_TIMESERIES); + dataSet = sessionPool.executeQueryStatement(SHOW_TIMESERIES_ALL); while (dataSet.hasNext()) { record = dataSet.next(); if (record == null || record.getFields().size() < 4) { @@ -187,21 +206,25 @@ record = dataSet.next(); } @Override - public void release() throws PhysicalException { + public void release() { sessionPool.close(); } @Override - public List getColumns() throws PhysicalException { + public List getColumns(Set patterns, TagFilter tagFilter) + throws PhysicalException { List columns = new ArrayList<>(); - getColumns2StorageUnit(columns, null); + getColumns2StorageUnit(columns, null, patterns, tagFilter); return columns; } - private void getColumns2StorageUnit(List columns, Map columns2StorageUnit) + private void getColumnsFromDataSet( + List columns, + Map columns2StorageUnit, + TagFilter tagFilter, + SessionDataSetWrapper dataSet) throws PhysicalException { try { - SessionDataSetWrapper dataSet = sessionPool.executeQueryStatement(SHOW_TIMESERIES); while (dataSet.hasNext()) { RowRecord record = dataSet.next(); if (record == null || record.getFields().size() < 4) { @@ -221,6 +244,10 @@ private void getColumns2StorageUnit(List columns, Map co if (columns2StorageUnit != null) { columns2StorageUnit.put(pair.k, fragment); } + // get columns by tag filter + if (tagFilter != null && !TagKVUtils.match(pair.v, tagFilter)) { + continue; + } switch (dataTypeName) { case "BOOLEAN": @@ -243,7 +270,40 @@ private void getColumns2StorageUnit(List columns, Map co break; } } - dataSet.close(); + } catch (IoTDBConnectionException | StatementExecutionException e) { + throw new IoTDBTaskExecuteFailureException("get time series failure: ", e); + } + } + + private void getColumns2StorageUnit( + List columns, + Map columns2StorageUnit, + Set patterns, + TagFilter tagFilter) + throws PhysicalException { + try { + Iterator iterator = patterns.iterator(); + do { + String pattern = iterator.hasNext() ? iterator.next() : null; + SessionDataSetWrapper dataSet; + LOGGER.debug("get time series: {}", pattern); + if (pattern != null) { + LOGGER.debug("do show timeseries: {}", pattern); + dataSet = sessionPool.executeQueryStatement(String.format(SHOW_TIMESERIES, pattern)); + getColumnsFromDataSet(columns, columns2StorageUnit, tagFilter, dataSet); + dataSet.close(); + + dataSet = + sessionPool.executeQueryStatement(String.format(SHOW_TIMESERIES_DUMMY, pattern)); + getColumnsFromDataSet(columns, columns2StorageUnit, tagFilter, dataSet); + dataSet.close(); + } else { + LOGGER.debug("do show all timeseries"); + dataSet = sessionPool.executeQueryStatement(SHOW_TIMESERIES_ALL); + getColumnsFromDataSet(columns, columns2StorageUnit, tagFilter, dataSet); + dataSet.close(); + } + } while (iterator.hasNext()); } catch (IoTDBConnectionException | StatementExecutionException e) { throw new IoTDBTaskExecuteFailureException("get time series failure: ", e); } @@ -301,8 +361,7 @@ private TaskExecuteResult executeProjectWithFilter( builder.append(','); } String statement = - String.format( - QUERY_DATA, builder.deleteCharAt(builder.length() - 1).toString(), storageUnit); + String.format(QUERY_DATA, builder.deleteCharAt(builder.length() - 1), storageUnit); String filterStr = getFilterString(filter, storageUnit); if (!filterStr.isEmpty()) { @@ -358,7 +417,7 @@ private TaskExecuteResult executeProjectDummyWithFilter(Project project, Filter builder.append(','); } String statement = - String.format(QUERY_HISTORY_DATA, builder.deleteCharAt(builder.length() - 1).toString()); + String.format(QUERY_HISTORY_DATA, builder.deleteCharAt(builder.length() - 1)); String filterStr = getFilterString(filter, ""); if (!filterStr.isEmpty()) { @@ -790,7 +849,7 @@ private List determineDeletePathList(String storageUnit, Delete delete) } else { List patterns = delete.getPatterns(); TagFilter tagFilter = delete.getTagFilter(); - List timeSeries = getColumns(); + List timeSeries = getColumns(new HashSet<>(), null); List pathList = new ArrayList<>(); for (Column ts : timeSeries) { @@ -809,7 +868,7 @@ private List determineDeletePathList(String storageUnit, Delete delete) private List determinePathList(String storageUnit, List patterns) throws IoTDBConnectionException, StatementExecutionException { Set pathSet = new HashSet<>(); - String showColumns = SHOW_TIMESERIES; + String showColumns = SHOW_TIMESERIES_ALL; showColumns = storageUnit == null ? showColumns : showColumns + " " + PREFIX + storageUnit; SessionDataSetWrapper dataSet = sessionPool.executeQueryStatement(showColumns); while (dataSet.hasNext()) { @@ -859,7 +918,7 @@ private String getFilterString(Filter filter, String storageUnit) throws Physica if (filterStr.contains("*")) { List columns = new ArrayList<>(); Map columns2Fragment = new HashMap<>(); - getColumns2StorageUnit(columns, columns2Fragment); + getColumns2StorageUnit(columns, columns2Fragment, new HashSet<>(), null); filterStr = FilterTransformer.toString( expandFilterWildcard(filter.copy(), columns, columns2Fragment, storageUnit)); diff --git a/dataSource/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/exception/IoTDBException.java b/dataSource/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/exception/IoTDBException.java new file mode 100644 index 0000000000..a838b0fc21 --- /dev/null +++ b/dataSource/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/exception/IoTDBException.java @@ -0,0 +1,32 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.iotdb.exception; + +import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; + +public class IoTDBException extends PhysicalException { + + public IoTDBException(String message) { + super(message); + } + + public IoTDBException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/dataSource/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/exception/IoTDBTaskExecuteFailureException.java b/dataSource/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/exception/IoTDBTaskExecuteFailureException.java new file mode 100644 index 0000000000..4bdb056ee5 --- /dev/null +++ b/dataSource/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/exception/IoTDBTaskExecuteFailureException.java @@ -0,0 +1,32 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.iotdb.exception; + +import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalTaskExecuteFailureException; + +public class IoTDBTaskExecuteFailureException extends PhysicalTaskExecuteFailureException { + + public IoTDBTaskExecuteFailureException(String message) { + super(message); + } + + public IoTDBTaskExecuteFailureException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/query/entity/IoTDBQueryRowStream.java b/dataSource/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/query/entity/IoTDBQueryRowStream.java similarity index 88% rename from dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/query/entity/IoTDBQueryRowStream.java rename to dataSource/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/query/entity/IoTDBQueryRowStream.java index ad4e0baf79..c7427655de 100644 --- a/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/query/entity/IoTDBQueryRowStream.java +++ b/dataSource/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/query/entity/IoTDBQueryRowStream.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.iotdb.query.entity; diff --git a/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/tools/DataTypeTransformer.java b/dataSource/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/tools/DataTypeTransformer.java similarity index 69% rename from dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/tools/DataTypeTransformer.java rename to dataSource/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/tools/DataTypeTransformer.java index d8c3f14272..bdf312acda 100644 --- a/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/tools/DataTypeTransformer.java +++ b/dataSource/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/tools/DataTypeTransformer.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.iotdb.tools; diff --git a/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/tools/DataViewWrapper.java b/dataSource/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/tools/DataViewWrapper.java similarity index 62% rename from dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/tools/DataViewWrapper.java rename to dataSource/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/tools/DataViewWrapper.java index 5ae6578e65..7746f7c039 100644 --- a/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/tools/DataViewWrapper.java +++ b/dataSource/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/tools/DataViewWrapper.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.iotdb.tools; diff --git a/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/tools/FilterTransformer.java b/dataSource/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/tools/FilterTransformer.java similarity index 75% rename from dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/tools/FilterTransformer.java rename to dataSource/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/tools/FilterTransformer.java index b948bef5ef..27d1b8322c 100644 --- a/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/tools/FilterTransformer.java +++ b/dataSource/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/tools/FilterTransformer.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.iotdb.tools; diff --git a/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/tools/TagKVUtils.java b/dataSource/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/tools/TagKVUtils.java similarity index 86% rename from dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/tools/TagKVUtils.java rename to dataSource/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/tools/TagKVUtils.java index 1893703f46..18fa42916a 100644 --- a/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/tools/TagKVUtils.java +++ b/dataSource/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/tools/TagKVUtils.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.iotdb.tools; diff --git a/dataSource/mongodb/pom.xml b/dataSource/mongodb/pom.xml new file mode 100644 index 0000000000..ba3347602c --- /dev/null +++ b/dataSource/mongodb/pom.xml @@ -0,0 +1,54 @@ + + + + 4.0.0 + + + cn.edu.tsinghua + iginx-driver + ${revision} + + + mongodb + IGinX MongoDB + + + + org.mongodb + mongodb-driver + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + org.apache.maven.plugins + maven-antrun-plugin + + + + + diff --git a/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/MongoDBStorage.java b/dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/MongoDBStorage.java similarity index 86% rename from dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/MongoDBStorage.java rename to dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/MongoDBStorage.java index 2dde3d2ca8..b323f292e8 100644 --- a/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/MongoDBStorage.java +++ b/dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/MongoDBStorage.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.mongodb; import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; @@ -36,17 +54,25 @@ import cn.edu.tsinghua.iginx.thrift.DataType; import cn.edu.tsinghua.iginx.thrift.StorageEngineType; import cn.edu.tsinghua.iginx.utils.Pair; -import com.mongodb.MongoBulkWriteException; -import com.mongodb.MongoClientSettings; -import com.mongodb.ServerAddress; -import com.mongodb.WriteError; +import com.mongodb.*; import com.mongodb.client.MongoClient; import com.mongodb.client.MongoClients; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoDatabase; -import com.mongodb.client.model.*; +import com.mongodb.client.model.BulkWriteOptions; +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.InsertManyOptions; +import com.mongodb.client.model.ReplaceOneModel; +import com.mongodb.client.model.ReplaceOptions; +import com.mongodb.client.model.WriteModel; import java.text.ParseException; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import org.bson.BsonDocument; @@ -64,6 +90,7 @@ public class MongoDBStorage implements IStorage { private static final int SESSION_POOL_MAX_SIZE = 200; public static final String VALUE_FIELD = "v"; public static final String[] SYSTEM_DBS = new String[] {"admin", "config", "local"}; + public static final String CONNECTION_STRING = "uri"; public static final String SCHEMA_SAMPLE_SIZE = "schema.sample.size"; public static final String QUERY_SAMPLE_SIZE = "dummy.sample.size"; public static final String SCHEMA_SAMPLE_SIZE_DEFAULT = "1000"; @@ -79,6 +106,10 @@ public MongoDBStorage(StorageEngineMeta meta) throws StorageInitializationExcept throw new StorageInitializationException("unexpected database: " + meta.getStorageEngine()); } + String defaultConnection = String.format("mongodb://%s:%d", meta.getIp(), meta.getPort()); + String connectionString = + meta.getExtraParams().getOrDefault(CONNECTION_STRING, defaultConnection); + String sampleSize = meta.getExtraParams().getOrDefault(SCHEMA_SAMPLE_SIZE, SCHEMA_SAMPLE_SIZE_DEFAULT); this.schemaSampleSize = Integer.parseInt(sampleSize); @@ -88,7 +119,7 @@ public MongoDBStorage(StorageEngineMeta meta) throws StorageInitializationExcept this.querySampleSize = Integer.parseInt(querySampleSize); try { - this.client = connect(meta.getIp(), meta.getPort()); + this.client = connect(connectionString); } catch (Exception e) { String message = "fail to connect " + meta.getIp() + ":" + meta.getPort(); LOGGER.error(message, e); @@ -96,17 +127,16 @@ public MongoDBStorage(StorageEngineMeta meta) throws StorageInitializationExcept } } - private MongoClient connect(String ip, int port) { - ServerAddress address = new ServerAddress(ip, port); + private MongoClient connect(String connectionString) { MongoClientSettings settings = MongoClientSettings.builder() - .applyToClusterSettings(builder -> builder.hosts(Collections.singletonList(address))) .applyToConnectionPoolSettings( builder -> builder .maxWaitTime(MAX_WAIT_TIME, TimeUnit.SECONDS) .maxSize(SESSION_POOL_MAX_SIZE) .maxConnectionIdleTime(60, TimeUnit.SECONDS)) + .applyConnectionString(new ConnectionString(connectionString)) .build(); return MongoClients.create(settings); @@ -295,7 +325,11 @@ private static long getDuplicateKey(WriteError error) { } @Override - public List getColumns() { + public List getColumns(Set patterns, TagFilter tagFilter) { + List patternList = new ArrayList<>(patterns); + if (patternList.isEmpty()) { + patternList.add("*"); + } List columns = new ArrayList<>(); for (String dbName : getDatabaseNames(this.client)) { MongoDatabase db = this.client.getDatabase(dbName); @@ -303,7 +337,9 @@ public List getColumns() { try { if (dbName.startsWith("unit")) { Field field = NameUtils.parseCollectionName(collectionName); - columns.add(new Column(field.getName(), field.getType(), field.getTags(), false)); + if (NameUtils.match(field.getName(), field.getTags(), patternList, tagFilter)) { + columns.add(new Column(field.getName(), field.getType(), field.getTags(), false)); + } continue; } } catch (Exception ignored) { @@ -315,7 +351,9 @@ public List getColumns() { Map sampleSchema = new SchemaSample(schemaSampleSize).query(collection, true); for (Map.Entry entry : sampleSchema.entrySet()) { - columns.add(new Column(entry.getKey(), entry.getValue(), null, true)); + if (NameUtils.match(entry.getKey(), Collections.emptyMap(), patternList, null)) { + columns.add(new Column(entry.getKey(), entry.getValue(), null, true)); + } } continue; } diff --git a/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/DummyQuery.java b/dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/DummyQuery.java similarity index 88% rename from dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/DummyQuery.java rename to dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/DummyQuery.java index 1588ec5fe5..938835248b 100644 --- a/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/DummyQuery.java +++ b/dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/DummyQuery.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.mongodb.dummy; import cn.edu.tsinghua.iginx.engine.shared.data.read.RowStream; diff --git a/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/FilterUtils.java b/dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/FilterUtils.java similarity index 88% rename from dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/FilterUtils.java rename to dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/FilterUtils.java index 07e05b8004..0c24841be6 100644 --- a/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/FilterUtils.java +++ b/dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/FilterUtils.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.mongodb.dummy; import static com.mongodb.client.model.Filters.*; diff --git a/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/FindRowStream.java b/dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/FindRowStream.java similarity index 79% rename from dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/FindRowStream.java rename to dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/FindRowStream.java index 3c985187e4..a9ec690449 100644 --- a/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/FindRowStream.java +++ b/dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/FindRowStream.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.mongodb.dummy; import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; diff --git a/dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/NameUtils.java b/dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/NameUtils.java new file mode 100644 index 0000000000..840e07de9e --- /dev/null +++ b/dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/NameUtils.java @@ -0,0 +1,37 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.mongodb.dummy; + +class NameUtils { + static String getSuffix(String path, String prefix) { + if (path.startsWith(prefix + ".")) { + return path.substring(prefix.length() + 1); + } + throw new IllegalArgumentException(prefix + " is not prefix of " + path); + } + + static boolean containNumberNode(String path) { + for (String node : path.split("\\.")) { + if (node.chars().allMatch(Character::isDigit)) { + return true; + } + } + return false; + } +} diff --git a/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/PathTree.java b/dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/PathTree.java similarity index 78% rename from dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/PathTree.java rename to dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/PathTree.java index 0a82168510..59f8d6747f 100644 --- a/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/PathTree.java +++ b/dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/PathTree.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.mongodb.dummy; import cn.edu.tsinghua.iginx.mongodb.tools.NameUtils; diff --git a/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/QueryRowStream.java b/dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/QueryRowStream.java similarity index 78% rename from dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/QueryRowStream.java rename to dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/QueryRowStream.java index 646dafa8fb..beeb79c25c 100644 --- a/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/QueryRowStream.java +++ b/dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/QueryRowStream.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.mongodb.dummy; import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; diff --git a/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/QueryUtils.java b/dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/QueryUtils.java similarity index 86% rename from dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/QueryUtils.java rename to dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/QueryUtils.java index e0970e238b..d6221925e2 100644 --- a/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/QueryUtils.java +++ b/dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/QueryUtils.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.mongodb.dummy; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; diff --git a/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/ResultColumn.java b/dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/ResultColumn.java similarity index 82% rename from dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/ResultColumn.java rename to dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/ResultColumn.java index 600a5acddc..c197c94e40 100644 --- a/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/ResultColumn.java +++ b/dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/ResultColumn.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.mongodb.dummy; import cn.edu.tsinghua.iginx.thrift.DataType; diff --git a/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/ResultRow.java b/dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/ResultRow.java similarity index 77% rename from dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/ResultRow.java rename to dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/ResultRow.java index 7c6ae252e7..2b231a841e 100644 --- a/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/ResultRow.java +++ b/dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/ResultRow.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.mongodb.dummy; import static java.util.AbstractMap.SimpleImmutableEntry; diff --git a/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/ResultTable.java b/dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/ResultTable.java similarity index 86% rename from dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/ResultTable.java rename to dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/ResultTable.java index 46b8471def..21514ffeeb 100644 --- a/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/ResultTable.java +++ b/dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/ResultTable.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.mongodb.dummy; import cn.edu.tsinghua.iginx.thrift.DataType; diff --git a/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/SampleQuery.java b/dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/SampleQuery.java similarity index 86% rename from dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/SampleQuery.java rename to dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/SampleQuery.java index e7c338a3a2..21feb5a5e6 100644 --- a/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/SampleQuery.java +++ b/dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/SampleQuery.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.mongodb.dummy; import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; diff --git a/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/SchemaSample.java b/dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/SchemaSample.java similarity index 74% rename from dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/SchemaSample.java rename to dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/SchemaSample.java index 50c5393ceb..d9da918fd3 100644 --- a/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/SchemaSample.java +++ b/dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/SchemaSample.java @@ -1,17 +1,19 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.mongodb.dummy; diff --git a/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/TypeUtils.java b/dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/TypeUtils.java similarity index 93% rename from dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/TypeUtils.java rename to dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/TypeUtils.java index 1ae75248a4..1644289403 100644 --- a/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/TypeUtils.java +++ b/dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/TypeUtils.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.mongodb.dummy; import cn.edu.tsinghua.iginx.engine.shared.data.Value; diff --git a/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/entity/ColumnQuery.java b/dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/entity/ColumnQuery.java similarity index 81% rename from dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/entity/ColumnQuery.java rename to dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/entity/ColumnQuery.java index 4b8e9addc6..b06a9d6c50 100644 --- a/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/entity/ColumnQuery.java +++ b/dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/entity/ColumnQuery.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.mongodb.entity; import cn.edu.tsinghua.iginx.engine.shared.data.read.Field; diff --git a/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/entity/JoinQuery.java b/dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/entity/JoinQuery.java similarity index 88% rename from dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/entity/JoinQuery.java rename to dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/entity/JoinQuery.java index 23eff1d5a0..5dff770071 100644 --- a/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/entity/JoinQuery.java +++ b/dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/entity/JoinQuery.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.mongodb.entity; import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; diff --git a/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/entity/SourceTable.java b/dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/entity/SourceTable.java similarity index 80% rename from dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/entity/SourceTable.java rename to dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/entity/SourceTable.java index 402a32516a..871bc179db 100644 --- a/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/entity/SourceTable.java +++ b/dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/entity/SourceTable.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.mongodb.entity; import cn.edu.tsinghua.iginx.engine.shared.data.read.Field; diff --git a/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/tools/FilterUtils.java b/dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/tools/FilterUtils.java similarity index 95% rename from dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/tools/FilterUtils.java rename to dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/tools/FilterUtils.java index 35b119712e..d8d6d2491c 100644 --- a/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/tools/FilterUtils.java +++ b/dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/tools/FilterUtils.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.mongodb.tools; import static com.mongodb.client.model.Filters.*; diff --git a/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/tools/NameUtils.java b/dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/tools/NameUtils.java similarity index 68% rename from dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/tools/NameUtils.java rename to dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/tools/NameUtils.java index 5871b82b20..b56b44b2d2 100644 --- a/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/tools/NameUtils.java +++ b/dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/tools/NameUtils.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.mongodb.tools; import cn.edu.tsinghua.iginx.engine.physical.storage.domain.ColumnKey; @@ -63,19 +81,26 @@ public static List match( Iterable fieldList, Iterable patterns, TagFilter tagFilter) { List fields = new ArrayList<>(); for (Field field : fieldList) { - if (tagFilter != null && !TagKVUtils.match(field.getTags(), tagFilter)) { - continue; - } - for (String pattern : patterns) { - if (Pattern.matches(StringUtils.reformatPath(pattern), field.getName())) { - fields.add(field); - break; - } + if (match(field.getName(), field.getTags(), patterns, tagFilter)) { + fields.add(field); } } return fields; } + public static boolean match( + String columnName, Map tags, Iterable patterns, TagFilter tagFilter) { + if (tagFilter != null && !TagKVUtils.match(tags, tagFilter)) { + return false; + } + for (String pattern : patterns) { + if (Pattern.matches(StringUtils.reformatPath(pattern), columnName)) { + return true; + } + } + return false; + } + public static boolean isWildcard(String node) { return node.contains("*"); } diff --git a/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/tools/TypeUtils.java b/dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/tools/TypeUtils.java similarity index 76% rename from dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/tools/TypeUtils.java rename to dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/tools/TypeUtils.java index 970afa472e..7feae6169e 100644 --- a/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/tools/TypeUtils.java +++ b/dataSource/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/tools/TypeUtils.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.mongodb.tools; import cn.edu.tsinghua.iginx.thrift.DataType; diff --git a/dataSources/mongodb/src/test/java/cn/edu/tsinghua/iginx/mongodb/dummy/TypeUtilsTest.java b/dataSource/mongodb/src/test/java/cn/edu/tsinghua/iginx/mongodb/dummy/TypeUtilsTest.java similarity index 78% rename from dataSources/mongodb/src/test/java/cn/edu/tsinghua/iginx/mongodb/dummy/TypeUtilsTest.java rename to dataSource/mongodb/src/test/java/cn/edu/tsinghua/iginx/mongodb/dummy/TypeUtilsTest.java index 33b7364ce9..01249ee43c 100644 --- a/dataSources/mongodb/src/test/java/cn/edu/tsinghua/iginx/mongodb/dummy/TypeUtilsTest.java +++ b/dataSource/mongodb/src/test/java/cn/edu/tsinghua/iginx/mongodb/dummy/TypeUtilsTest.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.mongodb.dummy; import org.bson.*; diff --git a/dataSources/parquet/pom.xml b/dataSource/parquet/pom.xml similarity index 52% rename from dataSources/parquet/pom.xml rename to dataSource/parquet/pom.xml index ab783e5dd5..8fdeeef642 100644 --- a/dataSources/parquet/pom.xml +++ b/dataSource/parquet/pom.xml @@ -1,4 +1,23 @@ + 4.0.0 @@ -11,13 +30,6 @@ parquet IGinX Parquet - - .. - parquet - 8 - 8 - - cn.edu.tsinghua.iginx @@ -66,47 +78,9 @@ org.apache.maven.plugins maven-assembly-plugin - - org.apache.maven.plugins - maven-dependency-plugin - 2.10 - - - copy-dependencies - - copy-dependencies - - package - - ../../core/target/iginx-core-${project.version}/driver/parquet/ - provided - junit,hamcrest-core - - - - org.apache.maven.plugins maven-antrun-plugin - 1.7 - - - copy-native-libraries - - run - - package - - - - - - - - - - - diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/ParquetStorage.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/ParquetStorage.java similarity index 63% rename from dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/ParquetStorage.java rename to dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/ParquetStorage.java index d39b62d931..6379069ba8 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/ParquetStorage.java +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/ParquetStorage.java @@ -1,17 +1,19 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.parquet; @@ -24,15 +26,23 @@ import cn.edu.tsinghua.iginx.engine.physical.storage.domain.Column; import cn.edu.tsinghua.iginx.engine.physical.storage.domain.DataArea; import cn.edu.tsinghua.iginx.engine.physical.task.TaskExecuteResult; +import cn.edu.tsinghua.iginx.engine.shared.function.Function; +import cn.edu.tsinghua.iginx.engine.shared.function.FunctionCall; +import cn.edu.tsinghua.iginx.engine.shared.function.FunctionParams; +import cn.edu.tsinghua.iginx.engine.shared.function.FunctionType; import cn.edu.tsinghua.iginx.engine.shared.operator.Delete; import cn.edu.tsinghua.iginx.engine.shared.operator.Insert; import cn.edu.tsinghua.iginx.engine.shared.operator.Project; import cn.edu.tsinghua.iginx.engine.shared.operator.Select; +import cn.edu.tsinghua.iginx.engine.shared.operator.SetTransform; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.AndFilter; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.KeyFilter; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Op; -import cn.edu.tsinghua.iginx.metadata.entity.*; +import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; +import cn.edu.tsinghua.iginx.metadata.entity.ColumnsInterval; +import cn.edu.tsinghua.iginx.metadata.entity.KeyInterval; +import cn.edu.tsinghua.iginx.metadata.entity.StorageEngineMeta; import cn.edu.tsinghua.iginx.parquet.exec.Executor; import cn.edu.tsinghua.iginx.parquet.exec.LocalExecutor; import cn.edu.tsinghua.iginx.parquet.exec.RemoteExecutor; @@ -44,6 +54,7 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.thrift.transport.TTransportException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -111,7 +122,12 @@ public TaskExecuteResult executeProject(Project project, DataArea dataArea) { new KeyFilter(Op.GE, keyInterval.getStartKey()), new KeyFilter(Op.L, keyInterval.getEndKey()))); return executor.executeProjectTask( - project.getPatterns(), project.getTagFilter(), filter, dataArea.getStorageUnit(), false); + project.getPatterns(), + project.getTagFilter(), + filter, + null, + dataArea.getStorageUnit(), + false); } @Override @@ -123,7 +139,12 @@ public TaskExecuteResult executeProjectDummy(Project project, DataArea dataArea) new KeyFilter(Op.GE, keyInterval.getStartKey()), new KeyFilter(Op.L, keyInterval.getEndKey()))); return executor.executeProjectTask( - project.getPatterns(), project.getTagFilter(), filter, dataArea.getStorageUnit(), true); + project.getPatterns(), + project.getTagFilter(), + filter, + null, + dataArea.getStorageUnit(), + true); } @Override @@ -142,7 +163,12 @@ public TaskExecuteResult executeProjectWithSelect( new KeyFilter(Op.L, keyInterval.getEndKey()), select.getFilter())); return executor.executeProjectTask( - project.getPatterns(), project.getTagFilter(), filter, dataArea.getStorageUnit(), false); + project.getPatterns(), + project.getTagFilter(), + filter, + null, + dataArea.getStorageUnit(), + false); } @Override @@ -156,7 +182,57 @@ public TaskExecuteResult executeProjectDummyWithSelect( new KeyFilter(Op.L, keyInterval.getEndKey()), select.getFilter())); return executor.executeProjectTask( - project.getPatterns(), project.getTagFilter(), filter, dataArea.getStorageUnit(), true); + project.getPatterns(), + project.getTagFilter(), + filter, + null, + dataArea.getStorageUnit(), + true); + } + + @Override + public boolean isSupportProjectWithSetTransform(SetTransform setTransform, DataArea dataArea) { + // just push down in full column fragment + KeyInterval keyInterval = dataArea.getKeyInterval(); + if (keyInterval.getStartKey() > 0 || keyInterval.getEndKey() < Long.MAX_VALUE) { + return false; + } + + // just push down count(*) for now + List functionCalls = setTransform.getFunctionCallList(); + if (functionCalls.size() != 1) { + return false; + } + FunctionCall functionCall = functionCalls.get(0); + Function function = functionCall.getFunction(); + FunctionParams params = functionCall.getParams(); + if (function.getFunctionType() != FunctionType.System) { + return false; + } + if (!function.getIdentifier().equals("count")) { + return false; + } + if (params.getPaths().size() != 1) { + return false; + } + String path = params.getPaths().get(0); + return path.equals("*") || path.equals("*.*"); + } + + @Override + public TaskExecuteResult executeProjectWithSetTransform( + Project project, SetTransform setTransform, DataArea dataArea) { + if (!isSupportProjectWithSetTransform(setTransform, dataArea)) { + throw new IllegalArgumentException("unsupported set transform"); + } + + return executor.executeProjectTask( + project.getPatterns(), + project.getTagFilter(), + null, + setTransform.getFunctionCallList(), + dataArea.getStorageUnit(), + false); } @Override @@ -174,8 +250,9 @@ public TaskExecuteResult executeInsert(Insert insert, DataArea dataArea) { } @Override - public List getColumns() throws PhysicalException { - return executor.getColumnsOfStorageUnit("*"); + public List getColumns(Set patterns, TagFilter tagFilter) + throws PhysicalException { + return executor.getColumnsOfStorageUnit("*", patterns, tagFilter); } @Override diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/Database.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/Database.java new file mode 100644 index 0000000000..9881bb8476 --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/Database.java @@ -0,0 +1,51 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.parquet.db; + +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; +import cn.edu.tsinghua.iginx.parquet.db.util.AreaSet; +import cn.edu.tsinghua.iginx.parquet.db.util.iterator.Scanner; +import cn.edu.tsinghua.iginx.parquet.util.exception.StorageException; +import cn.edu.tsinghua.iginx.thrift.DataType; +import com.google.common.collect.RangeSet; +import java.io.IOException; +import java.util.Map; +import java.util.Set; +import org.apache.arrow.vector.types.pojo.Field; + +public interface Database extends AutoCloseable { + + Scanner> query( + Set fields, RangeSet ranges, Filter filter) throws StorageException, IOException; + + Map count(Set strings) + throws InterruptedException, IOException, StorageException; + + Set schema() throws StorageException; + + void upsertRows(Scanner> scanner, Map schema) + throws StorageException, InterruptedException; + + void upsertColumns(Scanner> scanner, Map schema) + throws StorageException, InterruptedException; + + void delete(AreaSet areas) throws StorageException; + + void clear() throws StorageException; +} diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/OneTierDB.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/OneTierDB.java new file mode 100644 index 0000000000..76001ecbee --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/OneTierDB.java @@ -0,0 +1,228 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.parquet.db.lsm; + +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; +import cn.edu.tsinghua.iginx.parquet.db.Database; +import cn.edu.tsinghua.iginx.parquet.db.lsm.api.ReadWriter; +import cn.edu.tsinghua.iginx.parquet.db.lsm.buffer.DataBuffer; +import cn.edu.tsinghua.iginx.parquet.db.lsm.buffer.MemTableQueue; +import cn.edu.tsinghua.iginx.parquet.db.lsm.buffer.chunk.Chunk; +import cn.edu.tsinghua.iginx.parquet.db.lsm.compact.Flusher; +import cn.edu.tsinghua.iginx.parquet.db.lsm.table.TableStorage; +import cn.edu.tsinghua.iginx.parquet.db.util.AreaSet; +import cn.edu.tsinghua.iginx.parquet.db.util.WriteBatches; +import cn.edu.tsinghua.iginx.parquet.db.util.iterator.BatchPlaneScanner; +import cn.edu.tsinghua.iginx.parquet.db.util.iterator.Scanner; +import cn.edu.tsinghua.iginx.parquet.manager.utils.TagKVUtils; +import cn.edu.tsinghua.iginx.parquet.util.NoexceptAutoCloseable; +import cn.edu.tsinghua.iginx.parquet.util.NoexceptAutoCloseables; +import cn.edu.tsinghua.iginx.parquet.util.Shared; +import cn.edu.tsinghua.iginx.parquet.util.arrow.ArrowFields; +import cn.edu.tsinghua.iginx.parquet.util.exception.StorageException; +import cn.edu.tsinghua.iginx.parquet.util.exception.StorageRuntimeException; +import cn.edu.tsinghua.iginx.parquet.util.exception.TypeConflictedException; +import cn.edu.tsinghua.iginx.thrift.DataType; +import com.google.common.collect.Range; +import com.google.common.collect.RangeSet; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.stream.Collectors; +import javax.annotation.WillClose; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.util.AutoCloseables; +import org.apache.arrow.vector.types.pojo.Field; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class OneTierDB implements Database { + private static final Logger LOGGER = LoggerFactory.getLogger(OneTierDB.class); + + private final ReadWriteLock lock = new ReentrantReadWriteLock(true); + private final String name; + private final Shared shared; + private final BufferAllocator allocator; + private final TableStorage tableStorage; + private final MemTableQueue memTableQueue; + private final Flusher flusher; + + public OneTierDB(String name, Shared shared, ReadWriter readerWriter) throws IOException { + this.name = name; + this.shared = shared; + this.allocator = shared.getAllocator().newChildAllocator(name, 0, Long.MAX_VALUE); + this.tableStorage = new TableStorage(shared, readerWriter); + this.memTableQueue = new MemTableQueue(shared, allocator); + this.flusher = new Flusher(name, shared, allocator, memTableQueue, tableStorage); + } + + @Override + public Scanner> query( + Set fields, RangeSet ranges, Filter filter) + throws IOException, StorageException { + Set innerFields = + fields.stream() + .map(field -> TagKVUtils.toFullName(ArrowFields.toColumnKey(field))) + .collect(Collectors.toSet()); + + lock.readLock().lock(); + try { + List>> inMemories = + memTableQueue.scan(new ArrayList<>(fields), ranges, allocator); + try (AutoCloseable c = AutoCloseables.all(inMemories)) { + DataBuffer readBuffer = + tableStorage.query(innerFields, ranges, filter); + for (Scanner> scanner : inMemories) { + readBuffer.putRows(scanner); + } + return readBuffer.scanRows(innerFields, Range.all()); + } catch (Exception e) { + throw new IllegalStateException(e); + } + } finally { + lock.readLock().unlock(); + } + } + + @Override + public Map count(Set fields) + throws InterruptedException, IOException, StorageException { + lock.readLock().lock(); + try { + // to simplify the implementation, we flush the memTableQueue before counting + memTableQueue.compact(); + memTableQueue.flush(); + Set innerFields = + fields.stream() + .map(field -> TagKVUtils.toFullName(ArrowFields.toColumnKey(field))) + .collect(Collectors.toSet()); + return tableStorage.count(innerFields); + } finally { + lock.readLock().unlock(); + } + } + + @Override + public Set schema() throws StorageException { + lock.readLock().lock(); + try { + Map types = tableStorage.schema(); + return ArrowFields.of(types); + } finally { + lock.readLock().unlock(); + } + } + + @Override + public void upsertRows( + Scanner> scanner, Map schema) + throws StorageException, InterruptedException { + try (Scanner>> batchScanner = + new BatchPlaneScanner<>(scanner, shared.getStorageProperties().getWriteBatchSize())) { + while (batchScanner.iterate()) { + try (Scanner> batch = batchScanner.value()) { + putAll(WriteBatches.recordOfRows(batch, schema, allocator), schema); + } + } + } + } + + @Override + public void upsertColumns( + Scanner> scanner, Map schema) + throws StorageException, InterruptedException { + try (Scanner>> batchScanner = + new BatchPlaneScanner<>(scanner, shared.getStorageProperties().getWriteBatchSize())) { + while (batchScanner.iterate()) { + try (Scanner> batch = batchScanner.value()) { + putAll(WriteBatches.recordOfColumns(batch, schema, allocator), schema); + } + } + } + } + + private void putAll(@WillClose Iterable chunks, Map schema) + throws TypeConflictedException, InterruptedException { + lock.readLock().lock(); + try (NoexceptAutoCloseable guarder = NoexceptAutoCloseables.all(chunks)) { + tableStorage.declareFields(schema); + memTableQueue.store(chunks); + if (shared.getStorageProperties().getWriteBufferTimeout().toMillis() <= 0) { + memTableQueue.flush(); + } + } finally { + lock.readLock().unlock(); + } + } + + @Override + public void delete(AreaSet range) throws StorageException { + AreaSet innerAreas = ArrowFields.toInnerAreas(range); + lock.writeLock().lock(); + try { + LOGGER.debug("start to delete {} in {}", range, name); + memTableQueue.delete(range); + tableStorage.delete(innerAreas); + } catch (IOException e) { + throw new StorageRuntimeException(e); + } finally { + lock.writeLock().unlock(); + } + } + + @Override + public void clear() throws StorageException { + lock.writeLock().lock(); + try { + LOGGER.debug("start to clear {}", name); + flusher.stop(); + memTableQueue.clear(); + tableStorage.clear(); + if (allocator.getAllocatedMemory() > 0) { + throw new IllegalStateException("allocator is not empty: " + allocator.toVerboseString()); + } + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("cleared {}, allocator: {}", name, allocator); + } + flusher.start(); + } finally { + lock.writeLock().unlock(); + } + } + + @Override + public void close() throws Exception { + lock.writeLock().lock(); + try { + if (shared.getStorageProperties().toFlushOnClose()) { + memTableQueue.flush(); + } + flusher.close(); + memTableQueue.close(); + tableStorage.close(); + allocator.close(); + } finally { + lock.writeLock().unlock(); + } + } +} diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/api/ReadWriter.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/api/ReadWriter.java new file mode 100644 index 0000000000..7f6488432d --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/api/ReadWriter.java @@ -0,0 +1,47 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.parquet.db.lsm.api; + +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; +import cn.edu.tsinghua.iginx.parquet.db.util.AreaSet; +import cn.edu.tsinghua.iginx.parquet.db.util.iterator.Scanner; +import com.google.common.collect.RangeSet; +import java.io.IOException; +import java.util.Set; + +public interface ReadWriter { + + String getName(); + + void flush(String name, TableMeta meta, Scanner> scanner) + throws IOException; + + TableMeta readMeta(String name) throws IOException; + + Scanner> scanData( + String name, Set fields, RangeSet ranges, Filter predicate) throws IOException; + + void delete(String name, AreaSet areas) throws IOException; + + void delete(String name); + + Iterable tableNames() throws IOException; + + void clear() throws IOException; +} diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/api/TableMeta.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/api/TableMeta.java new file mode 100644 index 0000000000..85a6d21764 --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/api/TableMeta.java @@ -0,0 +1,46 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.parquet.db.lsm.api; + +import cn.edu.tsinghua.iginx.thrift.DataType; +import com.google.common.collect.Range; +import java.util.Map; +import javax.annotation.Nullable; + +public interface TableMeta { + Map getSchema(); + + Range getRange(String field); + + default Range getRange(Iterable fields) { + Range range = null; + for (String field : fields) { + Range fieldRange = getRange(field); + if (range == null) { + range = getRange(field); + } else { + range = range.span(fieldRange); + } + } + return range; + } + + @Nullable + Long getValueCount(String field); +} diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/buffer/ActiveMemTable.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/buffer/ActiveMemTable.java new file mode 100644 index 0000000000..bf50ba2cdd --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/buffer/ActiveMemTable.java @@ -0,0 +1,247 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cn.edu.tsinghua.iginx.parquet.db.lsm.buffer; + +import cn.edu.tsinghua.iginx.parquet.db.lsm.buffer.chunk.Chunk; +import cn.edu.tsinghua.iginx.parquet.db.lsm.table.MemoryTable; +import cn.edu.tsinghua.iginx.parquet.db.util.AreaSet; +import cn.edu.tsinghua.iginx.parquet.db.util.iterator.Scanner; +import cn.edu.tsinghua.iginx.parquet.util.Awaitable; +import cn.edu.tsinghua.iginx.parquet.util.NoexceptAutoCloseable; +import cn.edu.tsinghua.iginx.parquet.util.Shared; +import cn.edu.tsinghua.iginx.parquet.util.arrow.ArrowFields; +import com.google.common.collect.RangeSet; +import java.io.IOException; +import java.util.*; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.locks.ReentrantLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.Consumer; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.util.Preconditions; +import org.apache.arrow.vector.types.pojo.Field; + +public class ActiveMemTable { + + private final ReentrantReadWriteLock switchTableLock = new ReentrantReadWriteLock(true); + private final ReentrantLock createLock = new ReentrantLock(true); + private final ReentrantReadWriteLock flushLock = new ReentrantReadWriteLock(true); + + private final Shared shared; + private final BufferAllocator allocator; + + private long activeId = 0; + private BufferAllocator activeAllocator = null; + private MemTable activeTable = null; + private final NavigableMap awaiting = new TreeMap<>(); + + ActiveMemTable(Shared shared, BufferAllocator allocator) { + this.shared = Preconditions.checkNotNull(shared); + this.allocator = Preconditions.checkNotNull(allocator); + } + + public boolean isOverloaded() { + switchTableLock.readLock().lock(); + try { + return activeAllocator != null + && activeAllocator.getAllocatedMemory() + >= shared.getStorageProperties().getWriteBufferSize(); + } finally { + switchTableLock.readLock().unlock(); + } + } + + public void store(Iterable data) { + switchTableLock.readLock().lock(); + try { + createMemtableIfNotExist(); + activeTable.store(data); + } finally { + switchTableLock.readLock().unlock(); + } + } + + private void createMemtableIfNotExist() { + createLock.lock(); + try { + if (activeTable == null) { + assert activeAllocator == null; + String name = + String.join( + "-", allocator.getName(), MemTable.class.getSimpleName(), String.valueOf(activeId)); + activeAllocator = allocator.newChildAllocator(name, 0, Long.MAX_VALUE); + activeTable = + new MemTable( + shared.getStorageProperties().getWriteBufferChunkFactory(), + activeAllocator, + shared.getStorageProperties().getWriteBufferChunkValuesMax(), + shared.getStorageProperties().getWriteBufferChunkValuesMin()); + } + } finally { + createLock.unlock(); + } + } + + public Awaitable flush() { + flushLock.writeLock().lock(); + try { + switchTableLock.readLock().lock(); + try { + if (activeTable == null) { + return () -> {}; + } + } finally { + switchTableLock.readLock().unlock(); + } + CountDownLatch latch = new CountDownLatch(1); + awaiting.put(activeId++, latch); + return latch::await; + } finally { + flushLock.writeLock().unlock(); + } + } + + public Map archive() throws InterruptedException { + shared.getMemTablePermits().acquire(); + flushLock.writeLock().lock(); + switchTableLock.writeLock().lock(); + try { + if (activeTable == null) { + shared.getMemTablePermits().release(); + return Collections.emptyMap(); + } + Map result = new HashMap<>(); + List onClose = new ArrayList<>(); + awaiting.values().forEach(latch -> onClose.add(latch::countDown)); + onClose.add(activeAllocator::close); + onClose.add(() -> shared.getMemTablePermits().release()); + result.put(activeId++, new ArchivedMemTable(activeTable, onClose)); + awaiting.clear(); + activeTable = null; + activeAllocator = null; + return result; + } finally { + switchTableLock.writeLock().unlock(); + flushLock.writeLock().unlock(); + } + } + + public MemoryTable snapshot(long id, BufferAllocator allocator) { + flushLock.readLock().lock(); + try { + if (awaiting.containsKey(id)) { + switchTableLock.readLock().lock(); + try { + return activeTable.snapshot(allocator); + } finally { + switchTableLock.readLock().unlock(); + } + } else { + return MemoryTable.empty(); + } + } finally { + flushLock.readLock().unlock(); + } + } + + public Long newestKey(long idAtLeast) { + flushLock.readLock().lock(); + try { + if (!awaiting.isEmpty()) { + long id = awaiting.lastKey(); + if (id >= idAtLeast) { + return id; + } + } + return null; + } finally { + flushLock.readLock().unlock(); + } + } + + public boolean eliminate(long id) throws InterruptedException { + CountDownLatch latch = null; + flushLock.writeLock().lock(); + try { + if (awaiting.containsKey(id)) { + latch = new CountDownLatch(1); + } + SortedMap older = awaiting.headMap(id, true); + older.values().forEach(CountDownLatch::countDown); + older.clear(); + if (latch != null) { + awaiting.put(id, latch); + } + } finally { + flushLock.writeLock().unlock(); + } + if (latch != null) { + latch.await(); + } + return latch != null; + } + + public void delete(AreaSet areas) { + switchTableLock.readLock().lock(); + try { + if (activeTable != null) { + activeTable.delete(areas); + } + } finally { + switchTableLock.readLock().unlock(); + } + } + + public void reset() { + flushLock.writeLock().lock(); + switchTableLock.writeLock().lock(); + try { + if (activeTable != null) { + activeTable.close(); + activeAllocator.close(); + } + activeId = 0; + activeTable = null; + activeAllocator = null; + awaiting.values().forEach(CountDownLatch::countDown); + awaiting.clear(); + } finally { + switchTableLock.writeLock().unlock(); + flushLock.writeLock().unlock(); + } + } + + public void scan( + List fields, + RangeSet ranges, + BufferAllocator allocator, + Consumer>> consumer) + throws IOException { + switchTableLock.readLock().lock(); + try { + if (activeTable != null) { + Set innerFields = ArrowFields.toInnerFields(fields); + try (MemoryTable table = activeTable.snapshot(fields, ranges, allocator)) { + consumer.accept(table.scan(innerFields, ranges)); + } + } + } finally { + switchTableLock.readLock().unlock(); + } + } +} diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/buffer/ArchivedMemTable.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/buffer/ArchivedMemTable.java new file mode 100644 index 0000000000..06d1886438 --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/buffer/ArchivedMemTable.java @@ -0,0 +1,81 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cn.edu.tsinghua.iginx.parquet.db.lsm.buffer; + +import cn.edu.tsinghua.iginx.parquet.db.lsm.table.MemoryTable; +import cn.edu.tsinghua.iginx.parquet.db.util.AreaSet; +import cn.edu.tsinghua.iginx.parquet.util.NoexceptAutoCloseable; +import com.google.common.collect.RangeSet; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import javax.annotation.WillCloseWhenClosed; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.util.Preconditions; +import org.apache.arrow.vector.types.pojo.Field; + +public class ArchivedMemTable implements NoexceptAutoCloseable { + private final MemTable memTable; + private final Collection onClose; + private final AreaSet deleted = new AreaSet<>(); + private final CountDownLatch latch = new CountDownLatch(1); + private boolean snapshot = false; + + public ArchivedMemTable( + @WillCloseWhenClosed MemTable memTable, + @WillCloseWhenClosed Collection onClose) { + this.memTable = Preconditions.checkNotNull(memTable); + this.onClose = new ArrayList<>(onClose); + } + + public synchronized MemoryTable snapshot(BufferAllocator allocator) { + snapshot = true; + memTable.compact(); + return memTable.snapshot(allocator); + } + + public synchronized MemoryTable snapshot( + List fields, RangeSet ranges, BufferAllocator allocator) { + snapshot = true; + memTable.compact(); + return memTable.snapshot(fields, ranges, allocator); + } + + public synchronized AreaSet getDeleted() { + return AreaSet.create(deleted); + } + + public synchronized void delete(AreaSet ranges) { + memTable.delete(ranges); + if (snapshot) { + deleted.addAll(ranges); + } + } + + public void waitUntilClosed() throws InterruptedException { + latch.await(); + } + + @Override + public void close() { + latch.countDown(); + memTable.close(); + onClose.forEach(NoexceptAutoCloseable::close); + } +} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/buffer/DataBuffer.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/buffer/DataBuffer.java similarity index 85% rename from dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/buffer/DataBuffer.java rename to dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/buffer/DataBuffer.java index ef6435c51e..1c0830a9ba 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/buffer/DataBuffer.java +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/buffer/DataBuffer.java @@ -1,17 +1,19 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.parquet.db.lsm.buffer; @@ -163,4 +165,9 @@ public String toString() { public void close() throws StorageException { data.clear(); } + + public int count(F field) { + NavigableMap column = data.get(field); + return column == null ? 0 : column.size(); + } } diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/buffer/MemColumn.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/buffer/MemColumn.java new file mode 100644 index 0000000000..a96aa9e2aa --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/buffer/MemColumn.java @@ -0,0 +1,371 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cn.edu.tsinghua.iginx.parquet.db.lsm.buffer; + +import cn.edu.tsinghua.iginx.parquet.db.lsm.buffer.chunk.Chunk; +import cn.edu.tsinghua.iginx.parquet.db.lsm.buffer.chunk.IndexedChunk; +import cn.edu.tsinghua.iginx.parquet.util.iterator.DedupIterator; +import cn.edu.tsinghua.iginx.parquet.util.iterator.StableMergeIterator; +import com.google.common.collect.*; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import javax.annotation.WillCloseWhenClosed; +import javax.annotation.concurrent.ThreadSafe; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.util.Preconditions; + +@ThreadSafe +public class MemColumn implements AutoCloseable { + + private final int maxChunkValueCount; + private final int minChunkValueCount; + private final List compactedChunkSnapshots = new ArrayList<>(); + private final ChunkHolder active; + + public MemColumn( + IndexedChunk.Factory factory, + BufferAllocator allocator, + int maxChunkValueCount, + int minChunkValueCount) { + Preconditions.checkNotNull(allocator); + Preconditions.checkNotNull(factory); + Preconditions.checkArgument(maxChunkValueCount > 0); + Preconditions.checkArgument(minChunkValueCount > 0); + + this.active = new ChunkHolder(factory, allocator); + this.maxChunkValueCount = maxChunkValueCount; + this.minChunkValueCount = minChunkValueCount; + } + + // TODO: make use of ValueFilter + public synchronized Snapshot snapshot(RangeSet ranges, BufferAllocator allocator) { + active.refresh(compactedChunkSnapshots); + return createSnapshot(ranges, compactedChunkSnapshots, allocator); + } + + public synchronized Snapshot snapshot(BufferAllocator allocator) { + return snapshot(ImmutableRangeSet.of(Range.all()), allocator); + } + + private static Snapshot createSnapshot( + RangeSet ranges, + List snapshotHolders, + @Nullable BufferAllocator allocator) { + List snapshots = new ArrayList<>(); + for (ChunkSnapshotHolder snapshot : snapshotHolders) { + ChunkSnapshotHolder filtered; + if (allocator != null) { + filtered = snapshot.slice(allocator); + } else { + filtered = snapshot.slice(); + } + filtered.delete(ranges.complement()); + if (filtered.isEmpty()) { + filtered.close(); + } else { + snapshots.add(filtered); + } + } + return new Snapshot(snapshots); + } + + public synchronized void store(Chunk.Snapshot snapshot) { + int length = snapshot.getValueCount(); + if (length < minChunkValueCount) { + copyStore(snapshot, 0, length); + } else { + splitStore(snapshot, 0, length); + } + } + + public synchronized void splitStore(Chunk.Snapshot snapshot, int offset, int length) { + int activeValueCount = active.getValueCount(); + int padding = minChunkValueCount - active.getValueCount(); + if (activeValueCount > 0) { + if (padding > 0) { + if (padding >= length) { + active.store(snapshot, offset, length); + return; + } + active.store(snapshot, offset, padding); + offset += padding; + length -= padding; + } + compact(); + } + + if (length >= minChunkValueCount) { + compactedChunkSnapshots.add(active.sorted(snapshot, offset, length)); + } else { + active.store(snapshot, offset, length); + } + } + + public synchronized void copyStore(Chunk.Snapshot snapshot, int offset, int length) { + for (int written = offset; written != length; ) { + int free = maxChunkValueCount - active.getValueCount(); + if (free == 0) { + compact(); + } + int toWrite = Math.min(free, length - written); + active.store(snapshot, written, toWrite); + written += toWrite; + } + } + + public synchronized void delete(RangeSet ranges) { + active.delete(ranges); + for (ChunkSnapshotHolder snapshot : compactedChunkSnapshots) { + snapshot.delete(ranges); + } + } + + @Override + public synchronized void close() { + compactedChunkSnapshots.forEach(ChunkSnapshotHolder::close); + compactedChunkSnapshots.clear(); + active.close(); + } + + public synchronized void compact() { + active.refresh(compactedChunkSnapshots); + active.reset(); + } + + public static class Snapshot implements AutoCloseable, Iterable> { + + private final List snapshots; + + Snapshot(@WillCloseWhenClosed List snapshots) { + this.snapshots = Preconditions.checkNotNull(snapshots); + } + + public Snapshot slice(RangeSet ranges, BufferAllocator allocator) { + return createSnapshot(ranges, snapshots, allocator); + } + + public Snapshot slice(RangeSet ranges) { + return createSnapshot(ranges, snapshots, null); + } + + @Override + public void close() { + snapshots.forEach(ChunkSnapshotHolder::close); + snapshots.clear(); + } + + @Override + @Nonnull + public Iterator> iterator() { + Iterator> mergedIterator = + new StableMergeIterator<>(getIterators(), Map.Entry.comparingByKey()); + return new DedupIterator<>(mergedIterator, Map.Entry::getKey); + } + + private List>> getIterators() { + // TODO: 对 Iterator 进行 concat 减少 StableMergeIterator 内 queue 中的项数 + // 目前使用贪心算法,可能不是最优解 + List> rangeGroups = new ArrayList<>(); + RangeMap currentRanges = TreeRangeMap.create(); + for (int i = 0; i < snapshots.size(); i++) { + ChunkSnapshotHolder snapshot = snapshots.get(i); + Range keyRange = snapshot.getKeyRange(); + if (currentRanges.subRangeMap(keyRange).asMapOfRanges().isEmpty()) { + currentRanges.put(keyRange, i); + } else { + rangeGroups.add(currentRanges); + currentRanges = TreeRangeMap.create(); + currentRanges.put(keyRange, i); + } + } + rangeGroups.add(currentRanges); + List>> iterators = new ArrayList<>(); + for (RangeMap rangeMap : rangeGroups) { + List>> groupIterators = new ArrayList<>(); + for (int i : rangeMap.asMapOfRanges().values()) { + groupIterators.add(snapshots.get(i).iterator()); + } + iterators.add(Iterators.concat(groupIterators.iterator())); + } + return iterators; + } + + public RangeSet getRanges() { + RangeSet range = TreeRangeSet.create(); + snapshots.forEach(snapshot -> range.add(snapshot.getKeyRange())); + return range; + } + } + + private static class ChunkSnapshotHolder + implements AutoCloseable, Iterable> { + + private final Chunk.Snapshot snapshot; + private RangeSet mask; + + public ChunkSnapshotHolder(@WillCloseWhenClosed Chunk.Snapshot snapshot) { + this(snapshot, null); + } + + private ChunkSnapshotHolder(@WillCloseWhenClosed Chunk.Snapshot snapshot, RangeSet mask) { + Preconditions.checkArgument(snapshot.getValueCount() > 0); + this.snapshot = snapshot; + if (mask != null) { + this.mask = TreeRangeSet.create(mask); + } else { + this.mask = null; + } + } + + public boolean isEmpty() { + return mask != null && mask.isEmpty(); + } + + public void delete(RangeSet ranges) { + if (mask == null) { + Range fullRange = getKeyRange(snapshot); + if (!ranges.intersects(fullRange)) { + return; + } + mask = TreeRangeSet.create(); + mask.add(fullRange); + } + mask.removeAll(ranges); + } + + public Range getKeyRange() { + if (mask == null) { + return getKeyRange(snapshot); + } + if (mask.isEmpty()) { + return Range.closedOpen(0L, 0L); + } + return mask.span(); + } + + private static Range getKeyRange(Chunk.Snapshot snapshot) { + return Range.closed(snapshot.getKey(0), snapshot.getKey(snapshot.getValueCount() - 1)); + } + + @Override + public void close() { + snapshot.close(); + } + + public ChunkSnapshotHolder slice(BufferAllocator allocator) { + return new ChunkSnapshotHolder(snapshot.slice(allocator), mask); + } + + public ChunkSnapshotHolder slice() { + return new ChunkSnapshotHolder(snapshot.slice(), mask); + } + + @Override + @Nonnull + public Iterator> iterator() { + Iterator> iterator = snapshot.iterator(); + if (mask == null) { + return iterator; + } + return Iterators.filter(iterator, entry -> mask.contains(entry.getKey())); + } + } + + private static class ChunkHolder implements AutoCloseable { + private final IndexedChunk.Factory factory; + private final BufferAllocator allocator; + private IndexedChunk activeChunk = null; + private boolean isDirty = false; + private boolean hasOld = false; + + ChunkHolder(IndexedChunk.Factory factory, BufferAllocator allocator) { + this.factory = factory; + this.allocator = allocator; + } + + public int getValueCount() { + return activeChunk == null ? 0 : activeChunk.getValueCount(); + } + + public ChunkSnapshotHolder sorted(Chunk.Snapshot snapshot, int offset, int length) { + Chunk.Snapshot sorted = factory.sorted(snapshot.slice(offset, length, allocator), allocator); + return new ChunkSnapshotHolder(sorted); + } + + public void store(Chunk.Snapshot data) { + if (data.getValueCount() == 0) { + return; + } + if (activeChunk == null) { + activeChunk = factory.wrap(data, allocator); + } + activeChunk.store(data); + isDirty = true; + } + + public void store(Chunk.Snapshot snapshot, int offset, int length) { + try (Chunk.Snapshot slice = snapshot.slice(offset, length, allocator)) { + store(slice); + } + } + + public void delete(RangeSet ranges) { + if (activeChunk != null) { + activeChunk.delete(ranges); + } + } + + public void refresh(List compactedChunkSnapshots) { + if (!isDirty) { + return; + } + isDirty = false; + + if (hasOld) { + int offset = compactedChunkSnapshots.size() - 1; + compactedChunkSnapshots.remove(offset).close(); + } + + Chunk.Snapshot snapshot = activeChunk.snapshot(allocator); + if (snapshot.getValueCount() > 0) { + compactedChunkSnapshots.add(new ChunkSnapshotHolder(snapshot)); + hasOld = true; + } else { + snapshot.close(); + reset(); + } + } + + public void reset() { + if (activeChunk != null) { + activeChunk.close(); + activeChunk = null; + } + isDirty = false; + hasOld = false; + } + + public void close() { + reset(); + } + } +} diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/buffer/MemTable.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/buffer/MemTable.java new file mode 100644 index 0000000000..bd6697cc5c --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/buffer/MemTable.java @@ -0,0 +1,174 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cn.edu.tsinghua.iginx.parquet.db.lsm.buffer; + +import cn.edu.tsinghua.iginx.parquet.db.lsm.buffer.chunk.Chunk; +import cn.edu.tsinghua.iginx.parquet.db.lsm.buffer.chunk.IndexedChunk; +import cn.edu.tsinghua.iginx.parquet.db.lsm.table.MemoryTable; +import cn.edu.tsinghua.iginx.parquet.db.util.AreaSet; +import cn.edu.tsinghua.iginx.parquet.util.arrow.ArrowFields; +import com.google.common.collect.RangeSet; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import javax.annotation.concurrent.ThreadSafe; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.types.pojo.Field; + +@ThreadSafe +public class MemTable implements AutoCloseable { + + private final ConcurrentHashMap columns = new ConcurrentHashMap<>(); + private final IndexedChunk.Factory factory; + private final BufferAllocator allocator; + private final int maxChunkValueCount; + private final int minChunkValueCount; + + private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + private volatile boolean closed = false; + + public MemTable( + IndexedChunk.Factory factory, + BufferAllocator allocator, + int maxChunkValueCount, + int minChunkValueCount) { + this.factory = factory; + this.allocator = allocator; + this.maxChunkValueCount = maxChunkValueCount; + this.minChunkValueCount = minChunkValueCount; + } + + public void reset() { + lock.writeLock().lock(); + try { + columns.values().forEach(MemColumn::close); + columns.clear(); + } finally { + lock.writeLock().unlock(); + } + } + + @Override + public void close() { + closed = true; + reset(); + } + + public Set getFields() { + return Collections.unmodifiableSet(columns.keySet()); + } + + public MemoryTable snapshot( + List fields, RangeSet ranges, BufferAllocator allocator) { + LinkedHashMap columns = new LinkedHashMap<>(); + lock.readLock().lock(); + try { + for (Field field : fields) { + if (columns.containsKey(field)) { + throw new IllegalArgumentException("Duplicate field: " + field); + } + MemColumn column = this.columns.get(field); + if (column != null) { + columns.put(field, column.snapshot(ranges, allocator)); + } + } + } finally { + lock.readLock().unlock(); + } + return new MemoryTable(columns); + } + + public MemoryTable snapshot(BufferAllocator allocator) { + LinkedHashMap columns = new LinkedHashMap<>(); + lock.readLock().lock(); + try { + for (Map.Entry entry : this.columns.entrySet()) { + columns.put(entry.getKey(), entry.getValue().snapshot(allocator)); + } + } finally { + lock.readLock().unlock(); + } + return new MemoryTable(columns); + } + + public void store(Iterable data) { + lock.readLock().lock(); + try { + data.forEach(this::store); + } finally { + lock.readLock().unlock(); + } + } + + public void store(Chunk.Snapshot data) { + lock.readLock().lock(); + try { + if (closed) { + throw new IllegalStateException("MemTable is closed"); + } + Field field = ArrowFields.nullable(data.getField()); + MemColumn column = + columns.computeIfAbsent( + field, + key -> new MemColumn(factory, allocator, maxChunkValueCount, minChunkValueCount)); + column.store(data); + } finally { + lock.readLock().unlock(); + } + } + + public void compact() { + lock.readLock().lock(); + try { + columns.values().forEach(MemColumn::compact); + } finally { + lock.readLock().unlock(); + } + } + + public void delete(AreaSet areas) { + lock.writeLock().lock(); + try { + for (Field field : areas.getFields()) { + MemColumn column = columns.remove(field); + if (column != null) { + column.close(); + } + } + lock.readLock().lock(); + } finally { + lock.writeLock().unlock(); + } + try { + RangeSet keys = areas.getKeys(); + if (!keys.isEmpty()) { + columns.values().forEach(column -> column.delete(keys)); + } + for (Map.Entry> entry : areas.getSegments().entrySet()) { + Field field = entry.getKey(); + RangeSet ranges = entry.getValue(); + MemColumn column = columns.get(field); + if (column != null) { + column.delete(ranges); + } + } + } finally { + lock.readLock().unlock(); + } + } +} diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/buffer/MemTableQueue.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/buffer/MemTableQueue.java new file mode 100644 index 0000000000..9200b718ca --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/buffer/MemTableQueue.java @@ -0,0 +1,247 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cn.edu.tsinghua.iginx.parquet.db.lsm.buffer; + +import cn.edu.tsinghua.iginx.parquet.db.lsm.buffer.chunk.Chunk; +import cn.edu.tsinghua.iginx.parquet.db.lsm.table.MemoryTable; +import cn.edu.tsinghua.iginx.parquet.db.util.AreaSet; +import cn.edu.tsinghua.iginx.parquet.db.util.iterator.AreaFilterScanner; +import cn.edu.tsinghua.iginx.parquet.db.util.iterator.Scanner; +import cn.edu.tsinghua.iginx.parquet.util.Awaitable; +import cn.edu.tsinghua.iginx.parquet.util.NoexceptAutoCloseable; +import cn.edu.tsinghua.iginx.parquet.util.Shared; +import cn.edu.tsinghua.iginx.parquet.util.arrow.ArrowFields; +import com.google.common.collect.RangeSet; +import java.io.IOException; +import java.util.*; +import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.ReentrantLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.Consumer; +import javax.annotation.Nonnegative; +import javax.annotation.concurrent.ThreadSafe; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.util.AutoCloseables; +import org.apache.arrow.vector.types.pojo.Field; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@ThreadSafe +public class MemTableQueue implements NoexceptAutoCloseable { + private final Logger LOGGER = LoggerFactory.getLogger(MemTableQueue.class); + + private final ReentrantLock checkSizeLock = new ReentrantLock(true); + private final ReentrantReadWriteLock queueLock = new ReentrantReadWriteLock(true); + private final ReentrantLock pollLock = new ReentrantLock(true); + private final Condition pollLockCond = pollLock.newCondition(); + + private final NavigableMap archives = new TreeMap<>(); + private final BufferAllocator allocator; + private final ActiveMemTable active; + + public MemTableQueue(Shared shared, BufferAllocator allocator) { + String allocatorName = + String.join("-", allocator.getName(), MemTableQueue.class.getSimpleName()); + this.allocator = allocator.newChildAllocator(allocatorName, 0, Long.MAX_VALUE); + this.active = new ActiveMemTable(shared, this.allocator); + } + + public void store(Iterable data) throws InterruptedException { + checkSizeLock.lock(); + try { + if (active.isOverloaded()) { + compact(); + } + } finally { + checkSizeLock.unlock(); + } + active.store(data); + } + + public void compact() throws InterruptedException { + checkSizeLock.lock(); + try { + Map temp = active.archive(); + queueLock.writeLock().lock(); + try { + archives.putAll(temp); + temp.clear(); + } finally { + queueLock.writeLock().unlock(); + temp.values().forEach(ArchivedMemTable::close); + } + } finally { + checkSizeLock.unlock(); + } + signalAll(); + } + + public void flush() throws InterruptedException { + Awaitable flush; + List waiters = new ArrayList<>(); + queueLock.readLock().lock(); + try { + flush = active.flush(); + for (ArchivedMemTable archivedMemTable : archives.values()) { + waiters.add(archivedMemTable::waitUntilClosed); + } + } finally { + queueLock.readLock().unlock(); + } + signalAll(); + flush.await(); + for (Awaitable waiter : waiters) { + waiter.await(); + } + } + + public void delete(AreaSet areas) { + queueLock.readLock().lock(); + try { + active.delete(areas); + archives.values().forEach(archived -> archived.delete(areas)); + } finally { + queueLock.readLock().unlock(); + } + } + + private void await() throws InterruptedException { + pollLock.lock(); + try { + pollLockCond.await(); + } finally { + pollLock.unlock(); + } + } + + private void signalAll() { + pollLock.lock(); + try { + pollLockCond.signalAll(); + } finally { + pollLock.unlock(); + } + } + + /** + * take next table id, if no table id available, block until new table id available + * + * @param idAtLeast the non-negative table id at least + * @return the next non-negative table id + */ + @Nonnegative + public long awaitNext(@Nonnegative long idAtLeast) throws InterruptedException { + while (true) { + queueLock.readLock().lock(); + try { + Long nextId = archives.ceilingKey(idAtLeast); + if (nextId != null) { + return nextId; + } + Long activeNextId = active.newestKey(idAtLeast); + if (activeNextId != null) { + return activeNextId; + } + } finally { + queueLock.readLock().unlock(); + } + await(); + } + } + + public void eliminate(long id, Consumer> commiter) + throws InterruptedException { + active.eliminate(id); + + queueLock.writeLock().lock(); + try { + if (archives.containsKey(id)) { + try (ArchivedMemTable memTable = archives.remove(id)) { + commiter.accept(memTable.getDeleted()); + } + return; + } + } finally { + queueLock.writeLock().unlock(); + } + + commiter.accept(AreaSet.all()); + } + + public MemoryTable snapshot(long id, BufferAllocator allocator) { + queueLock.readLock().lock(); + try { + ArchivedMemTable archived = archives.get(id); + if (archived != null) { + return archived.snapshot(allocator); + } + return active.snapshot(id, allocator); + } finally { + queueLock.readLock().unlock(); + } + } + + public List>> scan( + List fields, RangeSet ranges, BufferAllocator allocator) throws IOException { + Set innerFields = ArrowFields.toInnerFields(fields); + List>> scanners = new ArrayList<>(); + queueLock.readLock().lock(); + try { + for (ArchivedMemTable archivedMemTable : archives.values()) { + try (MemoryTable table = archivedMemTable.snapshot(fields, ranges, allocator)) { + Scanner> scanner = table.scan(innerFields, ranges); + AreaSet deleted = archivedMemTable.getDeleted(); + if (deleted.isEmpty()) { + scanners.add(scanner); + } else { + AreaSet innerDelete = ArrowFields.toInnerAreas(deleted); + scanners.add(new AreaFilterScanner<>(scanner, innerDelete)); + } + } + } + active.scan(fields, ranges, allocator, scanners::add); + } catch (Exception e) { + try { + AutoCloseables.close(scanners); + } catch (Exception ex) { + e.addSuppressed(ex); + } + throw e; + } finally { + queueLock.readLock().unlock(); + } + return scanners; + } + + public void clear() { + queueLock.writeLock().lock(); + try { + active.reset(); + archives.values().forEach(ArchivedMemTable::close); + archives.clear(); + } finally { + queueLock.writeLock().unlock(); + } + } + + @Override + public void close() { + clear(); + allocator.close(); + } +} diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/buffer/chunk/Chunk.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/buffer/chunk/Chunk.java new file mode 100644 index 0000000000..36fa71f0e1 --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/buffer/chunk/Chunk.java @@ -0,0 +1,184 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cn.edu.tsinghua.iginx.parquet.db.lsm.buffer.chunk; + +import cn.edu.tsinghua.iginx.parquet.util.NoexceptAutoCloseable; +import cn.edu.tsinghua.iginx.parquet.util.arrow.ArrowVectors; +import java.util.AbstractMap; +import java.util.Iterator; +import java.util.Map; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import javax.annotation.WillCloseWhenClosed; +import javax.annotation.concurrent.GuardedBy; +import javax.annotation.concurrent.Immutable; +import javax.annotation.concurrent.ThreadSafe; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.util.Preconditions; +import org.apache.arrow.vector.BigIntVector; +import org.apache.arrow.vector.ValueVector; +import org.apache.arrow.vector.ZeroVector; +import org.apache.arrow.vector.types.pojo.Field; + +@ThreadSafe +public class Chunk implements AutoCloseable { + + @GuardedBy("this") + protected final ValueVector keys; + + @GuardedBy("this") + protected final ValueVector values; + + protected Chunk(@WillCloseWhenClosed ValueVector keys, @WillCloseWhenClosed ValueVector values) { + Preconditions.checkNotNull(keys); + Preconditions.checkNotNull(values); + Preconditions.checkArgument(keys.getValueCount() == values.getValueCount()); + + this.keys = keys; + this.values = values; + } + + public static Chunk like(Snapshot snapshot, BufferAllocator allocator) { + return new Chunk( + ArrowVectors.like(snapshot.keys, allocator), ArrowVectors.like(snapshot.values, allocator)); + } + + public synchronized Snapshot snapshot(BufferAllocator allocator) { + return new Snapshot(ArrowVectors.slice(keys, allocator), ArrowVectors.slice(values, allocator)); + } + + public synchronized int store(Snapshot data) { + int offset = keys.getValueCount(); + ArrowVectors.append(keys, data.keys); + ArrowVectors.append(values, data.values); + return offset; + } + + public synchronized int getValueCount() { + return keys.getValueCount(); + } + + @Override + public synchronized void close() { + keys.close(); + values.close(); + } + + @Immutable + public static class Snapshot implements NoexceptAutoCloseable, Iterable> { + + protected final ValueVector keys; + protected final ValueVector values; + + public Snapshot( + @WillCloseWhenClosed ValueVector keys, @WillCloseWhenClosed ValueVector values) { + Preconditions.checkNotNull(keys); + Preconditions.checkNotNull(values); + Preconditions.checkArgument(keys.getValueCount() == values.getValueCount()); + Preconditions.checkArgument(!keys.getField().isNullable()); + Preconditions.checkArgument(!values.getField().isNullable()); + + this.keys = keys; + this.values = values; + } + + public static Snapshot empty(Field key, Field value) { + Preconditions.checkNotNull(key); + Preconditions.checkNotNull(value); + return new Snapshot(new ZeroVector(key), new ZeroVector(value)); + } + + public Field getField() { + return values.getField(); + } + + public int getValueCount() { + return keys.getValueCount(); + } + + public Snapshot slice() { + return doSlice(0, keys.getValueCount(), null); + } + + public Snapshot slice(int start, int length) { + return doSlice(start, length, null); + } + + public Snapshot slice(BufferAllocator allocator) { + return doSlice(0, keys.getValueCount(), allocator); + } + + public Snapshot slice(int start, int length, BufferAllocator allocator) { + return doSlice(start, length, allocator); + } + + private Snapshot doSlice(int start, int length, @Nullable BufferAllocator allocator) { + if (allocator == null) { + return new Snapshot( + ArrowVectors.slice(keys, start, length), ArrowVectors.slice(values, start, length)); + } + return new Snapshot( + ArrowVectors.slice(keys, start, length, allocator), + ArrowVectors.slice(values, start, length, allocator)); + } + + @Override + public void close() { + keys.close(); + values.close(); + } + + public long getKey(int index) { + return doGetKey(index); + } + + private long doGetKey(int index) { + return keys.getDataBuffer().getLong((long) index * BigIntVector.TYPE_WIDTH); + } + + public Map.Entry get(int index) { + long key = doGetKey(index); + Object value = values.getObject(index); + return new AbstractMap.SimpleImmutableEntry<>(key, value); + } + + @Override + @Nonnull + public Iterator> iterator() { + return new ChunkSnapshotReader(); + } + + public class ChunkSnapshotReader implements Iterator> { + private int position = 0; + private final int valueCount = getValueCount(); + + @Override + public boolean hasNext() { + return position < valueCount; + } + + @Override + public Map.Entry next() { + if (!hasNext()) { + throw new IndexOutOfBoundsException(); + } + return get(position++); + } + } + } +} diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/buffer/chunk/IndexedChunk.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/buffer/chunk/IndexedChunk.java new file mode 100644 index 0000000000..aa06ab9223 --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/buffer/chunk/IndexedChunk.java @@ -0,0 +1,156 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cn.edu.tsinghua.iginx.parquet.db.lsm.buffer.chunk; + +import cn.edu.tsinghua.iginx.parquet.util.arrow.ArrowVectors; +import com.google.common.collect.RangeSet; +import java.util.Map; +import javax.annotation.Nullable; +import javax.annotation.WillClose; +import javax.annotation.WillCloseWhenClosed; +import javax.annotation.concurrent.Immutable; +import javax.annotation.concurrent.ThreadSafe; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.util.Preconditions; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.types.Types; + +@ThreadSafe +public abstract class IndexedChunk extends Chunk { + + protected final BufferAllocator allocator; + + protected IndexedChunk(@WillCloseWhenClosed Chunk chunk, BufferAllocator allocator) { + super(chunk.keys, chunk.values); + this.allocator = allocator; + } + + @Override + public synchronized Snapshot snapshot(BufferAllocator allocator) { + Snapshot snapshot = super.snapshot(allocator); + if (ArrowVectors.isSorted(snapshot.keys)) { + return snapshot; + } + IntVector indexes = indexOf(snapshot, allocator); + return new IndexedSnapshot(snapshot, indexes); + } + + @Override + public synchronized int store(Snapshot data) { + int offset = super.store(data); + updateIndex(data, offset); + return offset; + } + + public synchronized void delete(RangeSet rangeSet) { + Preconditions.checkNotNull(rangeSet); + deleteIndex(rangeSet); + } + + @Override + public synchronized void close() { + super.close(); + } + + protected abstract IntVector indexOf(Snapshot snapshot, BufferAllocator allocator); + + protected abstract void updateIndex(Snapshot data, int offset); + + protected abstract void deleteIndex(RangeSet rangeSet); + + public interface Factory { + IndexedChunk wrap(@WillCloseWhenClosed Chunk chunk, BufferAllocator allocator); + + default IndexedChunk wrap(Snapshot snapshot, BufferAllocator allocator) { + return wrap(Chunk.like(snapshot, allocator), allocator); + } + + default Snapshot sorted(@WillClose Snapshot snapshot, BufferAllocator allocator) { + try (IndexedChunk chunk = wrap(new Chunk(snapshot.keys, snapshot.values), allocator)) { + return chunk.snapshot(allocator); + } + } + } + + @Immutable + protected static class IndexedSnapshot extends Snapshot { + + protected final IntVector indexes; + + public IndexedSnapshot( + @WillCloseWhenClosed Snapshot snapshot, @WillCloseWhenClosed IntVector indexes) { + super(snapshot.keys, snapshot.values); + this.indexes = indexes; + Preconditions.checkArgument(snapshot.keys.getMinorType() == Types.MinorType.BIGINT); + Preconditions.checkArgument(!indexes.getField().isNullable()); + } + + @Override + public void close() { + super.close(); + indexes.close(); + } + + @Override + public int getValueCount() { + return indexes.getValueCount(); + } + + @Override + public IndexedSnapshot slice() { + return doSlice(0, indexes.getValueCount(), null); + } + + @Override + public IndexedSnapshot slice(int start, int length) { + return doSlice(start, length, null); + } + + @Override + public IndexedSnapshot slice(BufferAllocator allocator) { + return doSlice(0, indexes.getValueCount(), allocator); + } + + @Override + public IndexedSnapshot slice(int start, int length, BufferAllocator allocator) { + return doSlice(start, length, allocator); + } + + private IndexedSnapshot doSlice(int start, int length, @Nullable BufferAllocator allocator) { + if (allocator == null) { + return new IndexedSnapshot(super.slice(), ArrowVectors.slice(indexes, start, length)); + } + return new IndexedSnapshot( + super.slice(allocator), ArrowVectors.slice(indexes, start, length, allocator)); + } + + public int getIndex(int index) { + return indexes.getDataBuffer().getInt((long) index * IntVector.TYPE_WIDTH); + } + + @Override + public long getKey(int index) { + return super.getKey(getIndex(index)); + } + + @Override + public Map.Entry get(int index) { + return super.get(getIndex(index)); + } + } +} diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/buffer/chunk/IndexedChunkType.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/buffer/chunk/IndexedChunkType.java new file mode 100644 index 0000000000..7be938fe92 --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/buffer/chunk/IndexedChunkType.java @@ -0,0 +1,33 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cn.edu.tsinghua.iginx.parquet.db.lsm.buffer.chunk; + +public enum IndexedChunkType { + SKIP_LIST(SkipListChunk::new), + NONE(NoIndexChunk::new); + + private final IndexedChunk.Factory factory; + + IndexedChunkType(IndexedChunk.Factory factory) { + this.factory = factory; + } + + public IndexedChunk.Factory factory() { + return factory; + } +} diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/buffer/chunk/NoIndexChunk.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/buffer/chunk/NoIndexChunk.java new file mode 100644 index 0000000000..4cdeba279f --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/buffer/chunk/NoIndexChunk.java @@ -0,0 +1,72 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cn.edu.tsinghua.iginx.parquet.db.lsm.buffer.chunk; + +import cn.edu.tsinghua.iginx.parquet.util.arrow.ArrowVectors; +import com.google.common.collect.RangeSet; +import com.google.common.collect.TreeRangeSet; +import java.util.Map; +import java.util.TreeMap; +import javax.annotation.WillCloseWhenClosed; +import javax.annotation.concurrent.ThreadSafe; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.IntVector; + +@ThreadSafe +public class NoIndexChunk extends IndexedChunk { + + private final TreeMap> tombstone = new TreeMap<>(); + + private int valueCount = 0; + + public NoIndexChunk(@WillCloseWhenClosed Chunk chunk, BufferAllocator allocator) { + super(chunk, allocator); + tombstone.put(0, TreeRangeSet.create()); + } + + @Override + protected IntVector indexOf(Snapshot snapshot, BufferAllocator allocator) { + IntVector indexes = ArrowVectors.stableSortIndexes(snapshot.keys, allocator); + ArrowVectors.dedupSortedIndexes(snapshot.keys, indexes); + if (!tombstone.isEmpty()) { + ArrowVectors.filter(indexes, i -> !isDeleted(snapshot, i)); + } + return indexes; + } + + private boolean isDeleted(Snapshot snapshot, int index) { + Map.Entry> entry = tombstone.higherEntry(index); + if (entry == null) { + return false; + } + RangeSet deleted = entry.getValue(); + long key = snapshot.getKey(index); + return deleted.contains(key); + } + + @Override + protected void updateIndex(Snapshot data, int offset) { + valueCount = offset + data.getValueCount(); + } + + @Override + protected void deleteIndex(RangeSet rangeSet) { + tombstone.computeIfAbsent(valueCount, k -> TreeRangeSet.create()).addAll(rangeSet); + tombstone.values().forEach(r -> r.removeAll(rangeSet)); + } +} diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/buffer/chunk/SkipListChunk.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/buffer/chunk/SkipListChunk.java new file mode 100644 index 0000000000..6da6697a31 --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/buffer/chunk/SkipListChunk.java @@ -0,0 +1,89 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cn.edu.tsinghua.iginx.parquet.db.lsm.buffer.chunk; + +import cn.edu.tsinghua.iginx.parquet.util.arrow.ArrowVectors; +import com.google.common.collect.BoundType; +import com.google.common.collect.Range; +import com.google.common.collect.RangeSet; +import java.util.NavigableMap; +import java.util.concurrent.ConcurrentSkipListMap; +import javax.annotation.WillCloseWhenClosed; +import javax.annotation.concurrent.ThreadSafe; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.IntVector; + +@ThreadSafe +public class SkipListChunk extends IndexedChunk { + + protected final ConcurrentSkipListMap index = new ConcurrentSkipListMap<>(); + + public SkipListChunk(@WillCloseWhenClosed Chunk chunk, BufferAllocator allocator) { + super(chunk, allocator); + try (Snapshot snapshot = chunk.snapshot(allocator)) { + updateIndex(snapshot, 0); + } + } + + @Override + protected IntVector indexOf(Snapshot snapshot, BufferAllocator allocator) { + IntVector indexes = ArrowVectors.nonnullIntVector("indexes", allocator); + ArrowVectors.collect(indexes, index.values()); + return indexes; + } + + @Override + protected void updateIndex(Snapshot snapshot, int offset) { + for (int i = 0; i < snapshot.getValueCount(); i++) { + index.put(snapshot.getKey(i), i + offset); + } + } + + @Override + protected void deleteIndex(RangeSet rangeSet) { + rangeSet + .asRanges() + .forEach( + range -> { + subMapRefOf(index, range).clear(); + }); + } + + private static , V> NavigableMap subMapRefOf( + NavigableMap column, Range range) { + if (range.encloses(Range.all())) { + return column; + } else if (!range.hasLowerBound()) { + return column.headMap(range.upperEndpoint(), range.upperBoundType() == BoundType.CLOSED); + } else if (!range.hasUpperBound()) { + return column.tailMap(range.lowerEndpoint(), range.lowerBoundType() == BoundType.CLOSED); + } else { + return column.subMap( + range.lowerEndpoint(), + range.lowerBoundType() == BoundType.CLOSED, + range.upperEndpoint(), + range.upperBoundType() == BoundType.CLOSED); + } + } + + @Override + public void close() { + super.close(); + index.clear(); + } +} diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/compact/Flusher.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/compact/Flusher.java new file mode 100644 index 0000000000..0c1b00e29a --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/compact/Flusher.java @@ -0,0 +1,201 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cn.edu.tsinghua.iginx.parquet.db.lsm.compact; + +import cn.edu.tsinghua.iginx.parquet.db.lsm.buffer.MemTableQueue; +import cn.edu.tsinghua.iginx.parquet.db.lsm.table.MemoryTable; +import cn.edu.tsinghua.iginx.parquet.db.lsm.table.TableStorage; +import cn.edu.tsinghua.iginx.parquet.util.NoexceptAutoCloseable; +import cn.edu.tsinghua.iginx.parquet.util.Shared; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.*; +import javax.annotation.Nonnegative; +import javax.annotation.concurrent.NotThreadSafe; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.util.Preconditions; +import org.apache.arrow.vector.types.pojo.Field; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@NotThreadSafe +public class Flusher implements NoexceptAutoCloseable { + + private static final Logger LOGGER = LoggerFactory.getLogger(Flusher.class); + + private final String name; + private final Shared shared; + private final BufferAllocator allocator; + private final MemTableQueue memTableQueue; + private final TableStorage tableStorage; + + private ScheduledExecutorService scheduler; + private ExecutorService dispatcher; + private ExecutorService leader; + private ExecutorService worker; + private boolean running = false; + + public Flusher( + String name, + Shared shared, + BufferAllocator allocator, + MemTableQueue memTableQueue, + TableStorage tableStorage) { + this.name = name; + this.shared = shared; + this.allocator = + allocator.newChildAllocator(allocator.getName() + "-flusher", 0, Long.MAX_VALUE); + this.memTableQueue = memTableQueue; + this.tableStorage = tableStorage; + start(); + } + + @Override + public void close() { + stop(); + allocator.close(); + } + + public void start() { + Preconditions.checkState(!running, "flusher is already running"); + + ThreadFactory expireFactory = + new ThreadFactoryBuilder().setNameFormat("flusher-" + name + "-scheduler-%d").build(); + this.scheduler = Executors.newSingleThreadScheduledExecutor(expireFactory); + long timeout = shared.getStorageProperties().getWriteBufferTimeout().toMillis(); + if (timeout > 0) { + LOGGER.info("flusher {} start to force flush every {} ms", name, timeout); + this.scheduler.scheduleWithFixedDelay( + handleInterruption(memTableQueue::flush), timeout, timeout, TimeUnit.MILLISECONDS); + } + + ThreadFactory workerFactory = + new ThreadFactoryBuilder().setNameFormat("flusher-" + name + "-worker-%d").build(); + this.worker = Executors.newCachedThreadPool(workerFactory); + + ThreadFactory leaderFactory = + new ThreadFactoryBuilder().setNameFormat("flusher-" + name + "-leader-%d").build(); + this.leader = Executors.newCachedThreadPool(leaderFactory); + + ThreadFactory dispatcherFactory = + new ThreadFactoryBuilder().setNameFormat("flusher-" + name + "-dispatcher-%d").build(); + this.dispatcher = Executors.newSingleThreadExecutor(dispatcherFactory); + dispatcher.submit(handleInterruption((this::dispatch))); + + running = true; + } + + public void stop() { + Preconditions.checkState(running, "flusher is not running"); + + scheduler.shutdownNow(); + dispatcher.shutdownNow(); + worker.shutdownNow(); + leader.shutdownNow(); + + try { + boolean schedulerTerminated = scheduler.awaitTermination(1, TimeUnit.MINUTES); + boolean dispatcherTerminated = dispatcher.awaitTermination(1, TimeUnit.MINUTES); + boolean workerTerminated = worker.awaitTermination(1, TimeUnit.MINUTES); + boolean leaderTerminated = leader.awaitTermination(1, TimeUnit.MINUTES); + if (!schedulerTerminated || !dispatcherTerminated || !workerTerminated || !leaderTerminated) { + throw new IllegalStateException("flusher is not terminated"); + } + running = false; + } catch (InterruptedException e) { + LOGGER.debug("flusher is interrupted:", e); + } + } + + interface InterruptibleRunnable { + void run() throws InterruptedException, ExecutionException; + } + + private Runnable handleInterruption(InterruptibleRunnable runnable) { + return () -> { + try { + runnable.run(); + } catch (InterruptedException e) { + LOGGER.debug("interrupted", e); + } catch (Exception e) { + LOGGER.error("unexpected error", e); + } + }; + } + + private void dispatch() throws InterruptedException { + long memtableIdAtLeast = 0; + while (!Thread.currentThread().isInterrupted()) { + long memtableId = memTableQueue.awaitNext(memtableIdAtLeast); + LOGGER.debug("memtable {} is ready to flush", memtableId); + CountDownLatch pullNext = new CountDownLatch(1); + leader.submit(handleInterruption(() -> submitAndWaitFlush(memtableId, pullNext))); + pullNext.await(); + memtableIdAtLeast = memtableId + 1; + } + throw new InterruptedException(); + } + + private void submitAndWaitFlush(@Nonnegative long memtableId, CountDownLatch onSubmit) + throws InterruptedException, ExecutionException { + List tableNames = new ArrayList<>(); + + LOGGER.debug("start to flush memtable {}", memtableId); + + try (MemoryTable snapshot = memTableQueue.snapshot(memtableId, allocator)) { + List>> futureFlushed = new ArrayList<>(); + + Field[] fields = snapshot.getFields().toArray(new Field[0]); + for (int columnNumber = 0; columnNumber < fields.length; columnNumber++) { + Field field = fields[columnNumber]; + String suffix = String.valueOf(columnNumber); + + shared.getFlusherPermits().acquire(); + Future> future = + worker.submit( + () -> { + try { + return tableStorage.flush(memtableId, suffix, snapshot.subTable(field)); + } finally { + shared.getFlusherPermits().release(); + } + }); + futureFlushed.add(future); + } + + onSubmit.countDown(); + + for (Future> future : futureFlushed) { + tableNames.addAll(future.get()); + } + } + + LOGGER.debug("memtable {} is flushed to tables {}", memtableId, tableNames); + + memTableQueue.eliminate( + memtableId, + tombstone -> { + for (String tableName : tableNames) { + tableStorage.commit(tableName, tombstone); + } + }); + + LOGGER.debug("memtable {} is eliminated", memtableId); + } +} diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/table/DeletedTable.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/table/DeletedTable.java new file mode 100644 index 0000000000..d2bcee8d6b --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/table/DeletedTable.java @@ -0,0 +1,53 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.parquet.db.lsm.table; + +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; +import cn.edu.tsinghua.iginx.parquet.db.lsm.api.TableMeta; +import cn.edu.tsinghua.iginx.parquet.db.util.AreaSet; +import cn.edu.tsinghua.iginx.parquet.db.util.iterator.AreaFilterScanner; +import cn.edu.tsinghua.iginx.parquet.db.util.iterator.Scanner; +import com.google.common.collect.RangeSet; +import java.io.IOException; +import java.util.Set; +import javax.annotation.Nullable; + +public class DeletedTable implements Table { + + private final Table table; + + private final AreaSet deleted; + + public DeletedTable(Table table, AreaSet deleted) { + this.table = table; + this.deleted = deleted; + } + + @Override + public TableMeta getMeta() throws IOException { + return new DeletedTableMeta(table.getMeta(), deleted); + } + + @Override + public Scanner> scan( + Set fields, RangeSet range, @Nullable Filter superSetPredicate) + throws IOException { + return new AreaFilterScanner<>(table.scan(fields, range, superSetPredicate), deleted); + } +} diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/table/DeletedTableMeta.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/table/DeletedTableMeta.java new file mode 100644 index 0000000000..f47dfc8fa5 --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/table/DeletedTableMeta.java @@ -0,0 +1,104 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.parquet.db.lsm.table; + +import cn.edu.tsinghua.iginx.parquet.db.lsm.api.TableMeta; +import cn.edu.tsinghua.iginx.parquet.db.util.AreaSet; +import cn.edu.tsinghua.iginx.thrift.DataType; +import com.google.common.collect.Range; +import com.google.common.collect.RangeSet; +import com.google.common.collect.TreeRangeSet; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.NoSuchElementException; +import javax.annotation.Nullable; + +public class DeletedTableMeta implements TableMeta { + private final Map schema; + private final Map> ranges; + private final Map counts; + + public DeletedTableMeta(TableMeta tableMeta, AreaSet tombstone) { + this.schema = new HashMap<>(tableMeta.getSchema()); + schema.keySet().removeAll(tombstone.getFields()); + + this.counts = new HashMap<>(); + + Map> rangeSetMap = new HashMap<>(); + for (Map.Entry entry : schema.entrySet()) { + String field = entry.getKey(); + Range range = tableMeta.getRange(field); + rangeSetMap.put(field, TreeRangeSet.create(Collections.singleton(range))); + Long count = tableMeta.getValueCount(field); + if (count != null) { + counts.put(field, count); + } + } + + RangeSet deletedKeys = tombstone.getKeys(); + if (!deletedKeys.isEmpty()) { + counts.clear(); + rangeSetMap.values().forEach(rangeSet -> rangeSet.removeAll(deletedKeys)); + } + + for (Map.Entry> entry : tombstone.getSegments().entrySet()) { + String field = entry.getKey(); + RangeSet rangeSetDeleted = entry.getValue(); + RangeSet rangeSet = rangeSetMap.get(field); + if (rangeSet != null) { + rangeSet.removeAll(rangeSetDeleted); + } + counts.remove(field); + } + + this.ranges = new HashMap<>(); + for (Map.Entry> entry : rangeSetMap.entrySet()) { + String field = entry.getKey(); + RangeSet rangeSet = entry.getValue(); + if (!rangeSet.isEmpty()) { + ranges.put(field, rangeSet.span()); + } else { + ranges.put(field, Range.closedOpen(0L, 0L)); + } + } + } + + @Override + public Map getSchema() { + return schema; + } + + @Override + public Range getRange(String field) { + if (!schema.containsKey(field)) { + throw new NoSuchElementException(); + } + return ranges.get(field); + } + + @Override + @Nullable + public Long getValueCount(String field) { + if (!schema.containsKey(field)) { + throw new NoSuchElementException(); + } + return counts.get(field); + } +} diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/table/FileTable.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/table/FileTable.java new file mode 100644 index 0000000000..43478aca33 --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/table/FileTable.java @@ -0,0 +1,64 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.parquet.db.lsm.table; + +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; +import cn.edu.tsinghua.iginx.parquet.db.lsm.api.ReadWriter; +import cn.edu.tsinghua.iginx.parquet.db.lsm.api.TableMeta; +import cn.edu.tsinghua.iginx.parquet.db.util.iterator.Scanner; +import com.google.common.collect.RangeSet; +import java.io.IOException; +import java.util.Set; +import java.util.StringJoiner; +import javax.annotation.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class FileTable implements Table { + private static final Logger LOGGER = LoggerFactory.getLogger(FileTable.class); + private final String tableName; + + private final ReadWriter readWriter; + + public FileTable(String tableName, ReadWriter readWriter) { + this.tableName = tableName; + this.readWriter = readWriter; + } + + @Override + public TableMeta getMeta() throws IOException { + return readWriter.readMeta(tableName); + } + + @Override + public Scanner> scan( + Set fields, RangeSet ranges, @Nullable Filter superSetPredicate) + throws IOException { + LOGGER.debug("read {} where {} & {} from {}", fields, ranges, superSetPredicate, tableName); + return readWriter.scanData(tableName, fields, ranges, superSetPredicate); + } + + @Override + public String toString() { + return new StringJoiner(", ", FileTable.class.getSimpleName() + "[", "]") + .add("tableName='" + tableName + "'") + .add("readWriter=" + readWriter) + .toString(); + } +} diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/table/MemoryTable.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/table/MemoryTable.java new file mode 100644 index 0000000000..779e4cab06 --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/table/MemoryTable.java @@ -0,0 +1,195 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.parquet.db.lsm.table; + +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; +import cn.edu.tsinghua.iginx.parquet.db.lsm.api.TableMeta; +import cn.edu.tsinghua.iginx.parquet.db.lsm.buffer.MemColumn; +import cn.edu.tsinghua.iginx.parquet.db.util.iterator.*; +import cn.edu.tsinghua.iginx.parquet.db.util.iterator.Scanner; +import cn.edu.tsinghua.iginx.parquet.manager.utils.TagKVUtils; +import cn.edu.tsinghua.iginx.parquet.util.NoexceptAutoCloseable; +import cn.edu.tsinghua.iginx.parquet.util.SingleCache; +import cn.edu.tsinghua.iginx.parquet.util.arrow.ArrowFields; +import cn.edu.tsinghua.iginx.thrift.DataType; +import com.google.common.collect.Range; +import com.google.common.collect.RangeSet; +import java.util.*; +import java.util.stream.Collectors; +import javax.annotation.Nullable; +import javax.annotation.WillCloseWhenClosed; +import org.apache.arrow.vector.types.pojo.Field; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MemoryTable implements Table, NoexceptAutoCloseable { + private static final Logger LOGGER = LoggerFactory.getLogger(MemoryTable.class); + + private final LinkedHashMap columns; + private final Map fieldMap = new HashMap<>(); + private final SingleCache meta = + new SingleCache<>(() -> new MemoryTableMeta(getSchema(), getRanges(), getCounts())); + + public MemoryTable(@WillCloseWhenClosed LinkedHashMap columns) { + this.columns = new LinkedHashMap<>(columns); + for (Field field : columns.keySet()) { + fieldMap.put(getFieldString(field), field); + } + } + + public static MemoryTable empty() { + return new MemoryTable(new LinkedHashMap<>()); + } + + public Set getFields() { + return columns.keySet(); + } + + public boolean isEmpty() { + return columns.isEmpty(); + } + + private Map getSchema() { + return (Map) ArrowFields.toIginxSchema(columns.keySet()); + } + + private Map> getRanges() { + return columns.keySet().stream() + .collect(Collectors.toMap(this::getFieldString, this::getRange)); + } + + private Map getCounts() { + // TODO: give statistics + return Collections.emptyMap(); + } + + private String getFieldString(Field field) { + return TagKVUtils.toFullName(ArrowFields.toColumnKey(field)); + } + + private Range getRange(Field field) { + MemColumn.Snapshot snapshot = columns.get(field); + RangeSet ranges = snapshot.getRanges(); + if (ranges.isEmpty()) { + return Range.closed(0L, 0L); + } + return ranges.span(); + } + + @Override + public String toString() { + return new StringJoiner(", ", MemoryTable.class.getSimpleName() + "[", "]") + .add("meta=" + meta) + .toString(); + } + + @Override + public TableMeta getMeta() { + return meta.get(); + } + + @Override + public Scanner> scan( + Set fields, RangeSet ranges, @Nullable Filter superSetPredicate) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("read {} where {} from {}", fields, ranges, meta); + } + Map> columns = new HashMap<>(); + for (String field : fields) { + if (!fieldMap.containsKey(field)) { + continue; + } + Field arrowField = fieldMap.get(field); + MemColumn.Snapshot snapshot = this.columns.get(arrowField); + columns.put(field, scan(snapshot, ranges)); + } + return new ColumnUnionRowScanner<>(columns); + } + + private Scanner scan(MemColumn.Snapshot snapshot, RangeSet ranges) { + if (ranges.isEmpty()) { + return new EmptyScanner<>(); + } + MemColumn.Snapshot sliced = snapshot.slice(ranges); + return new ListenCloseScanner<>(new IteratorScanner<>(sliced.iterator()), sliced::close); + } + + @Override + public void close() { + columns.values().forEach(MemColumn.Snapshot::close); + columns.clear(); + } + + public MemoryTable subTable(Field field) { + LinkedHashMap subColumns = new LinkedHashMap<>(); + subColumns.put(field, columns.get(field)); + return new MemoryTable(subColumns) { + @Override + public void close() { + // do nothing + } + }; + } + + public static class MemoryTableMeta implements TableMeta { + + private final Map schema; + private final Map> ranges; + private final Map counts; + + MemoryTableMeta( + Map schema, Map> ranges, Map counts) { + this.schema = Collections.unmodifiableMap(schema); + this.ranges = Collections.unmodifiableMap(ranges); + this.counts = Collections.unmodifiableMap(counts); + } + + public Map getSchema() { + return schema; + } + + @Override + public Range getRange(String field) { + if (!schema.containsKey(field)) { + throw new NoSuchElementException(); + } + return Objects.requireNonNull(ranges.get(field)); + } + + @Override + public Long getValueCount(String field) { + if (!schema.containsKey(field)) { + throw new NoSuchElementException(); + } + return counts.get(field); + } + + public Map> getRanges() { + return ranges; + } + + @Override + public String toString() { + return new StringJoiner(", ", MemoryTableMeta.class.getSimpleName() + "[", "]") + .add("schema=" + schema) + .add("ranges=" + ranges) + .toString(); + } + } +} diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/table/Table.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/table/Table.java new file mode 100644 index 0000000000..7cfa02713b --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/table/Table.java @@ -0,0 +1,55 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.parquet.db.lsm.table; + +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; +import cn.edu.tsinghua.iginx.parquet.db.lsm.api.TableMeta; +import cn.edu.tsinghua.iginx.parquet.db.util.iterator.LazyRowScanner; +import cn.edu.tsinghua.iginx.parquet.db.util.iterator.Scanner; +import cn.edu.tsinghua.iginx.parquet.util.exception.StorageException; +import com.google.common.collect.RangeSet; +import java.io.IOException; +import java.util.Set; +import javax.annotation.Nullable; + +public interface Table { + + TableMeta getMeta() throws IOException; + + Scanner> scan( + Set fields, RangeSet range, @Nullable Filter superSetPredicate) + throws IOException; + + default Scanner> scan(Set fields, RangeSet ranges) + throws IOException { + return scan(fields, ranges, null); + } + + default Scanner> lazyScan( + Set fields, RangeSet ranges) { + return new LazyRowScanner<>( + () -> { + try { + return scan(fields, ranges); + } catch (IOException e) { + throw new StorageException(e); + } + }); + } +} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/table/TableIndex.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/table/TableIndex.java similarity index 60% rename from dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/table/TableIndex.java rename to dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/table/TableIndex.java index fe3afd87e0..41a1ec3d98 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/table/TableIndex.java +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/table/TableIndex.java @@ -1,17 +1,19 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.parquet.db.lsm.table; @@ -21,6 +23,7 @@ import cn.edu.tsinghua.iginx.parquet.util.exception.NotIntegrityException; import cn.edu.tsinghua.iginx.parquet.util.exception.StorageRuntimeException; import cn.edu.tsinghua.iginx.parquet.util.exception.TypeConflictedException; +import cn.edu.tsinghua.iginx.thrift.DataType; import com.google.common.collect.ImmutableRangeSet; import com.google.common.collect.Range; import com.google.common.collect.RangeSet; @@ -33,16 +36,17 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class TableIndex, F, T, V> implements AutoCloseable { +public class TableIndex { + private static final Logger LOGGER = LoggerFactory.getLogger(TableIndex.class); private final ReadWriteLock lock = new ReentrantReadWriteLock(true); - private final Map> indexes = new HashMap<>(); + private final Map indexes = new HashMap<>(); - public TableIndex(TableStorage tableStorage) { + public TableIndex(TableStorage tableStorage) { try { for (String tableName : tableStorage.tableNames()) { - TableMeta meta = tableStorage.get(tableName).getMeta(); + TableMeta meta = tableStorage.getMeta(tableName); declareFields(meta.getSchema()); addTable(tableName, meta); } @@ -51,27 +55,27 @@ public TableIndex(TableStorage tableStorage) { } } - public Set find(AreaSet areas) { + public Set find(AreaSet areas) { Set result = new HashSet<>(); lock.readLock().lock(); try { - RangeSet rangeSet = areas.getKeys(); + RangeSet rangeSet = areas.getKeys(); if (!rangeSet.isEmpty()) { - for (FieldIndex fieldIndex : indexes.values()) { + for (FieldIndex fieldIndex : indexes.values()) { Set tables = fieldIndex.find(rangeSet); result.addAll(tables); } } - for (F field : areas.getFields()) { - FieldIndex fieldIndex = indexes.get(field); + for (String field : areas.getFields()) { + FieldIndex fieldIndex = indexes.get(field); if (fieldIndex == null) { continue; } Set tables = fieldIndex.find(); result.addAll(tables); } - for (Map.Entry> entry : areas.getSegments().entrySet()) { - FieldIndex fieldIndex = indexes.get(entry.getKey()); + for (Map.Entry> entry : areas.getSegments().entrySet()) { + FieldIndex fieldIndex = indexes.get(entry.getKey()); if (fieldIndex == null) { continue; } @@ -84,12 +88,12 @@ public Set find(AreaSet areas) { return result; } - public Map> ranges() { - Map> result = new HashMap<>(); + public Map> ranges() { + Map> result = new HashMap<>(); lock.readLock().lock(); try { - for (Map.Entry> entry : indexes.entrySet()) { - RangeSet rangeSet = entry.getValue().ranges(); + for (Map.Entry entry : indexes.entrySet()) { + RangeSet rangeSet = entry.getValue().ranges(); if (!rangeSet.isEmpty()) { result.put(entry.getKey(), rangeSet.span()); } @@ -100,19 +104,19 @@ public Map> ranges() { return result; } - public void declareFields(Map schema) throws TypeConflictedException { + public void declareFields(Map schema) throws TypeConflictedException { boolean hasNewField = checkOldFields(schema); if (hasNewField) { declareNewFields(schema); } } - private boolean checkOldFields(Map schema) throws TypeConflictedException { + private boolean checkOldFields(Map schema) throws TypeConflictedException { boolean hasNewField = false; lock.readLock().lock(); try { - for (Map.Entry entry : schema.entrySet()) { - FieldIndex fieldIndex = indexes.get(entry.getKey()); + for (Map.Entry entry : schema.entrySet()) { + FieldIndex fieldIndex = indexes.get(entry.getKey()); if (fieldIndex == null) { hasNewField = true; } else if (!fieldIndex.getType().equals(entry.getValue())) { @@ -128,12 +132,12 @@ private boolean checkOldFields(Map schema) throws TypeConflictedException return hasNewField; } - private void declareNewFields(Map schema) throws TypeConflictedException { - Map newSchema = new HashMap<>(); + private void declareNewFields(Map schema) throws TypeConflictedException { + Map newSchema = new HashMap<>(); lock.writeLock().lock(); try { - for (Map.Entry entry : schema.entrySet()) { - FieldIndex fieldIndex = indexes.get(entry.getKey()); + for (Map.Entry entry : schema.entrySet()) { + FieldIndex fieldIndex = indexes.get(entry.getKey()); if (fieldIndex == null) { newSchema.put(entry.getKey(), entry.getValue()); } else if (!fieldIndex.getType().equals(entry.getValue())) { @@ -144,20 +148,20 @@ private void declareNewFields(Map schema) throws TypeConflictedException { } } LOGGER.debug("declare new fields: {}", newSchema); - for (Map.Entry entry : newSchema.entrySet()) { - indexes.put(entry.getKey(), new FieldIndex<>(entry.getValue())); + for (Map.Entry entry : newSchema.entrySet()) { + indexes.put(entry.getKey(), new FieldIndex(entry.getValue())); } } finally { lock.writeLock().unlock(); } } - public Map getType(Set fields, Consumer processMissingField) { - Map result = new HashMap<>(); + public Map getType(Set fields, Consumer processMissingField) { + Map result = new HashMap<>(); lock.readLock().lock(); try { - for (F field : fields) { - FieldIndex fieldIndex = indexes.get(field); + for (String field : fields) { + FieldIndex fieldIndex = indexes.get(field); if (fieldIndex == null) { processMissingField.accept(field); } else { @@ -170,11 +174,11 @@ public Map getType(Set fields, Consumer processMissingField) { return result; } - public Map getType() { - Map result = new HashMap<>(); + public Map getType() { + Map result = new HashMap<>(); lock.readLock().lock(); try { - for (Map.Entry> entry : indexes.entrySet()) { + for (Map.Entry entry : indexes.entrySet()) { result.put(entry.getKey(), entry.getValue().getType()); } } finally { @@ -183,18 +187,17 @@ public Map getType() { return result; } - public void addTable(String name, TableMeta meta) { + public void addTable(String name, TableMeta meta) { lock.readLock().lock(); try { - Map types = meta.getSchema(); - for (Map.Entry> entry : meta.getRanges().entrySet()) { - F field = entry.getKey(); - FieldIndex fieldIndex = indexes.get(field); + Map types = meta.getSchema(); + for (String field : types.keySet()) { + FieldIndex fieldIndex = indexes.get(field); if (fieldIndex == null) { throw new NotIntegrityException("field " + field + " is not found in schema"); } - T oldType = fieldIndex.getType(); - T newType = types.get(field); + DataType oldType = fieldIndex.getType(); + DataType newType = types.get(field); if (!oldType.equals(newType)) { throw new NotIntegrityException( "field " @@ -204,8 +207,8 @@ public void addTable(String name, TableMeta meta) { + ", new type: " + newType); } - Range range = entry.getValue(); - fieldIndex.addTable(name, range); + Range range = meta.getRange(field); + fieldIndex.addTable(name, Objects.requireNonNull(range)); } } finally { lock.readLock().unlock(); @@ -215,7 +218,7 @@ public void addTable(String name, TableMeta meta) { public void removeTable(String name) { lock.readLock().lock(); try { - for (FieldIndex fieldIndex : indexes.values()) { + for (FieldIndex fieldIndex : indexes.values()) { fieldIndex.removeTable(name); } } finally { @@ -223,15 +226,15 @@ public void removeTable(String name) { } } - public void delete(AreaSet areas) { + public void delete(AreaSet areas) { lock.writeLock().lock(); try { indexes.keySet().removeAll(areas.getFields()); - for (FieldIndex fieldIndex : indexes.values()) { + for (FieldIndex fieldIndex : indexes.values()) { fieldIndex.delete(areas.getKeys()); } - for (Map.Entry> entry : areas.getSegments().entrySet()) { - FieldIndex fieldIndex = indexes.get(entry.getKey()); + for (Map.Entry> entry : areas.getSegments().entrySet()) { + FieldIndex fieldIndex = indexes.get(entry.getKey()); if (fieldIndex != null) { fieldIndex.delete(entry.getValue()); } @@ -250,25 +253,20 @@ public void clear() { } } - @Override - public void close() { - // do nothing - } - - public static class FieldIndex, F, T, V> { + public static class FieldIndex { private final ReadWriteLock lock = new ReentrantReadWriteLock(true); - private final T type; - private final Map> tableRange = new HashMap<>(); + private final DataType type; + private final Map> tableRange = new HashMap<>(); - public FieldIndex(T type) { + public FieldIndex(DataType type) { this.type = type; } - public T getType() { + public DataType getType() { return type; } - public void addTable(String name, Range range) { + public void addTable(String name, Range range) { lock.writeLock().lock(); try { if (this.tableRange.containsKey(name)) { @@ -289,11 +287,11 @@ public void removeTable(String name) { } } - public Set find(RangeSet ranges) { + public Set find(RangeSet ranges) { Set result = new HashSet<>(); lock.readLock().lock(); try { - for (Map.Entry> entry : tableRange.entrySet()) { + for (Map.Entry> entry : tableRange.entrySet()) { if (ranges.intersects(entry.getValue())) { result.add(entry.getKey()); } @@ -315,14 +313,14 @@ public Set find() { return result; } - public void delete(RangeSet ranges) { + public void delete(RangeSet ranges) { lock.writeLock().lock(); try { - RangeSet validRanges = ranges.complement(); - Iterator>> iterator = tableRange.entrySet().iterator(); - Map> overlap = new HashMap<>(); + RangeSet validRanges = ranges.complement(); + Iterator>> iterator = tableRange.entrySet().iterator(); + Map> overlap = new HashMap<>(); while (iterator.hasNext()) { - Map.Entry> entry = iterator.next(); + Map.Entry> entry = iterator.next(); if (!ranges.intersects(entry.getValue())) { continue; } @@ -338,11 +336,11 @@ public void delete(RangeSet ranges) { } } - public RangeSet ranges() { - TreeRangeSet rangeSet = TreeRangeSet.create(); + public RangeSet ranges() { + TreeRangeSet rangeSet = TreeRangeSet.create(); lock.readLock().lock(); try { - for (Range range : tableRange.values()) { + for (Range range : tableRange.values()) { rangeSet.add(range); } } finally { diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/table/TableStorage.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/table/TableStorage.java new file mode 100644 index 0000000000..c9369b1aae --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/table/TableStorage.java @@ -0,0 +1,311 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.parquet.db.lsm.table; + +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; +import cn.edu.tsinghua.iginx.parquet.db.lsm.api.ReadWriter; +import cn.edu.tsinghua.iginx.parquet.db.lsm.api.TableMeta; +import cn.edu.tsinghua.iginx.parquet.db.lsm.buffer.DataBuffer; +import cn.edu.tsinghua.iginx.parquet.db.util.AreaSet; +import cn.edu.tsinghua.iginx.parquet.db.util.iterator.ConcatScanner; +import cn.edu.tsinghua.iginx.parquet.db.util.iterator.EmtpyHeadRowScanner; +import cn.edu.tsinghua.iginx.parquet.db.util.iterator.RowUnionScanner; +import cn.edu.tsinghua.iginx.parquet.db.util.iterator.Scanner; +import cn.edu.tsinghua.iginx.parquet.util.Shared; +import cn.edu.tsinghua.iginx.parquet.util.arrow.ArrowFields; +import cn.edu.tsinghua.iginx.parquet.util.exception.StorageException; +import cn.edu.tsinghua.iginx.parquet.util.exception.TypeConflictedException; +import cn.edu.tsinghua.iginx.thrift.DataType; +import com.google.common.collect.*; +import java.io.IOException; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.StreamSupport; +import org.apache.arrow.vector.types.pojo.Field; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TableStorage implements AutoCloseable { + private static final Logger LOGGER = LoggerFactory.getLogger(TableStorage.class); + + private final TableIndex tableIndex; + private final ReadWriter readWriter; + private long sqnBase; + + public TableStorage(Shared shared, ReadWriter readWriter) throws IOException { + this.readWriter = readWriter; + + Iterable tableNames = readWriter.tableNames(); + String last = + StreamSupport.stream(tableNames.spliterator(), false) + .max(Comparator.naturalOrder()) + .orElse("0-0"); + this.sqnBase = getSeq(last) + 1; + this.tableIndex = new TableIndex(this); + } + + static long getSeq(String tableName) { + Pattern pattern = Pattern.compile("^(\\d+)-.*$"); + Matcher matcher = pattern.matcher(tableName); + if (matcher.find()) { + return Long.parseLong(matcher.group(1)); + } else { + throw new IllegalArgumentException("invalid table name: " + tableName); + } + } + + public String getTableName(long sqn, String suffix) { + return String.format("%019d-%s", sqnBase + sqn, suffix); + } + + public List flush(long sqn, String suffix, MemoryTable table) + throws InterruptedException { + if (table.isEmpty()) { + return Collections.emptyList(); + } + String name = getTableName(sqn, suffix); + TableMeta meta = table.getMeta(); + try (Scanner> scanner = + table.scan(meta.getSchema().keySet(), ImmutableRangeSet.of(Range.all()))) { + readWriter.flush(name, meta, scanner); + } catch (IOException | StorageException e) { + LOGGER.error("flush table {} failed", name, e); + } + return Collections.singletonList(name); + } + + public void commit(String table, AreaSet tombstone) { + AreaSet innerTombstone = ArrowFields.toInnerAreas(tombstone); + try { + if (innerTombstone.isAll()) { + readWriter.delete(table); + return; + } + if (!innerTombstone.isEmpty()) { + readWriter.delete(table, innerTombstone); + } + TableMeta meta = readWriter.readMeta(table); + tableIndex.addTable(table, meta); + } catch (IOException e) { + LOGGER.error("commit table {} failed", table, e); + } + } + + public void clear() { + sqnBase = 0; + tableIndex.clear(); + try { + readWriter.clear(); + } catch (IOException e) { + LOGGER.error("clear failed", e); + } + } + + public TableMeta getMeta(String tableName) throws IOException { + return new FileTable(tableName, readWriter).getMeta(); + } + + public void delete(AreaSet areas) throws IOException { + Set tables = tableIndex.find(areas); + tableIndex.delete(areas); + for (String tableName : tables) { + readWriter.delete(tableName, areas); + } + } + + public Iterable tableNames() throws IOException { + return readWriter.tableNames(); + } + + @Override + public void close() {} + + public Map schema() { + return tableIndex.getType(); + } + + public void declareFields(Map schema) throws TypeConflictedException { + tableIndex.declareFields(schema); + } + + public DataBuffer query( + Set fields, RangeSet ranges, Filter filter) + throws StorageException, IOException { + + AreaSet areas = new AreaSet<>(); + areas.add(fields, ranges); + DataBuffer buffer = new DataBuffer<>(); + + Set tables = tableIndex.find(areas); + List sortedTableNames = new ArrayList<>(tables); + sortedTableNames.sort(Comparator.naturalOrder()); + + for (String tableName : sortedTableNames) { + try (Scanner> scanner = scan(tableName, fields, ranges)) { + buffer.putRows(scanner); + } + } + + return buffer; + } + + private Scanner> scan( + String tableName, Set fields, RangeSet ranges) throws IOException { + return new FileTable(tableName, readWriter).scan(fields, ranges); + } + + public Map count(Set innerFields) throws StorageException, IOException { + Map counts = new HashMap<>(); + + for (String field : innerFields) { + long count = count(field); + counts.put(field, count); + } + + return counts; + } + + public long count(String field) throws StorageException, IOException { + RangeMap> regionTableLists = getTablesGroupByRegion(field); + + long totalCount = 0; + for (List tables : regionTableLists.asMapOfRanges().values()) { + if (tables.isEmpty()) { + continue; + } else if (tables.size() == 1) { + TableMeta meta = readWriter.readMeta(tables.get(0)); + Long regionCount = meta.getValueCount(field); + if (regionCount != null) { + totalCount += regionCount; + continue; + } + } + totalCount += getOverlapCount(field, tables); + } + return totalCount; + } + + private static Range normalize(Range range) { + if (range.isEmpty()) { + return range; + } + long lower = range.lowerEndpoint(); + long upper = range.upperEndpoint(); + if (range.lowerBoundType() == BoundType.OPEN) { + lower++; + } + if (range.upperBoundType() == BoundType.OPEN) { + upper--; + } + return Range.closed(lower, upper); + } + + private RangeMap> getTablesGroupByRegion(String field) throws IOException { + AreaSet areas = new AreaSet<>(); + areas.add(Collections.singleton(field), ImmutableRangeSet.of(Range.all())); + Set tables = tableIndex.find(areas); + List sortedTableNames = new ArrayList<>(tables); + sortedTableNames.sort(Comparator.naturalOrder()); + + HashMap> tableRanges = new HashMap<>(); + for (String tableName : sortedTableNames) { + TableMeta meta = readWriter.readMeta(tableName); + Range range = meta.getRange(field); + tableRanges.put(tableName, range); + } + + RangeSet regions = TreeRangeSet.create(tableRanges.values()); + RangeMap> regionTableLists = TreeRangeMap.create(); + for (Range region : regions.asRanges()) { + regionTableLists.put(region, new ArrayList<>()); + } + + for (String tableName : sortedTableNames) { + Range range = normalize(tableRanges.get(tableName)); + List regionTableList = regionTableLists.get(range.lowerEndpoint()); + assert regionTableList != null; + regionTableList.add(tableName); + } + + return regionTableLists; + } + + private long getOverlapCount(String field, List sortedTableNames) + throws IOException, StorageException { + Set fields = Collections.singleton(field); + + long count = 0; + try (Scanner> scanner = scan(sortedTableNames, fields)) { + while (scanner.iterate()) { + Scanner row = scanner.value(); + while (row.iterate()) { + assert row.value() != null; + count++; + } + } + } + return count; + } + + private Scanner> scan(List tableNames, Set fields) + throws IOException, StorageException { + List tables = new ArrayList<>(); + for (String tableName : tableNames) { + tables.add(new FileTable(tableName, readWriter)); + } + List>> overlaps = getOverlapScannerList(fields, tables); + + Collections.reverse(overlaps); + return new RowUnionScanner<>(overlaps); + } + + private static List>> getOverlapScannerList( + Set fields, List tables) throws IOException { + RangeSet ranges = ImmutableRangeSet.of(Range.all()); + List>> overlaps = new ArrayList<>(); + + RangeSet tableRanges = TreeRangeSet.create(); + List>> noOverlaps = new ArrayList<>(); + for (FileTable table : tables) { + TableMeta meta = table.getMeta(); + Range range = normalize(meta.getRange(fields)); + if (range.isEmpty()) { + continue; + } + if (tableRanges.intersects(range)) { + overlaps.add(new ConcatScanner<>(noOverlaps.iterator())); + noOverlaps = new ArrayList<>(); + tableRanges = TreeRangeSet.create(); + } + long head = range.lowerEndpoint(); + Scanner> lazy = table.lazyScan(fields, ranges); + Scanner> emptyHead = new EmtpyHeadRowScanner<>(head); + Scanner> concat = + new ConcatScanner<>(Iterators.forArray(emptyHead, lazy)); + noOverlaps.add(concat); + tableRanges.add(range); + } + if (!noOverlaps.isEmpty()) { + overlaps.add(new ConcatScanner<>(noOverlaps.iterator())); + } + + return overlaps; + } +} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/AreaSet.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/AreaSet.java similarity index 67% rename from dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/AreaSet.java rename to dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/AreaSet.java index ff0908a3e7..ccdbde91c5 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/AreaSet.java +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/AreaSet.java @@ -1,21 +1,24 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.parquet.db.util; +import com.google.common.collect.Range; import com.google.common.collect.RangeSet; import com.google.common.collect.TreeRangeSet; import java.util.*; @@ -31,6 +34,20 @@ public AreaSet() { this(new HashMap<>(), new HashSet<>(), TreeRangeSet.create()); } + public static , F> AreaSet create(AreaSet areas) { + Map> deletedRanges = new HashMap<>(); + for (Map.Entry> entry : areas.deletedRanges.entrySet()) { + deletedRanges.put(entry.getKey(), TreeRangeSet.create(entry.getValue())); + } + return new AreaSet<>( + deletedRanges, new HashSet<>(areas.deletedColumns), TreeRangeSet.create(areas.deletedRows)); + } + + public static , F> AreaSet all() { + return new AreaSet( + new HashMap<>(), new HashSet<>(), TreeRangeSet.create(Collections.singleton(Range.all()))); + } + private AreaSet( Map> deletedRanges, Set deletedColumns, RangeSet deletedRows) { this.deletedRanges = deletedRanges; @@ -101,6 +118,10 @@ public boolean isEmpty() { return deletedRanges.isEmpty() && deletedColumns.isEmpty() && deletedRows.isEmpty(); } + public boolean isAll() { + return deletedRows.encloses(Range.all()); + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/SequenceGenerator.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/SequenceGenerator.java new file mode 100644 index 0000000000..ac1013bf38 --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/SequenceGenerator.java @@ -0,0 +1,38 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.parquet.db.util; + +import java.util.concurrent.atomic.AtomicLong; + +public class SequenceGenerator { + + private final AtomicLong current = new AtomicLong(); + + public long next() { + return current.incrementAndGet(); + } + + public void reset() { + reset(new AtomicLong().get()); + } + + public void reset(long last) { + current.set(last); + } +} diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/WriteBatches.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/WriteBatches.java new file mode 100644 index 0000000000..bd88c9af47 --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/WriteBatches.java @@ -0,0 +1,138 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cn.edu.tsinghua.iginx.parquet.db.util; + +import cn.edu.tsinghua.iginx.engine.physical.storage.domain.ColumnKey; +import cn.edu.tsinghua.iginx.parquet.db.lsm.buffer.chunk.Chunk; +import cn.edu.tsinghua.iginx.parquet.db.util.iterator.Scanner; +import cn.edu.tsinghua.iginx.parquet.manager.utils.TagKVUtils; +import cn.edu.tsinghua.iginx.parquet.util.arrow.ArrowFields; +import cn.edu.tsinghua.iginx.parquet.util.arrow.ArrowTypes; +import cn.edu.tsinghua.iginx.parquet.util.arrow.ArrowVectors; +import cn.edu.tsinghua.iginx.parquet.util.exception.StorageException; +import cn.edu.tsinghua.iginx.thrift.DataType; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Consumer; +import java.util.stream.Collectors; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.BigIntVector; +import org.apache.arrow.vector.ValueVector; +import org.apache.arrow.vector.complex.writer.FieldWriter; +import org.apache.arrow.vector.types.Types; + +public class WriteBatches { + + private static class ChunkSnapshotBuilder { + private final BigIntVector keyVector; + private final ValueVector valueVector; + private final FieldWriter valueWriter; + private final Consumer valueAppender; + private int count = 0; + + public ChunkSnapshotBuilder(ColumnKey columnKey, DataType type, BufferAllocator allocator) { + this.keyVector = ArrowVectors.key(allocator); + Types.MinorType minorType = ArrowTypes.minorTypeOf(type); + this.valueVector = + minorType.getNewVector(ArrowFields.of(false, columnKey, type), allocator, null); + this.valueWriter = minorType.getNewFieldWriter(valueVector); + switch (type) { + case BOOLEAN: + valueAppender = value -> valueWriter.writeBit(((Boolean) value) ? 1 : 0); + break; + case INTEGER: + valueAppender = value -> valueWriter.writeInt((Integer) value); + break; + case LONG: + valueAppender = value -> valueWriter.writeBigInt((Long) value); + break; + case FLOAT: + valueAppender = value -> valueWriter.writeFloat4((Float) value); + break; + case DOUBLE: + valueAppender = value -> valueWriter.writeFloat8((Double) value); + break; + case BINARY: + valueAppender = value -> valueWriter.writeVarBinary((byte[]) value); + break; + default: + throw new UnsupportedOperationException("Unsupported data type: " + type); + } + } + + public void append(long key, Object value) { + keyVector.setSafe(count, key); + valueWriter.setPosition(count); + valueAppender.accept(value); + count++; + } + + public Chunk.Snapshot build() { + keyVector.setValueCount(count); + valueVector.setValueCount(count); + return new Chunk.Snapshot(keyVector, valueVector); + } + } + + private static Map builders( + Map schema, BufferAllocator allocator) { + Map headers = new HashMap<>(); + for (Map.Entry entry : schema.entrySet()) { + String name = (String) entry.getKey(); + DataType type = (DataType) entry.getValue(); + ColumnKey columnKey = TagKVUtils.splitFullName(name); + ChunkSnapshotBuilder builder = new ChunkSnapshotBuilder(columnKey, type, allocator); + headers.put(name, builder); + } + return headers; + } + + public static , T> Iterable recordOfRows( + Scanner> rows, Map schema, BufferAllocator allocator) + throws StorageException { + Map builders = builders(schema, allocator); + while (rows.iterate()) { + long key = (Long) rows.key(); + Scanner row = rows.value(); + while (row.iterate()) { + String name = (String) row.key(); + Object value = row.value(); + ChunkSnapshotBuilder builder = builders.get(name); + builder.append(key, value); + } + } + return builders.values().stream().map(ChunkSnapshotBuilder::build).collect(Collectors.toList()); + } + + public static , T> Iterable recordOfColumns( + Scanner> batch, Map schema, BufferAllocator allocator) + throws StorageException { + Map builders = builders(schema, allocator); + while (batch.iterate()) { + String name = (String) batch.key(); + Scanner column = batch.value(); + ChunkSnapshotBuilder builder = builders.get(name); + while (column.iterate()) { + long key = (Long) column.key(); + Object value = column.value(); + builder.append(key, value); + } + } + return builders.values().stream().map(ChunkSnapshotBuilder::build).collect(Collectors.toList()); + } +} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/AreaFilterScanner.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/AreaFilterScanner.java similarity index 76% rename from dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/AreaFilterScanner.java rename to dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/AreaFilterScanner.java index b20e7830e0..9301ac34d0 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/AreaFilterScanner.java +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/AreaFilterScanner.java @@ -1,17 +1,19 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.parquet.db.util.iterator; @@ -19,7 +21,6 @@ import cn.edu.tsinghua.iginx.parquet.db.util.AreaSet; import cn.edu.tsinghua.iginx.parquet.util.exception.StorageException; import com.google.common.collect.RangeSet; -import javax.annotation.Nonnull; public class AreaFilterScanner, F, V> implements Scanner> { @@ -35,13 +36,11 @@ public AreaFilterScanner(Scanner> scanner, AreaSet exclus private K currentKey = null; private Scanner currentValue = null; - @Nonnull @Override public K key() { return currentKey; } - @Nonnull @Override public Scanner value() { return currentValue; @@ -68,10 +67,7 @@ public void close() throws StorageException { } private boolean excludeKey(K key) { - if (areas.getKeys().contains(key)) { - return true; - } - return false; + return areas.getKeys().contains(key); } private class RowFilterScanner implements Scanner { @@ -88,13 +84,11 @@ private RowFilterScanner(Scanner scanner, AreaSet exclusive) { private F currentField = null; private V currentValue = null; - @Nonnull @Override public F key() { return currentField; } - @Nonnull @Override public V value() { return currentValue; diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/BatchPlaneScanner.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/BatchPlaneScanner.java similarity index 78% rename from dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/BatchPlaneScanner.java rename to dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/BatchPlaneScanner.java index 66e26a751a..2b9f5e7f98 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/BatchPlaneScanner.java +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/BatchPlaneScanner.java @@ -1,24 +1,25 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.parquet.db.util.iterator; import cn.edu.tsinghua.iginx.parquet.util.exception.StorageException; import java.util.NoSuchElementException; -import javax.annotation.Nonnull; public class BatchPlaneScanner implements Scanner>> { @@ -38,13 +39,11 @@ public BatchPlaneScanner(Scanner> scannerHelper, long maxBatchS this.maxBatchSize = maxBatchSize; } - @Nonnull @Override public Long key() { return currentBatchSize; } - @Nonnull @Override public Scanner> value() throws NoSuchElementException { return planeScannerHelper; @@ -63,13 +62,11 @@ private class PlaneScannerHelper implements Scanner> { private final Scanner lineScannerHelper = new LineScannerHelper(); - @Nonnull @Override public K key() { return planeScanner.key(); } - @Nonnull @Override public Scanner value() throws NoSuchElementException { return lineScannerHelper; @@ -98,13 +95,11 @@ public void close() {} private class LineScannerHelper implements Scanner { - @Nonnull @Override public F key() throws NoSuchElementException { return planeScanner.value().key(); } - @Nonnull @Override public V value() throws NoSuchElementException { return planeScanner.value().value(); diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/ColumnUnionRowScanner.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/ColumnUnionRowScanner.java similarity index 75% rename from dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/ColumnUnionRowScanner.java rename to dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/ColumnUnionRowScanner.java index 429880b056..5199ed2880 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/ColumnUnionRowScanner.java +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/ColumnUnionRowScanner.java @@ -1,17 +1,19 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.parquet.db.util.iterator; @@ -21,7 +23,7 @@ import java.util.Map; import java.util.NoSuchElementException; import java.util.PriorityQueue; -import javax.annotation.Nonnull; +import org.apache.arrow.util.AutoCloseables; public class ColumnUnionRowScanner, F, V> implements Scanner> { @@ -30,8 +32,6 @@ public class ColumnUnionRowScanner, F, V> private final Map> scanners; - private K currentKey = null; - private Scanner currentScanner = null; public ColumnUnionRowScanner(Map> scanners) { @@ -47,7 +47,6 @@ private void init() throws StorageException { } } - @Nonnull @Override public K key() throws NoSuchElementException { if (currentScanner == null) { @@ -57,7 +56,6 @@ public K key() throws NoSuchElementException { return queue.peek().getValue().key(); } - @Nonnull @Override public Scanner value() throws NoSuchElementException { if (currentScanner == null) { @@ -85,9 +83,12 @@ public boolean iterate() throws StorageException { @Override public void close() throws StorageException { - Map.Entry> entry; - while ((entry = queue.poll()) != null) { - entry.getValue().close(); + try { + AutoCloseables.close(scanners.values()); + } catch (RuntimeException | StorageException e) { + throw e; + } catch (Exception e) { + throw new StorageException(e); } } @@ -95,7 +96,6 @@ private class QueueScanner implements Scanner { private K currentKey = null; - @Nonnull @Override public F key() throws NoSuchElementException { if (queue.peek() == null) { @@ -104,7 +104,6 @@ public F key() throws NoSuchElementException { return queue.peek().getKey(); } - @Nonnull @Override public V value() throws NoSuchElementException { if (queue.peek() == null) { diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/ConcatScanner.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/ConcatScanner.java similarity index 56% rename from dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/ConcatScanner.java rename to dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/ConcatScanner.java index 1afe3017dc..55cbb45511 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/ConcatScanner.java +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/ConcatScanner.java @@ -1,17 +1,19 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.parquet.db.util.iterator; @@ -19,7 +21,6 @@ import cn.edu.tsinghua.iginx.parquet.util.exception.StorageException; import java.util.Iterator; import java.util.NoSuchElementException; -import javax.annotation.Nonnull; public class ConcatScanner, V> implements Scanner { @@ -31,7 +32,6 @@ public ConcatScanner(Iterator> iterator) { this.scannerIterator = iterator; } - @Nonnull @Override public K key() throws NoSuchElementException { if (currentScanner == null) { @@ -40,7 +40,6 @@ public K key() throws NoSuchElementException { return currentScanner.key(); } - @Nonnull @Override public V value() throws NoSuchElementException { if (currentScanner == null) { @@ -51,16 +50,23 @@ public V value() throws NoSuchElementException { @Override public boolean iterate() throws StorageException { - if (currentScanner != null && currentScanner.iterate()) { - return true; + if (currentScanner != null) { + if (currentScanner.iterate()) { + return true; + } else { + currentScanner.close(); + currentScanner = null; + } } while (scannerIterator.hasNext()) { currentScanner = scannerIterator.next(); if (currentScanner.iterate()) { return true; + } else { + currentScanner.close(); + currentScanner = null; } } - currentScanner = null; return false; } diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/DelegateScanner.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/DelegateScanner.java new file mode 100644 index 0000000000..90bd881b42 --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/DelegateScanner.java @@ -0,0 +1,48 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cn.edu.tsinghua.iginx.parquet.db.util.iterator; + +import cn.edu.tsinghua.iginx.parquet.util.exception.StorageException; + +public class DelegateScanner implements Scanner { + private final Scanner scanner; + + public DelegateScanner(Scanner scanner) { + this.scanner = scanner; + } + + @Override + public K key() { + return scanner.key(); + } + + @Override + public V value() { + return scanner.value(); + } + + @Override + public boolean iterate() throws StorageException { + return scanner.iterate(); + } + + @Override + public void close() throws StorageException { + scanner.close(); + } +} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/EmptyScanner.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/EmptyScanner.java similarity index 50% rename from dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/EmptyScanner.java rename to dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/EmptyScanner.java index 4df8b9f5e0..4a7553fdff 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/EmptyScanner.java +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/EmptyScanner.java @@ -1,24 +1,25 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.parquet.db.util.iterator; import cn.edu.tsinghua.iginx.parquet.util.exception.StorageException; import java.util.NoSuchElementException; -import javax.annotation.Nonnull; public class EmptyScanner implements Scanner { @@ -29,13 +30,11 @@ public static Scanner getInstance() { return (Scanner) EMPTY; } - @Nonnull @Override public K key() { throw new NoSuchElementException(); } - @Nonnull @Override public V value() { throw new NoSuchElementException(); diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/EmtpyHeadRowScanner.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/EmtpyHeadRowScanner.java new file mode 100644 index 0000000000..8305adf853 --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/EmtpyHeadRowScanner.java @@ -0,0 +1,51 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cn.edu.tsinghua.iginx.parquet.db.util.iterator; + +import cn.edu.tsinghua.iginx.parquet.util.exception.StorageException; + +public class EmtpyHeadRowScanner, F, V> + implements Scanner> { + + private final K key; + private boolean hasNext = true; + + public EmtpyHeadRowScanner(K key) { + this.key = key; + } + + @Override + public K key() { + return key; + } + + @Override + public Scanner value() { + return EmptyScanner.getInstance(); + } + + @Override + public boolean iterate() throws StorageException { + boolean lastHasNext = hasNext; + hasNext = false; + return lastHasNext; + } + + @Override + public void close() throws StorageException {} +} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/IteratorScanner.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/IteratorScanner.java similarity index 58% rename from dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/IteratorScanner.java rename to dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/IteratorScanner.java index bdaa269e3c..9f8398a3a8 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/IteratorScanner.java +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/IteratorScanner.java @@ -1,17 +1,19 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.parquet.db.util.iterator; @@ -19,7 +21,6 @@ import java.util.Iterator; import java.util.Map; import java.util.NoSuchElementException; -import javax.annotation.Nonnull; public class IteratorScanner implements Scanner { @@ -33,7 +34,6 @@ public IteratorScanner(Iterator> iterator) { this.iterator = iterator; } - @Nonnull @Override public K key() throws NoSuchElementException { if (key == null) { @@ -42,7 +42,6 @@ public K key() throws NoSuchElementException { return key; } - @Nonnull @Override public V value() throws NoSuchElementException { if (value == null) { diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/LazyRowScanner.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/LazyRowScanner.java new file mode 100644 index 0000000000..2830c8f60c --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/LazyRowScanner.java @@ -0,0 +1,56 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cn.edu.tsinghua.iginx.parquet.db.util.iterator; + +import cn.edu.tsinghua.iginx.parquet.util.exception.StorageException; + +public class LazyRowScanner, F, V> implements Scanner> { + + private final RowScannerFactory factory; + + public LazyRowScanner(RowScannerFactory factory) { + this.factory = factory; + } + + private Scanner> scanner; + + @Override + public K key() { + return scanner.key(); + } + + @Override + public Scanner value() { + return scanner.value(); + } + + @Override + public boolean iterate() throws StorageException { + if (scanner == null) { + scanner = factory.create(); + } + return scanner.iterate(); + } + + @Override + public void close() throws StorageException { + if (scanner != null) { + scanner.close(); + } + } +} diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/ListenCloseScanner.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/ListenCloseScanner.java new file mode 100644 index 0000000000..b8bccfd805 --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/ListenCloseScanner.java @@ -0,0 +1,36 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cn.edu.tsinghua.iginx.parquet.db.util.iterator; + +import cn.edu.tsinghua.iginx.parquet.util.exception.StorageException; + +public class ListenCloseScanner extends DelegateScanner { + + private final Runnable callback; + + public ListenCloseScanner(Scanner scanner, Runnable callback) { + super(scanner); + this.callback = callback; + } + + @Override + public void close() throws StorageException { + super.close(); + callback.run(); + } +} diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/RowConcatScanner.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/RowConcatScanner.java new file mode 100644 index 0000000000..8da9c5cf84 --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/RowConcatScanner.java @@ -0,0 +1,20 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cn.edu.tsinghua.iginx.parquet.db.util.iterator; + +public class RowConcatScanner {} diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/RowScannerFactory.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/RowScannerFactory.java new file mode 100644 index 0000000000..4dbc4abc8e --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/RowScannerFactory.java @@ -0,0 +1,24 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cn.edu.tsinghua.iginx.parquet.db.util.iterator; + +import cn.edu.tsinghua.iginx.parquet.util.exception.StorageException; + +public interface RowScannerFactory, F, V> { + Scanner> create() throws StorageException; +} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/RowUnionScanner.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/RowUnionScanner.java similarity index 69% rename from dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/RowUnionScanner.java rename to dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/RowUnionScanner.java index a8f34162cc..b1afb04990 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/RowUnionScanner.java +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/RowUnionScanner.java @@ -1,24 +1,25 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.parquet.db.util.iterator; import cn.edu.tsinghua.iginx.parquet.util.exception.StorageException; import java.util.*; -import javax.annotation.Nonnull; public class RowUnionScanner, F, V> implements Scanner> { @@ -29,20 +30,33 @@ public RowUnionScanner(Iterable>> scanners) throws Stor Comparator.comparing(e -> e.getKey().key()); this.queue = new PriorityQueue<>(comparing.thenComparing(Map.Entry::getValue)); + StorageException exception = null; long i = 0; for (Scanner> scanner : scanners) { if (scanner.iterate()) { queue.add(new AbstractMap.SimpleImmutableEntry<>(scanner, i)); i++; + } else { + try { + scanner.close(); + } catch (StorageException e) { + if (exception == null) { + exception = e; + } else { + exception.addSuppressed(e); + } + } } } + if (exception != null) { + throw exception; + } } private Scanner currentRow = null; private K currentKey = null; - @Nonnull @Override public K key() throws NoSuchElementException { if (currentKey == null) { @@ -51,7 +65,6 @@ public K key() throws NoSuchElementException { return currentKey; } - @Nonnull @Override public Scanner value() throws NoSuchElementException { if (currentRow == null) { @@ -89,7 +102,6 @@ public boolean iterate() throws StorageException { private V value; - @Nonnull @Override public F key() throws NoSuchElementException { if (key == null) { @@ -98,7 +110,6 @@ public F key() throws NoSuchElementException { return key; } - @Nonnull @Override public V value() throws NoSuchElementException { if (value == null) { @@ -128,8 +139,20 @@ public void close() throws StorageException {} @Override public void close() throws StorageException { + StorageException exception = null; for (Map.Entry>, Long> entry : queue) { - entry.getKey().close(); + try { + entry.getKey().close(); + } catch (StorageException e) { + if (exception == null) { + exception = e; + } else { + exception.addSuppressed(e); + } + } + } + if (exception != null) { + throw exception; } } } diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/Scanner.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/Scanner.java new file mode 100644 index 0000000000..5971ee6292 --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/Scanner.java @@ -0,0 +1,33 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.parquet.db.util.iterator; + +import cn.edu.tsinghua.iginx.parquet.util.exception.StorageException; + +public interface Scanner extends AutoCloseable { + + K key(); + + V value(); + + boolean iterate() throws StorageException; + + @Override + void close() throws StorageException; +} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/SizeUtils.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/SizeUtils.java similarity index 51% rename from dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/SizeUtils.java rename to dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/SizeUtils.java index a1940a0f52..cae52979dc 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/SizeUtils.java +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/SizeUtils.java @@ -1,17 +1,19 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.parquet.db.util.iterator; @@ -19,7 +21,6 @@ import java.util.HashMap; import java.util.Map; import java.util.function.Function; -import javax.annotation.Nonnull; public class SizeUtils { private SizeUtils() {} @@ -35,7 +36,7 @@ private SizeUtils() {} sizeMap.put(byte[].class, (obj) -> (long) ((byte[]) obj).length); } - public static long sizeOf(@Nonnull Object obj) { + public static long sizeOf(Object obj) { Function sizeGetter = sizeMap.get(obj.getClass()); if (sizeGetter == null) { throw new UnsupportedOperationException(obj.getClass() + " is not supported"); diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/exec/Executor.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/exec/Executor.java similarity index 52% rename from dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/exec/Executor.java rename to dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/exec/Executor.java index 16ed980af1..5b88ba3819 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/exec/Executor.java +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/exec/Executor.java @@ -1,17 +1,19 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.parquet.exec; @@ -21,19 +23,23 @@ import cn.edu.tsinghua.iginx.engine.physical.task.TaskExecuteResult; import cn.edu.tsinghua.iginx.engine.shared.KeyRange; import cn.edu.tsinghua.iginx.engine.shared.data.write.DataView; +import cn.edu.tsinghua.iginx.engine.shared.function.FunctionCall; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; import cn.edu.tsinghua.iginx.metadata.entity.ColumnsInterval; import cn.edu.tsinghua.iginx.metadata.entity.KeyInterval; import cn.edu.tsinghua.iginx.utils.Pair; import java.util.List; +import java.util.Set; +import javax.annotation.Nullable; public interface Executor { TaskExecuteResult executeProjectTask( List paths, - TagFilter tagFilter, - Filter filter, + @Nullable TagFilter tagFilter, + @Nullable Filter filter, + @Nullable List calls, String storageUnit, boolean isDummyStorageUnit); @@ -42,7 +48,8 @@ TaskExecuteResult executeProjectTask( TaskExecuteResult executeDeleteTask( List paths, List keyRanges, TagFilter tagFilter, String storageUnit); - List getColumnsOfStorageUnit(String storageUnit) throws PhysicalException; + List getColumnsOfStorageUnit( + String storageUnit, Set patterns, TagFilter tagFilter) throws PhysicalException; Pair getBoundaryOfStorage(String dataPrefix) throws PhysicalException; diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/exec/LocalExecutor.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/exec/LocalExecutor.java similarity index 78% rename from dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/exec/LocalExecutor.java rename to dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/exec/LocalExecutor.java index 175811369a..9eacf439eb 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/exec/LocalExecutor.java +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/exec/LocalExecutor.java @@ -1,17 +1,19 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.parquet.exec; @@ -26,6 +28,7 @@ import cn.edu.tsinghua.iginx.engine.shared.data.read.FilterRowStreamWrapper; import cn.edu.tsinghua.iginx.engine.shared.data.read.RowStream; import cn.edu.tsinghua.iginx.engine.shared.data.write.DataView; +import cn.edu.tsinghua.iginx.engine.shared.function.FunctionCall; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; import cn.edu.tsinghua.iginx.metadata.entity.ColumnsInterval; @@ -39,19 +42,26 @@ import cn.edu.tsinghua.iginx.parquet.util.exception.IsClosedException; import cn.edu.tsinghua.iginx.utils.Pair; import cn.edu.tsinghua.iginx.utils.StringUtils; +import com.google.common.base.Preconditions; import com.google.common.collect.Iterables; import java.io.File; import java.io.IOException; import java.nio.channels.FileChannel; import java.nio.channels.FileLock; import java.nio.channels.OverlappingFileLockException; -import java.nio.file.*; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; +import javax.annotation.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.MDC; @@ -240,6 +250,24 @@ private Manager getOrCreateManager(String storageUnit) throws PhysicalException @Override public TaskExecuteResult executeProjectTask( + List paths, + @Nullable TagFilter tagFilter, + @Nullable Filter filter, + @Nullable List calls, + String storageUnit, + boolean isDummyStorageUnit) { + if (calls != null) { + Preconditions.checkArgument(!calls.isEmpty(), "calls should not be empty"); + Preconditions.checkArgument(filter == null, "filter should be null"); + Preconditions.checkArgument(!isDummyStorageUnit, "dummy storage unit not allowed"); + return executeAggregationTask(paths, tagFilter, calls, storageUnit); + } else { + Preconditions.checkNotNull(filter, "filter should not be null"); + return executeProjectTask(paths, tagFilter, filter, storageUnit, isDummyStorageUnit); + } + } + + private TaskExecuteResult executeProjectTask( List paths, TagFilter tagFilter, Filter filter, @@ -261,6 +289,17 @@ public TaskExecuteResult executeProjectTask( } } + private TaskExecuteResult executeAggregationTask( + List patterns, TagFilter tagFilter, List calls, String storageUnit) { + try { + Manager manager = getOrCreateManager(storageUnit); + RowStream rowStream = ((DataManager) manager).aggregation(patterns, tagFilter, calls); + return new TaskExecuteResult(rowStream); + } catch (PhysicalException e) { + return new TaskExecuteResult(e); + } + } + @Override public TaskExecuteResult executeInsertTask(DataView dataView, String storageUnit) { try { @@ -285,11 +324,17 @@ public TaskExecuteResult executeDeleteTask( } @Override - public List getColumnsOfStorageUnit(String storageUnit) throws PhysicalException { + public List getColumnsOfStorageUnit( + String storageUnit, Set patterns, TagFilter tagFilter) throws PhysicalException { + List patternList = new ArrayList<>(patterns); + if (patternList.isEmpty()) { + patternList.add("*"); + } if (storageUnit.equals("*")) { List columns = new ArrayList<>(); - for (Manager manager : getAllManagers()) { - columns.addAll(manager.getColumns()); + for (Manager manager : + Iterables.concat(managers.values(), Collections.singleton(dummyManager))) { + columns.addAll(manager.getColumns(patternList, tagFilter)); } return columns; } else { @@ -300,20 +345,10 @@ public List getColumnsOfStorageUnit(String storageUnit) throws PhysicalE @Override public Pair getBoundaryOfStorage(String dataPrefix) throws PhysicalException { - List paths = new ArrayList<>(); - long start = Long.MAX_VALUE, end = Long.MIN_VALUE; - for (Manager manager : getAllManagers()) { - for (Column column : manager.getColumns()) { - paths.add(column.getPath()); - } - KeyInterval interval = manager.getKeyInterval(); - if (interval.getStartKey() < start) { - start = interval.getStartKey(); - } - if (interval.getEndKey() > end) { - end = interval.getEndKey(); - } - } + List paths = + dummyManager.getColumns(Collections.singletonList("*"), null).stream() + .map(Column::getPath) + .collect(Collectors.toList()); if (dataPrefix != null) { paths = paths.stream().filter(path -> path.startsWith(dataPrefix)).collect(Collectors.toList()); @@ -322,17 +357,9 @@ public Pair getBoundaryOfStorage(String dataPrefix if (paths.isEmpty()) { throw new PhysicalTaskExecuteFailureException("no data"); } - if (start == Long.MAX_VALUE || end == Long.MIN_VALUE) { - throw new PhysicalTaskExecuteFailureException("time range error"); - } ColumnsInterval columnsInterval = new ColumnsInterval(paths.get(0), StringUtils.nextString(paths.get(paths.size() - 1))); - KeyInterval keyInterval = new KeyInterval(start, end); - return new Pair<>(columnsInterval, keyInterval); - } - - private Iterable getAllManagers() { - return Iterables.concat(managers.values(), Collections.singleton(dummyManager)); + return new Pair<>(columnsInterval, KeyInterval.getDefaultKeyInterval()); } @Override diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/exec/RemoteExecutor.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/exec/RemoteExecutor.java similarity index 80% rename from dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/exec/RemoteExecutor.java rename to dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/exec/RemoteExecutor.java index 2bae22e064..298a3682b5 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/exec/RemoteExecutor.java +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/exec/RemoteExecutor.java @@ -1,17 +1,19 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.parquet.exec; @@ -28,6 +30,10 @@ import cn.edu.tsinghua.iginx.engine.shared.data.write.BitmapView; import cn.edu.tsinghua.iginx.engine.shared.data.write.DataView; import cn.edu.tsinghua.iginx.engine.shared.data.write.RawDataType; +import cn.edu.tsinghua.iginx.engine.shared.function.Function; +import cn.edu.tsinghua.iginx.engine.shared.function.FunctionCall; +import cn.edu.tsinghua.iginx.engine.shared.function.FunctionParams; +import cn.edu.tsinghua.iginx.engine.shared.function.system.Count; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; import cn.edu.tsinghua.iginx.engine.shared.operator.tag.AndTagFilter; import cn.edu.tsinghua.iginx.engine.shared.operator.tag.BasePreciseTagFilter; @@ -38,8 +44,22 @@ import cn.edu.tsinghua.iginx.metadata.entity.ColumnsInterval; import cn.edu.tsinghua.iginx.metadata.entity.KeyInterval; import cn.edu.tsinghua.iginx.parquet.server.FilterTransformer; -import cn.edu.tsinghua.iginx.parquet.thrift.*; +import cn.edu.tsinghua.iginx.parquet.thrift.DeleteReq; +import cn.edu.tsinghua.iginx.parquet.thrift.GetColumnsOfStorageUnitResp; +import cn.edu.tsinghua.iginx.parquet.thrift.GetStorageBoundaryResp; +import cn.edu.tsinghua.iginx.parquet.thrift.InsertReq; +import cn.edu.tsinghua.iginx.parquet.thrift.ParquetHeader; +import cn.edu.tsinghua.iginx.parquet.thrift.ParquetKeyRange; +import cn.edu.tsinghua.iginx.parquet.thrift.ParquetRawData; import cn.edu.tsinghua.iginx.parquet.thrift.ParquetService.Client; +import cn.edu.tsinghua.iginx.parquet.thrift.ProjectReq; +import cn.edu.tsinghua.iginx.parquet.thrift.ProjectResp; +import cn.edu.tsinghua.iginx.parquet.thrift.RawFunction; +import cn.edu.tsinghua.iginx.parquet.thrift.RawFunctionCall; +import cn.edu.tsinghua.iginx.parquet.thrift.RawFunctionParams; +import cn.edu.tsinghua.iginx.parquet.thrift.RawTagFilter; +import cn.edu.tsinghua.iginx.parquet.thrift.Status; +import cn.edu.tsinghua.iginx.parquet.thrift.TagFilterType; import cn.edu.tsinghua.iginx.thrift.DataType; import cn.edu.tsinghua.iginx.utils.Bitmap; import cn.edu.tsinghua.iginx.utils.ByteUtils; @@ -51,6 +71,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import javax.annotation.Nullable; import org.apache.thrift.TException; import org.apache.thrift.protocol.TBinaryProtocol; import org.apache.thrift.transport.TTransport; @@ -74,8 +97,9 @@ public RemoteExecutor(String ip, int port, Map extraParams) @Override public TaskExecuteResult executeProjectTask( List paths, - TagFilter tagFilter, - Filter filter, + @Nullable TagFilter tagFilter, + @Nullable Filter filter, + @Nullable List calls, String storageUnit, boolean isDummyStorageUnit) { ProjectReq req = new ProjectReq(storageUnit, isDummyStorageUnit, paths); @@ -85,6 +109,11 @@ public TaskExecuteResult executeProjectTask( if (filter != null) { req.setFilter(FilterTransformer.toRawFilter(filter)); } + if (calls != null) { + List rawFunctionCalls = + calls.stream().map(RemoteExecutor::constructRawFunctionCall).collect(Collectors.toList()); + req.setAggregations(rawFunctionCalls); + } try { TTransport transport = thriftConnPool.borrowTransport(); @@ -277,6 +306,9 @@ public TaskExecuteResult executeDeleteTask( } private RawTagFilter constructRawTagFilter(TagFilter tagFilter) { + if (tagFilter == null) { + return null; + } switch (tagFilter.getType()) { case Base: { @@ -334,12 +366,32 @@ private RawTagFilter constructRawTagFilter(TagFilter tagFilter) { } } + private static RawFunctionCall constructRawFunctionCall(FunctionCall functionCall) { + RawFunction rawFunction = constructRawFunction(functionCall.getFunction()); + RawFunctionParams rawFunctionParam = constructRawFunctionParam(functionCall.getParams()); + return new RawFunctionCall(rawFunction, rawFunctionParam); + } + + private static RawFunction constructRawFunction(Function function) { + if (function instanceof Count) { + return new RawFunction(Count.COUNT); + } + throw new IllegalArgumentException("unsupported function type"); + } + + private static RawFunctionParams constructRawFunctionParam(FunctionParams params) { + List patterns = params.getPaths(); + return new RawFunctionParams(patterns); + } + @Override - public List getColumnsOfStorageUnit(String storageUnit) throws PhysicalException { + public List getColumnsOfStorageUnit( + String storageUnit, Set patterns, TagFilter tagFilter) throws PhysicalException { try { TTransport transport = thriftConnPool.borrowTransport(); Client client = new Client(new TBinaryProtocol(transport)); - GetColumnsOfStorageUnitResp resp = client.getColumnsOfStorageUnit(storageUnit); + GetColumnsOfStorageUnitResp resp = + client.getColumnsOfStorageUnit(storageUnit, patterns, constructRawTagFilter(tagFilter)); thriftConnPool.returnTransport(transport); List columnList = new ArrayList<>(); resp.getTsList() diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/parquet/FilterUtils.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/parquet/FilterUtils.java similarity index 81% rename from dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/parquet/FilterUtils.java rename to dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/parquet/FilterUtils.java index d50364fe32..2365c5fb24 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/parquet/FilterUtils.java +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/parquet/FilterUtils.java @@ -1,17 +1,19 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.parquet.io.parquet; @@ -23,15 +25,13 @@ import cn.edu.tsinghua.iginx.parquet.util.Constants; import cn.edu.tsinghua.iginx.utils.Pair; import java.util.Objects; -import javax.annotation.Nonnull; import shaded.iginx.org.apache.parquet.filter2.predicate.FilterApi; import shaded.iginx.org.apache.parquet.filter2.predicate.FilterPredicate; import shaded.iginx.org.apache.parquet.filter2.predicate.Operators; class FilterUtils { - @Nonnull - public static Pair toFilterPredicate(@Nonnull Filter filter) { + public static Pair toFilterPredicate(Filter filter) { switch (filter.getType()) { case Key: return toFilterPredicate((KeyFilter) filter); diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/parquet/IGroupConverter.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/parquet/IGroupConverter.java similarity index 89% rename from dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/parquet/IGroupConverter.java rename to dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/parquet/IGroupConverter.java index 4385f3ce32..9603000871 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/parquet/IGroupConverter.java +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/parquet/IGroupConverter.java @@ -1,17 +1,19 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.parquet.io.parquet; diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/parquet/IParquetReader.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/parquet/IParquetReader.java similarity index 77% rename from dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/parquet/IParquetReader.java rename to dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/parquet/IParquetReader.java index dc47c96014..011ddab7c4 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/parquet/IParquetReader.java +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/parquet/IParquetReader.java @@ -1,17 +1,19 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.parquet.io.parquet; @@ -23,9 +25,7 @@ import com.google.common.collect.Range; import java.io.IOException; import java.nio.file.Path; -import java.util.Objects; -import java.util.Set; -import javax.annotation.Nonnull; +import java.util.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import shaded.iginx.org.apache.parquet.ParquetReadOptions; @@ -37,6 +37,7 @@ import shaded.iginx.org.apache.parquet.hadoop.ParquetRecordReader; import shaded.iginx.org.apache.parquet.hadoop.metadata.BlockMetaData; import shaded.iginx.org.apache.parquet.hadoop.metadata.ColumnChunkMetaData; +import shaded.iginx.org.apache.parquet.hadoop.metadata.ColumnPath; import shaded.iginx.org.apache.parquet.hadoop.metadata.ParquetMetadata; import shaded.iginx.org.apache.parquet.io.InputFile; import shaded.iginx.org.apache.parquet.io.LocalInputFile; @@ -65,6 +66,29 @@ public static Builder builder(Path path) { return new Builder(new LocalInputFile(path)); } + public static Map getCountsOf(ParquetMetadata meta) { + Map counts = new HashMap<>(); + for (BlockMetaData block : meta.getBlocks()) { + List columns = block.getColumns(); + for (ColumnChunkMetaData column : columns) { + ColumnPath path = column.getPath(); + long count = column.getValueCount(); + long nulls = column.getStatistics().getNumNulls(); + long nonNulls = count - nulls; + counts.compute(path, (k, v) -> v == null ? nonNulls : v + nonNulls); + } + } + return counts; + } + + public static Map encodeCounts(Map counts) { + Map result = new HashMap<>(); + for (Map.Entry entry : counts.entrySet()) { + result.put(entry.getKey(), entry.getValue().toString()); + } + return result; + } + public MessageType getSchema() { return schema; } @@ -89,24 +113,32 @@ public void close() throws Exception { } public long getRowCount() { + return getRowCountOf(metadata); + } + + public static long getRowCountOf(ParquetMetadata metadata) { return metadata.getBlocks().stream().mapToLong(BlockMetaData::getRowCount).sum(); } public Range getRange() { + return getRangeOf(metadata); + } + + public static Range getRangeOf(ParquetMetadata metadata) { MessageType schema = metadata.getFileMetaData().getSchema(); if (schema.containsPath(new String[] {Constants.KEY_FIELD_NAME})) { Type type = schema.getType(Constants.KEY_FIELD_NAME); if (type.isPrimitive()) { PrimitiveType primitiveType = type.asPrimitiveType(); if (primitiveType.getPrimitiveTypeName() == PrimitiveType.PrimitiveTypeName.INT64) { - return getKeyRange(); + return getKeyRangeOf(metadata); } } } - return Range.closedOpen(0L, getRowCount()); + return Range.closedOpen(0L, getRowCountOf(metadata)); } - private Range getKeyRange() { + private static Range getKeyRangeOf(ParquetMetadata metadata) { long min = Long.MAX_VALUE; long max = Long.MIN_VALUE; @@ -179,7 +211,6 @@ public IParquetReader build(ParquetMetadata footer) throws IOException { return build(footer, optionsBuilder.build()); } - @Nonnull private IParquetReader build(ParquetMetadata footer, ParquetReadOptions options) throws IOException { MessageType schema = footer.getFileMetaData().getSchema(); diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/parquet/IParquetWriter.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/parquet/IParquetWriter.java similarity index 68% rename from dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/parquet/IParquetWriter.java rename to dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/parquet/IParquetWriter.java index 7e7aa90086..5304828a34 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/parquet/IParquetWriter.java +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/parquet/IParquetWriter.java @@ -1,17 +1,19 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.parquet.io.parquet; @@ -23,8 +25,10 @@ import java.nio.file.Path; import shaded.iginx.org.apache.parquet.ParquetWriteOptions; import shaded.iginx.org.apache.parquet.bytes.HeapByteBufferAllocator; +import shaded.iginx.org.apache.parquet.hadoop.CodecFactory; import shaded.iginx.org.apache.parquet.hadoop.ParquetFileWriter; import shaded.iginx.org.apache.parquet.hadoop.ParquetRecordWriter; +import shaded.iginx.org.apache.parquet.hadoop.metadata.CompressionCodecName; import shaded.iginx.org.apache.parquet.hadoop.metadata.ParquetMetadata; import shaded.iginx.org.apache.parquet.io.LocalOutputFile; import shaded.iginx.org.apache.parquet.io.OutputFile; @@ -43,7 +47,12 @@ public class IParquetWriter implements AutoCloseable { } public static Builder builder(Path path, MessageType schema) { - return new Builder(new LocalOutputFile(path, new HeapByteBufferAllocator(), 8 * 1024), schema); + return builder(path, schema, 8 * 1024); + } + + public static Builder builder(Path path, MessageType schema, int maxBufferSize) { + return new Builder( + new LocalOutputFile(path, new HeapByteBufferAllocator(), maxBufferSize), schema); } public void write(IRecord record) throws IOException { @@ -91,6 +100,12 @@ public Builder withPageSize(int pageSize) { optionsBuilder.asParquetPropertiesBuilder().withPageSize(pageSize); return this; } + + public Builder withCompressionCodec(String codec) { + CompressionCodecName codecName = CompressionCodecName.valueOf(codec); + optionsBuilder.withCompressor(new CodecFactory().getCompressor(codecName)); + return this; + } } public static IRecord getRecord(MessageType schema, Long key, Scanner value) @@ -100,6 +115,7 @@ public static IRecord getRecord(MessageType schema, Long key, Scanner. */ package cn.edu.tsinghua.iginx.parquet.io.parquet; @@ -53,4 +55,8 @@ public Map.Entry next() { } }; } + + public void sort() { + values.sort(Comparator.comparingInt(Map.Entry::getKey)); + } } diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/parquet/IRecordDematerializer.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/parquet/IRecordDematerializer.java similarity index 80% rename from dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/parquet/IRecordDematerializer.java rename to dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/parquet/IRecordDematerializer.java index d64d6adfaf..2c753bffed 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/parquet/IRecordDematerializer.java +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/parquet/IRecordDematerializer.java @@ -1,17 +1,19 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.parquet.io.parquet; diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/parquet/IRecordMaterializer.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/parquet/IRecordMaterializer.java similarity index 52% rename from dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/parquet/IRecordMaterializer.java rename to dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/parquet/IRecordMaterializer.java index d1e809e66f..e6808a0d3c 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/parquet/IRecordMaterializer.java +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/parquet/IRecordMaterializer.java @@ -1,17 +1,19 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.parquet.io.parquet; diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/parquet/ProjectUtils.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/parquet/ProjectUtils.java similarity index 55% rename from dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/parquet/ProjectUtils.java rename to dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/parquet/ProjectUtils.java index 0cba9c8696..bc5a9d533d 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/parquet/ProjectUtils.java +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/parquet/ProjectUtils.java @@ -1,17 +1,19 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.parquet.io.parquet; @@ -20,7 +22,6 @@ import java.util.HashSet; import java.util.Objects; import java.util.Set; -import javax.annotation.Nonnull; import javax.annotation.Nullable; import shaded.iginx.org.apache.parquet.schema.MessageType; import shaded.iginx.org.apache.parquet.schema.Type; @@ -29,8 +30,7 @@ public class ProjectUtils { private ProjectUtils() {} - @Nonnull - static MessageType projectMessageType(@Nonnull MessageType schema, @Nullable Set fields) { + static MessageType projectMessageType(MessageType schema, @Nullable Set fields) { Set schemaFields = new HashSet<>(Objects.requireNonNull(fields)); schemaFields.add(Constants.KEY_FIELD_NAME); diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/Manager.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/Manager.java new file mode 100644 index 0000000000..db8bd04ce8 --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/Manager.java @@ -0,0 +1,41 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.parquet.manager; + +import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; +import cn.edu.tsinghua.iginx.engine.physical.storage.domain.Column; +import cn.edu.tsinghua.iginx.engine.shared.KeyRange; +import cn.edu.tsinghua.iginx.engine.shared.data.read.RowStream; +import cn.edu.tsinghua.iginx.engine.shared.data.write.DataView; +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; +import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; +import java.util.List; + +public interface Manager extends AutoCloseable { + + RowStream project(List paths, TagFilter tagFilter, Filter filter) + throws PhysicalException; + + void insert(DataView dataView) throws PhysicalException; + + void delete(List paths, List keyRanges, TagFilter tagFilter) + throws PhysicalException; + + List getColumns(List paths, TagFilter tagFilter) throws PhysicalException; +} diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/AggregatedRowStream.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/AggregatedRowStream.java new file mode 100644 index 0000000000..bfe16a2c39 --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/AggregatedRowStream.java @@ -0,0 +1,81 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cn.edu.tsinghua.iginx.parquet.manager.data; + +import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; +import cn.edu.tsinghua.iginx.engine.shared.data.read.Field; +import cn.edu.tsinghua.iginx.engine.shared.data.read.Header; +import cn.edu.tsinghua.iginx.engine.shared.data.read.Row; +import cn.edu.tsinghua.iginx.engine.shared.data.read.RowStream; +import cn.edu.tsinghua.iginx.thrift.DataType; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class AggregatedRowStream implements RowStream { + + private final Row row; + private boolean hasNext = true; + + public AggregatedRowStream(Map values, String functionName) { + List fieldList = new ArrayList<>(values.size()); + List valuesList = new ArrayList<>(values.size()); + for (Map.Entry entry : values.entrySet()) { + Object value = entry.getValue(); + valuesList.add(value); + + Map.Entry> pathWithTags = + DataViewWrapper.parseFieldName(entry.getKey()); + String path = pathWithTags.getKey(); + Map tags = pathWithTags.getValue(); + String pathWithFunctionName = functionName + "(" + path + ")"; + + DataType dataType = typeFromValue(value); + Field field = new Field(pathWithFunctionName, dataType, tags); + fieldList.add(field); + } + row = new Row(new Header(fieldList), valuesList.toArray()); + } + + private static DataType typeFromValue(Object value) { + if (value instanceof Long) { + return DataType.LONG; + } else { + throw new UnsupportedOperationException("Unsupported type: " + value.getClass().getName()); + } + } + + @Override + public Header getHeader() throws PhysicalException { + return row.getHeader(); + } + + @Override + public void close() throws PhysicalException {} + + @Override + public boolean hasNext() throws PhysicalException { + return hasNext; + } + + @Override + public Row next() throws PhysicalException { + hasNext = false; + return row; + } +} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/DataManager.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/DataManager.java similarity index 54% rename from dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/DataManager.java rename to dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/DataManager.java index bbe4a16527..c72093a5ea 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/DataManager.java +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/DataManager.java @@ -1,74 +1,102 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.parquet.manager.data; import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; import cn.edu.tsinghua.iginx.engine.physical.storage.domain.Column; +import cn.edu.tsinghua.iginx.engine.physical.storage.domain.ColumnKey; import cn.edu.tsinghua.iginx.engine.shared.KeyRange; import cn.edu.tsinghua.iginx.engine.shared.data.read.RowStream; import cn.edu.tsinghua.iginx.engine.shared.data.write.DataView; +import cn.edu.tsinghua.iginx.engine.shared.function.FunctionCall; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; -import cn.edu.tsinghua.iginx.metadata.entity.KeyInterval; import cn.edu.tsinghua.iginx.parquet.db.Database; import cn.edu.tsinghua.iginx.parquet.db.lsm.OneTierDB; import cn.edu.tsinghua.iginx.parquet.db.lsm.api.ReadWriter; import cn.edu.tsinghua.iginx.parquet.db.util.AreaSet; import cn.edu.tsinghua.iginx.parquet.db.util.iterator.Scanner; import cn.edu.tsinghua.iginx.parquet.manager.Manager; -import cn.edu.tsinghua.iginx.parquet.manager.utils.RangeUtils; +import cn.edu.tsinghua.iginx.parquet.manager.utils.TagKVUtils; import cn.edu.tsinghua.iginx.parquet.util.Constants; import cn.edu.tsinghua.iginx.parquet.util.Shared; +import cn.edu.tsinghua.iginx.parquet.util.arrow.ArrowFields; import cn.edu.tsinghua.iginx.parquet.util.exception.StorageException; import cn.edu.tsinghua.iginx.thrift.DataType; -import com.google.common.collect.Range; import com.google.common.collect.RangeSet; import java.io.IOException; import java.nio.file.Path; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.arrow.vector.types.pojo.Field; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class DataManager implements Manager { private static final Logger LOGGER = LoggerFactory.getLogger(DataManager.class); - private final Database db; + private final Database db; private final Shared shared; public DataManager(Shared shared, Path dir) throws IOException { this.shared = shared; Path dataDir = dir.resolve(Constants.DIR_NAME_TABLE); - ReadWriter readWriter = new ParquetReadWriter(shared, dataDir); - this.db = new OneTierDB<>(dir.toString(), shared, readWriter); + ReadWriter readWriter = new ParquetReadWriter(shared, dataDir); + this.db = new OneTierDB(dir.toString(), shared, readWriter); } @Override public RowStream project(List paths, TagFilter tagFilter, Filter filter) throws PhysicalException { - Map schemaMatchTags = ProjectUtils.project(db.schema(), tagFilter); + Map schema = ArrowFields.toIginxSchema(db.schema()); + Map schemaMatchTags = ProjectUtils.project(schema, tagFilter); Map projectedSchema = ProjectUtils.project(schemaMatchTags, paths); Filter projectedFilter = ProjectUtils.project(filter, schemaMatchTags); RangeSet rangeSet = FilterRangeUtils.rangeSetOf(projectedFilter); - Scanner> scanner = - db.query(projectedSchema.keySet(), rangeSet, projectedFilter); - return new ScannerRowStream(projectedSchema, scanner); + try { + Scanner> scanner = + db.query(ArrowFields.of(projectedSchema), rangeSet, projectedFilter); + return new ScannerRowStream(projectedSchema, scanner); + } catch (IOException e) { + throw new StorageException(e); + } + } + + public RowStream aggregation(List patterns, TagFilter tagFilter, List calls) + throws PhysicalException { + Map schema = ArrowFields.toIginxSchema(db.schema()); + Map schemaMatchTags = ProjectUtils.project(schema, tagFilter); + Map projectedSchema = ProjectUtils.project(schemaMatchTags, patterns); + + try { + // TODO: just support count now + Map counts = db.count(ArrowFields.of(projectedSchema)); + return new AggregatedRowStream(counts, "count"); + } catch (InterruptedException | IOException e) { + throw new StorageException(e); + } } @Override @@ -84,8 +112,8 @@ public void insert(DataView data) throws PhysicalException { db.upsertColumns(scanner, wrappedData.getSchema()); } } - } catch (Exception e) { - throw new RuntimeException("failed to close scanner of DataView", e); + } catch (InterruptedException e) { + throw new StorageException(e); } } @@ -104,6 +132,7 @@ public void delete(List paths, List keyRanges, TagFilter tagFi } AreaSet areas = new AreaSet<>(); + Map schema = Collections.emptyMap(); if (paths.stream().anyMatch("*"::equals) && tagFilter == null) { if (rangeSet.isEmpty()) { db.clear(); @@ -111,7 +140,8 @@ public void delete(List paths, List keyRanges, TagFilter tagFi areas.add(rangeSet); } } else { - Map schemaMatchedTags = ProjectUtils.project(db.schema(), tagFilter); + schema = ArrowFields.toIginxSchema(db.schema()); + Map schemaMatchedTags = ProjectUtils.project(schema, tagFilter); Set fields = ProjectUtils.project(schemaMatchedTags, paths).keySet(); if (rangeSet.isEmpty()) { areas.add(fields); @@ -121,27 +151,28 @@ public void delete(List paths, List keyRanges, TagFilter tagFi } if (!areas.isEmpty()) { - db.delete(areas); + AreaSet arrowAreas = ArrowFields.of(areas, schema); + db.delete(arrowAreas); } } @Override - public List getColumns() throws StorageException { + public List getColumns(List paths, TagFilter tagFilter) throws StorageException { List columns = new ArrayList<>(); - for (Map.Entry entry : db.schema().entrySet()) { + Map schema = ArrowFields.toIginxSchema(db.schema()); + for (Map.Entry entry : schema.entrySet()) { Map.Entry> pathWithTags = DataViewWrapper.parseFieldName(entry.getKey()); - columns.add(new Column(pathWithTags.getKey(), entry.getValue(), pathWithTags.getValue())); + DataType dataType = entry.getValue(); + ColumnKey columnKey = new ColumnKey(pathWithTags.getKey(), pathWithTags.getValue()); + if (!TagKVUtils.match(columnKey, paths, tagFilter)) { + continue; + } + columns.add(new Column(columnKey.getPath(), dataType, columnKey.getTags())); } return columns; } - @Override - public KeyInterval getKeyInterval() throws PhysicalException { - Optional> optionalRange = db.range(); - return optionalRange.map(RangeUtils::toKeyInterval).orElseGet(() -> new KeyInterval(0, 0)); - } - @Override public void close() throws Exception { db.close(); diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/DataViewWrapper.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/DataViewWrapper.java similarity index 90% rename from dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/DataViewWrapper.java rename to dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/DataViewWrapper.java index d1650919a6..c36a4fe8b2 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/DataViewWrapper.java +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/DataViewWrapper.java @@ -1,17 +1,19 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.parquet.manager.data; @@ -24,7 +26,6 @@ import cn.edu.tsinghua.iginx.parquet.util.exception.StorageException; import cn.edu.tsinghua.iginx.thrift.DataType; import java.util.*; -import javax.annotation.Nonnull; class DataViewWrapper { private final DataView dataView; @@ -83,7 +84,6 @@ public DataViewRowsScanner() { } } - @Nonnull @Override public Long key() throws NoSuchElementException { if (key == null) { @@ -92,7 +92,6 @@ public Long key() throws NoSuchElementException { return key; } - @Nonnull @Override public Scanner value() throws NoSuchElementException { if (rowScanner == null) { @@ -134,7 +133,6 @@ public RowScanner(int keyIndex) { this.keyIndex = keyIndex; } - @Nonnull @Override public String key() throws NoSuchElementException { if (fieldName == null) { @@ -143,7 +141,6 @@ public String key() throws NoSuchElementException { return fieldName; } - @Nonnull @Override public Object value() throws NoSuchElementException { if (fieldName == null) { @@ -188,7 +185,6 @@ public DataViewColumnsScanner() { } } - @Nonnull @Override public String key() throws NoSuchElementException { if (fieldName == null) { @@ -197,7 +193,6 @@ public String key() throws NoSuchElementException { return fieldName; } - @Nonnull @Override public Scanner value() throws NoSuchElementException { if (columnScanner == null) { @@ -240,7 +235,6 @@ public ColumnScanner(int fieldIndex) { this.bitmapView = dataView.getBitmapView(fieldIndex); } - @Nonnull @Override public Long key() throws NoSuchElementException { if (key == null) { @@ -249,7 +243,6 @@ public Long key() throws NoSuchElementException { return key; } - @Nonnull @Override public Object value() throws NoSuchElementException { if (key == null) { diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/FilterRangeUtils.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/FilterRangeUtils.java similarity index 83% rename from dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/FilterRangeUtils.java rename to dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/FilterRangeUtils.java index b1e25d96da..7d268a5e37 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/FilterRangeUtils.java +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/FilterRangeUtils.java @@ -1,17 +1,19 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.parquet.manager.data; diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/LongFormat.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/LongFormat.java new file mode 100644 index 0000000000..8d7e166a4f --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/LongFormat.java @@ -0,0 +1,31 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.parquet.manager.data; + +public class LongFormat implements ObjectFormat { + @Override + public String format(Long value) { + return value.toString(); + } + + @Override + public Long parse(String source) { + return Long.parseLong(source); + } +} diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/ObjectFormat.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/ObjectFormat.java new file mode 100644 index 0000000000..a67d3878b3 --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/ObjectFormat.java @@ -0,0 +1,26 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.parquet.manager.data; + +public interface ObjectFormat { + + String format(V value); + + V parse(String source); +} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/ParquetReadWriter.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/ParquetReadWriter.java similarity index 78% rename from dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/ParquetReadWriter.java rename to dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/ParquetReadWriter.java index 351373c6d5..864aa356d5 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/ParquetReadWriter.java +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/ParquetReadWriter.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.parquet.manager.data; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.AndFilter; @@ -24,15 +42,15 @@ import java.io.IOException; import java.nio.file.*; import java.util.*; -import javax.annotation.Nonnull; -import org.ehcache.sizeof.SizeOf; +import javax.annotation.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import shaded.iginx.org.apache.parquet.hadoop.metadata.ColumnPath; import shaded.iginx.org.apache.parquet.hadoop.metadata.ParquetMetadata; import shaded.iginx.org.apache.parquet.schema.MessageType; import shaded.iginx.org.apache.parquet.schema.Type; -public class ParquetReadWriter implements ReadWriter { +public class ParquetReadWriter implements ReadWriter { private static final Logger LOGGER = LoggerFactory.getLogger(ParquetReadWriter.class); @@ -63,11 +81,14 @@ private void cleanTempFiles() { } } + @Override + public String getName() { + return dir.toString(); + } + @Override public void flush( - String tableName, - TableMeta meta, - Scanner> scanner) + String tableName, TableMeta meta, Scanner> scanner) throws IOException { Path path = getPath(tableName); Path tempPath = dir.resolve(tableName + Constants.SUFFIX_FILE_TEMP); @@ -76,9 +97,11 @@ public void flush( LOGGER.debug("flushing into {}", tempPath); MessageType parquetSchema = getMessageType(meta.getSchema()); - IParquetWriter.Builder builder = IParquetWriter.builder(tempPath, parquetSchema); + int maxBufferSize = shared.getStorageProperties().getParquetOutputBufferMaxSize(); + IParquetWriter.Builder builder = IParquetWriter.builder(tempPath, parquetSchema, maxBufferSize); builder.withRowGroupSize(shared.getStorageProperties().getParquetRowGroupSize()); builder.withPageSize((int) shared.getStorageProperties().getParquetPageSize()); + builder.withCompressionCodec(shared.getStorageProperties().getParquetCompression()); try (IParquetWriter writer = builder.build()) { while (scanner.iterate()) { @@ -86,8 +109,7 @@ public void flush( writer.write(record); } ParquetMetadata parquetMeta = writer.flush(); - ParquetTableMeta tableMeta = - new ParquetTableMeta(meta.getSchema(), meta.getRanges(), parquetMeta); + ParquetTableMeta tableMeta = ParquetTableMeta.of(parquetMeta); setParquetTableMeta(path.toString(), tableMeta); } catch (Exception e) { throw new IOException("failed to write " + path, e); @@ -100,7 +122,6 @@ public void flush( Files.move(tempPath, path, StandardCopyOption.REPLACE_EXISTING); } - @Nonnull private static MessageType getMessageType(Map schema) { List fields = new ArrayList<>(); fields.add( @@ -115,7 +136,7 @@ private static MessageType getMessageType(Map schema) { } @Override - public TableMeta readMeta(String tableName) { + public TableMeta readMeta(String tableName) { Path path = getPath(tableName); ParquetTableMeta tableMeta = getParquetTableMeta(path.toString()); AreaSet tombstone = tombstoneStorage.get(tableName); @@ -129,7 +150,6 @@ private void setParquetTableMeta(String fileName, ParquetTableMeta tableMeta) { shared.getCachePool().asMap().put(fileName, tableMeta); } - @Nonnull private ParquetTableMeta getParquetTableMeta(String fileName) { CachePool.Cacheable cacheable = shared.getCachePool().asMap().computeIfAbsent(fileName, this::doReadMeta); @@ -139,32 +159,12 @@ private ParquetTableMeta getParquetTableMeta(String fileName) { return (ParquetTableMeta) cacheable; } - @Nonnull private ParquetTableMeta doReadMeta(String fileName) { Path path = Paths.get(fileName); try (IParquetReader reader = IParquetReader.builder(path).build()) { - Map schemaDst = new HashMap<>(); - MessageType parquetSchema = reader.getSchema(); - for (int i = 0; i < parquetSchema.getFieldCount(); i++) { - Type type = parquetSchema.getType(i); - if (type.getName().equals(Constants.KEY_FIELD_NAME)) { - continue; - } - DataType iginxType = IParquetReader.toIginxType(type.asPrimitiveType()); - schemaDst.put(type.getName(), iginxType); - } - - Range ranges = reader.getRange(); - - Map> rangeMap = new HashMap<>(); - for (String field : schemaDst.keySet()) { - rangeMap.put(field, ranges); - } - ParquetMetadata meta = reader.getMeta(); - - return new ParquetTableMeta(schemaDst, rangeMap, meta); + return ParquetTableMeta.of(meta); } catch (Exception e) { throw new StorageRuntimeException(e); } @@ -254,14 +254,18 @@ public void clear() throws IOException { shared.getCachePool().asMap().remove(fileName); } } + try (DirectoryStream stream = + Files.newDirectoryStream(dir, "*" + Constants.SUFFIX_FILE_TEMP)) { + for (Path path : stream) { + Files.deleteIfExists(path); + } + } tombstoneStorage.clear(); Files.deleteIfExists(dir); } catch (NoSuchFileException e) { LOGGER.trace("Not a directory to clear: {}", dir); } catch (DirectoryNotEmptyException e) { LOGGER.warn("directory not empty to clear: {}", dir); - } catch (IOException e) { - throw new StorageRuntimeException(e); } } @@ -274,7 +278,6 @@ public ParquetScanner(IParquetReader reader) { this.reader = reader; } - @Nonnull @Override public Long key() throws NoSuchElementException { if (key == null) { @@ -283,7 +286,6 @@ public Long key() throws NoSuchElementException { return key; } - @Nonnull @Override public Scanner value() throws NoSuchElementException { if (rowScanner == null) { @@ -330,22 +332,53 @@ public void close() throws StorageException { } } - private static class ParquetTableMeta - implements TableMeta, CachePool.Cacheable { + private static class ParquetTableMeta implements TableMeta, CachePool.Cacheable { private final Map schemaDst; private final Map> rangeMap; + private final Map countMap; private final ParquetMetadata meta; - private final int weight; - public ParquetTableMeta( - Map schemaDst, Map> rangeMap, ParquetMetadata meta) { + public static ParquetTableMeta of(ParquetMetadata meta) { + Map schemaDst = new HashMap<>(); + Map> rangeMap = new HashMap<>(); + Map countMap = new HashMap<>(); + MessageType parquetSchema = meta.getFileMetaData().getSchema(); + + Range ranges = IParquetReader.getRangeOf(meta); + + for (int i = 0; i < parquetSchema.getFieldCount(); i++) { + Type type = parquetSchema.getType(i); + if (type.getName().equals(Constants.KEY_FIELD_NAME)) { + continue; + } + DataType iginxType = IParquetReader.toIginxType(type.asPrimitiveType()); + schemaDst.put(type.getName(), iginxType); + rangeMap.put(type.getName(), ranges); + } + + Map columnPathMap = IParquetReader.getCountsOf(meta); + columnPathMap.forEach( + (columnPath, count) -> { + String[] columnPathArray = columnPath.toArray(); + if (columnPathArray.length != 1) { + throw new IllegalStateException("invalid column path: " + columnPath); + } + String name = columnPath.toArray()[0]; + countMap.put(name, count); + }); + + return new ParquetTableMeta(schemaDst, rangeMap, countMap, meta); + } + + ParquetTableMeta( + Map schemaDst, + Map> rangeMap, + Map countMap, + ParquetMetadata meta) { this.schemaDst = schemaDst; this.rangeMap = rangeMap; + this.countMap = countMap; this.meta = meta; - int schemaWeight = schemaDst.toString().length(); - int rangeWeight = rangeMap.toString().length(); - int metaWeight = (int) SizeOf.newInstance().deepSizeOf(meta); - this.weight = schemaWeight + rangeWeight + metaWeight; } @Override @@ -354,17 +387,24 @@ public Map getSchema() { } @Override - public Map> getRanges() { - return rangeMap; + public Range getRange(String field) { + if (!schemaDst.containsKey(field)) { + throw new NoSuchElementException(); + } + return rangeMap.get(field); } - public ParquetMetadata getMeta() { - return meta; + @Nullable + @Override + public Long getValueCount(String field) { + if (!schemaDst.containsKey(field)) { + throw new NoSuchElementException(); + } + return countMap.get(field); } - @Override - public int getWeight() { - return weight; + public ParquetMetadata getMeta() { + return meta; } } } diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/ProjectUtils.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/ProjectUtils.java similarity index 90% rename from dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/ProjectUtils.java rename to dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/ProjectUtils.java index 613146041c..799f3d9810 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/ProjectUtils.java +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/ProjectUtils.java @@ -1,17 +1,19 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.parquet.manager.data; diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/ScannerRowStream.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/ScannerRowStream.java similarity index 77% rename from dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/ScannerRowStream.java rename to dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/ScannerRowStream.java index dd02c08600..e86102a56a 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/ScannerRowStream.java +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/ScannerRowStream.java @@ -1,17 +1,19 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.parquet.manager.data; diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/SerializeUtils.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/SerializeUtils.java similarity index 89% rename from dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/SerializeUtils.java rename to dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/SerializeUtils.java index 95222d6eec..4341052f78 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/SerializeUtils.java +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/SerializeUtils.java @@ -1,17 +1,19 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.parquet.manager.data; diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/SizeUtils.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/SizeUtils.java new file mode 100644 index 0000000000..4cc8f14494 --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/SizeUtils.java @@ -0,0 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.parquet.manager.data; + +public class SizeUtils {} diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/StringFormat.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/StringFormat.java new file mode 100644 index 0000000000..7963d81eaf --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/StringFormat.java @@ -0,0 +1,31 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.parquet.manager.data; + +public class StringFormat implements ObjectFormat { + @Override + public String format(String value) { + return value; + } + + @Override + public String parse(String source) { + return source; + } +} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/TombstoneStorage.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/TombstoneStorage.java similarity index 89% rename from dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/TombstoneStorage.java rename to dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/TombstoneStorage.java index 7e2fbadd5a..f0ad22fb29 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/TombstoneStorage.java +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/TombstoneStorage.java @@ -1,17 +1,19 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.parquet.manager.data; @@ -187,11 +189,8 @@ public static class CachedTombstone, String> private final AreaSet areas; - private final int weight; - public CachedTombstone(AreaSet areas, int size) { this.areas = areas; - this.weight = size + 32; } public CachedTombstone(int length) { @@ -201,10 +200,5 @@ public CachedTombstone(int length) { public AreaSet getTombstone() { return areas == null ? new AreaSet<>() : areas; } - - @Override - public int getWeight() { - return weight; - } } } diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/Column.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/Column.java similarity index 59% rename from dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/Column.java rename to dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/Column.java index abc460a87e..fa393f3b41 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/Column.java +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/Column.java @@ -1,17 +1,19 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.parquet.manager.dummy; diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/DummyManager.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/DummyManager.java similarity index 75% rename from dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/DummyManager.java rename to dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/DummyManager.java index 24afeaddee..00157eba1d 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/DummyManager.java +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/DummyManager.java @@ -1,17 +1,19 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.parquet.manager.dummy; @@ -26,15 +28,17 @@ import cn.edu.tsinghua.iginx.engine.shared.data.write.DataView; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; -import cn.edu.tsinghua.iginx.metadata.entity.KeyInterval; import cn.edu.tsinghua.iginx.parquet.manager.Manager; import cn.edu.tsinghua.iginx.parquet.manager.utils.TagKVUtils; -import cn.edu.tsinghua.iginx.utils.StringUtils; import java.io.IOException; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; import javax.validation.constraints.NotNull; @@ -103,21 +107,9 @@ private List determinePathList( Set paths, List patterns, TagFilter tagFilter) { List ret = new ArrayList<>(); for (String path : paths) { - for (String pattern : patterns) { - ColumnKey columnKey = TagKVUtils.splitFullName(path); - if (tagFilter == null) { - if (StringUtils.match(columnKey.getPath(), pattern)) { - ret.add(path); - break; - } - } else { - if (StringUtils.match(columnKey.getPath(), pattern) - && cn.edu.tsinghua.iginx.engine.physical.storage.utils.TagKVUtils.match( - columnKey.getTags(), tagFilter)) { - ret.add(path); - break; - } - } + ColumnKey columnKey = TagKVUtils.splitFullName(path); + if (TagKVUtils.match(columnKey, patterns, tagFilter)) { + ret.add(path); } } return ret; @@ -135,16 +127,18 @@ public void delete(List paths, List keyRanges, TagFilter tagFi } @Override - public List getColumns() throws PhysicalException { + public List getColumns(List paths, TagFilter tagFilter) throws PhysicalException { List columns = new ArrayList<>(); for (Path path : getFilePaths()) { try { List fields = new Loader(path).getHeader(); for (Field field : fields) { - ColumnKey columnKey = TagKVUtils.splitFullName(field.getName()); + ColumnKey columnKey = TagKVUtils.splitFullName(prefix + "." + field.getName()); + if (!TagKVUtils.match(columnKey, paths, tagFilter)) { + continue; + } Column column = - new Column( - prefix + "." + columnKey.getPath(), field.getType(), columnKey.getTags(), true); + new Column(columnKey.getPath(), field.getType(), columnKey.getTags(), true); columns.add(column); } } catch (IOException e) { @@ -154,11 +148,6 @@ public List getColumns() throws PhysicalException { return columns; } - @Override - public KeyInterval getKeyInterval() { - return KeyInterval.getDefaultKeyInterval(); - } - @Override public void close() throws IOException { LOGGER.info("{} closed", this); diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/EmptyManager.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/EmptyManager.java similarity index 68% rename from dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/EmptyManager.java rename to dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/EmptyManager.java index cbc6daf1b6..a8f9bf9af8 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/EmptyManager.java +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/EmptyManager.java @@ -1,17 +1,19 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.parquet.manager.dummy; @@ -26,7 +28,6 @@ import cn.edu.tsinghua.iginx.engine.shared.data.write.DataView; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; -import cn.edu.tsinghua.iginx.metadata.entity.KeyInterval; import cn.edu.tsinghua.iginx.parquet.manager.Manager; import java.util.Collections; import java.util.List; @@ -71,15 +72,10 @@ public void delete(List paths, List keyRanges, TagFilter tagFi } @Override - public List getColumns() throws PhysicalException { + public List getColumns(List paths, TagFilter tagFilter) { return Collections.emptyList(); } - @Override - public KeyInterval getKeyInterval() throws PhysicalException { - return new KeyInterval(0, 0); - } - @Override public void close() throws Exception {} } diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/Field.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/Field.java similarity index 57% rename from dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/Field.java rename to dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/Field.java index dc2bfb50c3..db2c57c838 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/Field.java +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/Field.java @@ -1,17 +1,19 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.parquet.manager.dummy; diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/Loader.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/Loader.java similarity index 88% rename from dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/Loader.java rename to dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/Loader.java index f3a1f67b6b..c6af146fae 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/Loader.java +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/Loader.java @@ -1,17 +1,19 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.parquet.manager.dummy; diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/NewQueryRowStream.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/NewQueryRowStream.java similarity index 74% rename from dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/NewQueryRowStream.java rename to dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/NewQueryRowStream.java index 08bed34b4b..34a038e0d4 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/NewQueryRowStream.java +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/NewQueryRowStream.java @@ -1,17 +1,19 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.parquet.manager.dummy; diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/Storer.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/Storer.java similarity index 80% rename from dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/Storer.java rename to dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/Storer.java index b1278eaa37..da74ee87e5 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/Storer.java +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/Storer.java @@ -1,17 +1,19 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.parquet.manager.dummy; diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/Table.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/Table.java similarity index 89% rename from dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/Table.java rename to dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/Table.java index 6dad70d6ec..ce3348d8ca 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/Table.java +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/Table.java @@ -1,17 +1,19 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.parquet.manager.dummy; diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/utils/RangeUtils.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/utils/RangeUtils.java similarity index 51% rename from dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/utils/RangeUtils.java rename to dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/utils/RangeUtils.java index 38f89068ea..2f24acba31 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/utils/RangeUtils.java +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/utils/RangeUtils.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.parquet.manager.utils; import cn.edu.tsinghua.iginx.metadata.entity.KeyInterval; diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/utils/TagKVUtils.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/utils/TagKVUtils.java new file mode 100644 index 0000000000..92a4b9bd21 --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/utils/TagKVUtils.java @@ -0,0 +1,82 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.parquet.manager.utils; + +import cn.edu.tsinghua.iginx.engine.physical.storage.domain.ColumnKey; +import cn.edu.tsinghua.iginx.engine.physical.storage.utils.ColumnKeyTranslator; +import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; +import cn.edu.tsinghua.iginx.utils.Escaper; +import cn.edu.tsinghua.iginx.utils.StringUtils; +import java.text.ParseException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class TagKVUtils { + private static final ColumnKeyTranslator COLUMN_KEY_TRANSLATOR = + new ColumnKeyTranslator(',', '=', getEscaper()); + + private static Escaper getEscaper() { + Map replacementMap = new HashMap<>(); + replacementMap.put('\\', '\\'); + replacementMap.put(',', ','); + replacementMap.put('=', '='); + return new Escaper('\\', replacementMap); + } + + public static String toFullName(String name, Map tags) { + if (tags == null) { + tags = Collections.emptyMap(); + } + ColumnKey columnKey = new ColumnKey(name, tags); + return toFullName(columnKey); + } + + public static String toFullName(ColumnKey columnKey) { + Objects.requireNonNull(columnKey); + return COLUMN_KEY_TRANSLATOR.translate(columnKey); + } + + public static ColumnKey splitFullName(String fullName) { + try { + return COLUMN_KEY_TRANSLATOR.translate(fullName); + } catch (ParseException e) { + throw new IllegalStateException("Failed to parse identifier: " + fullName, e); + } + } + + public static boolean match(ColumnKey columnKey, List patterns, TagFilter tagFilter) { + for (String pattern : patterns) { + if (tagFilter == null) { + if (StringUtils.match(columnKey.getPath(), pattern)) { + return true; + } + } else { + if (StringUtils.match(columnKey.getPath(), pattern) + && cn.edu.tsinghua.iginx.engine.physical.storage.utils.TagKVUtils.match( + columnKey.getTags(), tagFilter)) { + return true; + } + } + } + return false; + } +} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/server/FilterTransformer.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/server/FilterTransformer.java similarity index 91% rename from dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/server/FilterTransformer.java rename to dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/server/FilterTransformer.java index bce5c1cd14..9c04d60b41 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/server/FilterTransformer.java +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/server/FilterTransformer.java @@ -1,17 +1,19 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.parquet.server; diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/server/ParquetServer.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/server/ParquetServer.java similarity index 59% rename from dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/server/ParquetServer.java rename to dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/server/ParquetServer.java index 25d5734309..0b2f2c6c06 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/server/ParquetServer.java +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/server/ParquetServer.java @@ -1,17 +1,19 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.parquet.server; @@ -20,6 +22,11 @@ import cn.edu.tsinghua.iginx.conf.ConfigDescriptor; import cn.edu.tsinghua.iginx.parquet.exec.Executor; import cn.edu.tsinghua.iginx.parquet.thrift.ParquetService; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import org.apache.thrift.TProcessor; import org.apache.thrift.protocol.TBinaryProtocol; import org.apache.thrift.server.TServer; @@ -71,11 +78,18 @@ public void run() { } return; } + ExecutorService executorService = + new ThreadPoolExecutor( + config.getMinThriftWorkerThreadNum(), + config.getMaxThriftWrokerThreadNum(), + 60L, + TimeUnit.SECONDS, + new SynchronousQueue<>(), + new ThreadFactoryBuilder().setNameFormat("parquet-server-" + port + "-%d").build()); TThreadPoolServer.Args args = new TThreadPoolServer.Args(serverTransport) .processor(processor) - .minWorkerThreads(config.getMinThriftWorkerThreadNum()) - .maxWorkerThreads(config.getMaxThriftWrokerThreadNum()) + .executorService(executorService) .protocolFactory(new TBinaryProtocol.Factory()); server = new TThreadPoolServer(args); LOGGER.info("Parquet service starts successfully!"); diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/server/ParquetWorker.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/server/ParquetWorker.java similarity index 76% rename from dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/server/ParquetWorker.java rename to dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/server/ParquetWorker.java index 3201bbf694..503f774ea5 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/server/ParquetWorker.java +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/server/ParquetWorker.java @@ -1,17 +1,19 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.parquet.server; @@ -27,6 +29,10 @@ import cn.edu.tsinghua.iginx.engine.shared.data.write.RawData; import cn.edu.tsinghua.iginx.engine.shared.data.write.RawDataType; import cn.edu.tsinghua.iginx.engine.shared.data.write.RowDataView; +import cn.edu.tsinghua.iginx.engine.shared.function.Function; +import cn.edu.tsinghua.iginx.engine.shared.function.FunctionCall; +import cn.edu.tsinghua.iginx.engine.shared.function.FunctionParams; +import cn.edu.tsinghua.iginx.engine.shared.function.system.Count; import cn.edu.tsinghua.iginx.engine.shared.operator.tag.AndTagFilter; import cn.edu.tsinghua.iginx.engine.shared.operator.tag.BasePreciseTagFilter; import cn.edu.tsinghua.iginx.engine.shared.operator.tag.BaseTagFilter; @@ -37,7 +43,23 @@ import cn.edu.tsinghua.iginx.metadata.entity.ColumnsInterval; import cn.edu.tsinghua.iginx.metadata.entity.KeyInterval; import cn.edu.tsinghua.iginx.parquet.exec.Executor; -import cn.edu.tsinghua.iginx.parquet.thrift.*; +import cn.edu.tsinghua.iginx.parquet.thrift.DeleteReq; +import cn.edu.tsinghua.iginx.parquet.thrift.GetColumnsOfStorageUnitResp; +import cn.edu.tsinghua.iginx.parquet.thrift.GetStorageBoundaryResp; +import cn.edu.tsinghua.iginx.parquet.thrift.InsertReq; +import cn.edu.tsinghua.iginx.parquet.thrift.ParquetHeader; +import cn.edu.tsinghua.iginx.parquet.thrift.ParquetKeyRange; +import cn.edu.tsinghua.iginx.parquet.thrift.ParquetRawData; +import cn.edu.tsinghua.iginx.parquet.thrift.ParquetRow; +import cn.edu.tsinghua.iginx.parquet.thrift.ParquetService; +import cn.edu.tsinghua.iginx.parquet.thrift.ProjectReq; +import cn.edu.tsinghua.iginx.parquet.thrift.ProjectResp; +import cn.edu.tsinghua.iginx.parquet.thrift.RawFunction; +import cn.edu.tsinghua.iginx.parquet.thrift.RawFunctionCall; +import cn.edu.tsinghua.iginx.parquet.thrift.RawFunctionParams; +import cn.edu.tsinghua.iginx.parquet.thrift.RawTagFilter; +import cn.edu.tsinghua.iginx.parquet.thrift.Status; +import cn.edu.tsinghua.iginx.parquet.thrift.TS; import cn.edu.tsinghua.iginx.thrift.DataType; import cn.edu.tsinghua.iginx.utils.Bitmap; import cn.edu.tsinghua.iginx.utils.ByteUtils; @@ -49,8 +71,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; -import org.apache.thrift.TException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -77,14 +99,23 @@ public ParquetWorker(Executor executor) { } @Override - public ProjectResp executeProject(ProjectReq req) throws TException { + public ProjectResp executeProject(ProjectReq req) { TagFilter tagFilter = resolveRawTagFilter(req.getTagFilter()); + List calls = null; + if (req.getAggregations() != null) { + calls = + req.getAggregations().stream() + .map(this::resolveRawFunctionCall) + .collect(Collectors.toList()); + } + TaskExecuteResult result = executor.executeProjectTask( req.getPaths(), tagFilter, FilterTransformer.toFilter(req.getFilter()), + calls, req.getStorageUnit(), req.isDummyStorageUnit); @@ -151,7 +182,7 @@ public ProjectResp executeProject(ProjectReq req) throws TException { } @Override - public Status executeInsert(InsertReq req) throws TException { + public Status executeInsert(InsertReq req) { ParquetRawData parquetRawData = req.getRawData(); RawDataType rawDataType = strToRawDataType(parquetRawData.getRawDataType()); if (rawDataType == null) { @@ -222,7 +253,7 @@ private RawDataType strToRawDataType(String type) { } @Override - public Status executeDelete(DeleteReq req) throws TException { + public Status executeDelete(DeleteReq req) { TagFilter tagFilter = resolveRawTagFilter(req.getTagFilter()); // null timeRanges means delete columns @@ -282,11 +313,30 @@ private TagFilter resolveRawTagFilter(RawTagFilter rawTagFilter) { } } + private FunctionCall resolveRawFunctionCall(RawFunctionCall rawFunctionCall) { + Function function = resolveRawFunction(rawFunctionCall.getFunc()); + FunctionParams params = resolveRawFunctionParams(rawFunctionCall.getParams()); + return new FunctionCall(function, params); + } + + private Function resolveRawFunction(RawFunction rawFunction) { + if (rawFunction.getId().equals(Count.COUNT)) { + return Count.getInstance(); + } + throw new UnsupportedOperationException("unsupported function: " + rawFunction); + } + + private FunctionParams resolveRawFunctionParams(RawFunctionParams rawFunctionParams) { + return new FunctionParams(rawFunctionParams.getPatterns()); + } + @Override - public GetColumnsOfStorageUnitResp getColumnsOfStorageUnit(String storageUnit) throws TException { + public GetColumnsOfStorageUnitResp getColumnsOfStorageUnit( + String storageUnit, Set patterns, RawTagFilter tagFilter) { List ret = new ArrayList<>(); try { - List tsList = executor.getColumnsOfStorageUnit(storageUnit); + List tsList = + executor.getColumnsOfStorageUnit(storageUnit, patterns, resolveRawTagFilter(tagFilter)); tsList.forEach( timeseries -> { TS ts = new TS(timeseries.getPath(), timeseries.getDataType().toString()); @@ -305,7 +355,7 @@ public GetColumnsOfStorageUnitResp getColumnsOfStorageUnit(String storageUnit) t } @Override - public GetStorageBoundaryResp getBoundaryOfStorage(String dataPrefix) throws TException { + public GetStorageBoundaryResp getBoundaryOfStorage(String dataPrefix) { try { Pair pair = executor.getBoundaryOfStorage(dataPrefix); GetStorageBoundaryResp resp = new GetStorageBoundaryResp(SUCCESS); diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/Awaitable.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/Awaitable.java new file mode 100644 index 0000000000..ac60a9eed0 --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/Awaitable.java @@ -0,0 +1,22 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cn.edu.tsinghua.iginx.parquet.util; + +public interface Awaitable { + void await() throws InterruptedException; +} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/CachePool.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/CachePool.java similarity index 54% rename from dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/CachePool.java rename to dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/CachePool.java index e25c496768..7c17a66173 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/CachePool.java +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/CachePool.java @@ -1,17 +1,19 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.parquet.util; @@ -21,6 +23,7 @@ import com.github.benmanes.caffeine.cache.Scheduler; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.Executors; +import org.ehcache.sizeof.SizeOf; public class CachePool { @@ -44,6 +47,8 @@ public ConcurrentMap asMap() { } public interface Cacheable { - int getWeight(); + default int getWeight() { + return Math.toIntExact(SizeOf.newInstance().deepSizeOf(this)); + } } } diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/CloseableHolders.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/CloseableHolders.java new file mode 100644 index 0000000000..083b82491f --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/CloseableHolders.java @@ -0,0 +1,58 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cn.edu.tsinghua.iginx.parquet.util; + +import java.util.NoSuchElementException; +import org.apache.arrow.util.Preconditions; + +public class CloseableHolders { + + public static class NoexceptAutoCloseableHolder + implements NoexceptAutoCloseable { + private T closeable; + + NoexceptAutoCloseableHolder(T closeable) { + this.closeable = Preconditions.checkNotNull(closeable); + } + + public T transfer() { + if (closeable == null) { + throw new NoSuchElementException("Already transferred"); + } + T ret = closeable; + closeable = null; + return ret; + } + + public T peek() { + return closeable; + } + + @Override + public void close() { + if (closeable != null) { + closeable.close(); + closeable = null; + } + } + } + + public static NoexceptAutoCloseableHolder hold(T closeable) { + return new NoexceptAutoCloseableHolder<>(closeable); + } +} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/Constants.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/Constants.java similarity index 57% rename from dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/Constants.java rename to dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/Constants.java index 4193891785..706ca47a01 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/Constants.java +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/Constants.java @@ -1,17 +1,19 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.parquet.util; @@ -38,7 +40,7 @@ public final class Constants { public static final long SIZE_INSERT_BATCH = 4096; - public static final String KEY_FIELD_NAME = "*"; + public static final String KEY_FIELD_NAME = cn.edu.tsinghua.iginx.engine.shared.Constants.KEY; public static final String RECORD_FIELD_NAME = "iginx"; public static final String DIR_DB_LSM = "lsm"; @@ -51,4 +53,5 @@ public final class Constants { public static final String DIR_NAME_TOMBSTONE = "tombstones"; public static final String DIR_NAME_TABLE = "tables"; public static final String LOCK_FILE_NAME = "LOCK"; + public static final String INDICES_VECTOR_NAME = "indices"; } diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/NoexceptAutoCloseable.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/NoexceptAutoCloseable.java new file mode 100644 index 0000000000..139786fdd8 --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/NoexceptAutoCloseable.java @@ -0,0 +1,23 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cn.edu.tsinghua.iginx.parquet.util; + +public interface NoexceptAutoCloseable extends AutoCloseable { + @Override + void close(); +} diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/NoexceptAutoCloseables.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/NoexceptAutoCloseables.java new file mode 100644 index 0000000000..38ef81a0e9 --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/NoexceptAutoCloseables.java @@ -0,0 +1,37 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cn.edu.tsinghua.iginx.parquet.util; + +import org.apache.arrow.util.AutoCloseables; + +public class NoexceptAutoCloseables { + + public static void close(Iterable closeables) { + try { + AutoCloseables.close(closeables); + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new IllegalStateException(e); + } + } + + public static NoexceptAutoCloseable all(Iterable closeables) { + return () -> close(closeables); + } +} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/ParseUtils.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/ParseUtils.java similarity index 79% rename from dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/ParseUtils.java rename to dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/ParseUtils.java index ca839e3b15..12eba3c385 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/ParseUtils.java +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/ParseUtils.java @@ -1,17 +1,19 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.parquet.util; diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/Shared.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/Shared.java new file mode 100644 index 0000000000..203d817a9f --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/Shared.java @@ -0,0 +1,76 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.parquet.util; + +import java.util.concurrent.Semaphore; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; + +public class Shared { + private final StorageProperties storageProperties; + + private final Semaphore flusherPermits; + + private final Semaphore memTablePermits; + + private final CachePool cachePool; + + private final BufferAllocator allocator; + + public Shared( + StorageProperties storageProperties, + Semaphore flusherPermits, + Semaphore memTablePermits, + CachePool cachePool, + BufferAllocator allocator) { + this.storageProperties = storageProperties; + this.flusherPermits = flusherPermits; + this.memTablePermits = memTablePermits; + this.cachePool = cachePool; + this.allocator = allocator; + } + + public static Shared of(StorageProperties storageProperties) { + Semaphore flusherPermits = new Semaphore(storageProperties.getCompactPermits(), true); + Semaphore memTablePermits = new Semaphore(storageProperties.getWriteBufferPermits(), true); + CachePool cachePool = new CachePool(storageProperties); + BufferAllocator allocator = new RootAllocator(); + return new Shared(storageProperties, flusherPermits, memTablePermits, cachePool, allocator); + } + + public StorageProperties getStorageProperties() { + return storageProperties; + } + + public Semaphore getFlusherPermits() { + return flusherPermits; + } + + public Semaphore getMemTablePermits() { + return memTablePermits; + } + + public CachePool getCachePool() { + return cachePool; + } + + public BufferAllocator getAllocator() { + return allocator; + } +} diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/SingleCache.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/SingleCache.java new file mode 100644 index 0000000000..14e694c0b3 --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/SingleCache.java @@ -0,0 +1,44 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cn.edu.tsinghua.iginx.parquet.util; + +import java.util.function.Consumer; +import java.util.function.Supplier; + +public class SingleCache { + private T obj; + private final Supplier supplier; + + public SingleCache(Supplier supplier) { + this.supplier = supplier; + } + + public T get() { + if (obj == null) { + obj = supplier.get(); + } + return obj; + } + + public void invalidate(Consumer consumer) { + if (obj != null) { + consumer.accept(obj); + obj = null; + } + } +} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/StorageProperties.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/StorageProperties.java similarity index 66% rename from dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/StorageProperties.java rename to dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/StorageProperties.java index 5538ccd92a..b44308783b 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/StorageProperties.java +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/StorageProperties.java @@ -1,59 +1,60 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.parquet.util; +import cn.edu.tsinghua.iginx.parquet.db.lsm.buffer.chunk.IndexedChunk; +import cn.edu.tsinghua.iginx.parquet.db.lsm.buffer.chunk.IndexedChunkType; import java.time.Duration; import java.util.Map; import java.util.Optional; import java.util.StringJoiner; +import org.apache.arrow.vector.BaseValueVector; /** The properties of storage engine */ public class StorageProperties { private final boolean flushOnClose; private final long writeBufferSize; + private final int writeBufferChunkValuesMax; + private final int writeBufferChunkValuesMin; + private final IndexedChunkType writeBufferChunkType; private final Duration writeBufferTimeout; private final long writeBatchSize; private final int compactPermits; + private final int writeBufferPermits; private final long cacheCapacity; private final Duration cacheTimeout; private final boolean cacheSoftValues; private final long parquetRowGroupSize; private final long parquetPageSize; + private final int parquetOutputBufferMaxSize; private final String parquetCompression; private final int zstdLevel; private final int zstdWorkers; private final int parquetLz4BufferSize; - /** - * Construct a StorageProperties - * - * @param flushOnClose whether to flush on close - * @param writeBufferSize the size of write buffer, bytes - * @param writeBatchSize the size of write batch, bytes - * @param compactPermits the number of flusher permits - * @param cacheCapacity the capacity of cache, bytes - * @param cacheTimeout the expiry timeout of cache - * @param cacheSoftValues whether to enable soft values of cache - * @param parquetRowGroupSize the size of parquet row group, bytes - * @param parquetPageSize the size of parquet page, bytes - */ private StorageProperties( boolean flushOnClose, long writeBufferSize, + int writeBufferPermits, + int writeBufferChunkValuesMax, + int writeBufferChunkValuesMin, + IndexedChunkType writeBufferChunkType, Duration writeBufferTimeout, long writeBatchSize, int compactPermits, @@ -62,20 +63,26 @@ private StorageProperties( boolean cacheSoftValues, long parquetRowGroupSize, long parquetPageSize, + int parquetOutputBufferMaxSize, String parquetCompression, int zstdLevel, int zstdWorkers, int parquetLz4BufferSize) { this.flushOnClose = flushOnClose; this.writeBufferSize = writeBufferSize; + this.writeBufferChunkValuesMax = writeBufferChunkValuesMax; + this.writeBufferChunkValuesMin = writeBufferChunkValuesMin; + this.writeBufferChunkType = writeBufferChunkType; this.writeBufferTimeout = writeBufferTimeout; this.writeBatchSize = writeBatchSize; this.compactPermits = compactPermits; + this.writeBufferPermits = writeBufferPermits; this.cacheCapacity = cacheCapacity; this.cacheTimeout = cacheTimeout; this.cacheSoftValues = cacheSoftValues; this.parquetRowGroupSize = parquetRowGroupSize; this.parquetPageSize = parquetPageSize; + this.parquetOutputBufferMaxSize = parquetOutputBufferMaxSize; this.parquetCompression = parquetCompression; this.zstdLevel = zstdLevel; this.zstdWorkers = zstdWorkers; @@ -100,6 +107,43 @@ public long getWriteBufferSize() { return writeBufferSize; } + /** + * Get the shared permits of write buffer, which is used to control the total number of write + * buffer + * + * @return the number of write buffer permits + */ + public int getWriteBufferPermits() { + return writeBufferPermits; + } + + /** + * Get the max number of write buffer chunk values + * + * @return the max number of write buffer chunk values + */ + public int getWriteBufferChunkValuesMax() { + return writeBufferChunkValuesMax; + } + + /** + * Get the min number of write buffer chunk values + * + * @return the min number of write buffer chunk values + */ + public int getWriteBufferChunkValuesMin() { + return writeBufferChunkValuesMin; + } + + /** + * Get the write buffer chunk factory + * + * @return the write buffer chunk factory + */ + public IndexedChunk.Factory getWriteBufferChunkFactory() { + return writeBufferChunkType.factory(); + } + /** * Get the timeout of write buffer to flush * @@ -119,8 +163,7 @@ public long getWriteBatchSize() { } /** - * Get the shared permits allocator of flusher, which is used to control the total number of - * flusher + * Get the shared permits of flusher, which is used to control the total number of flusher * * @return the shared permits allocator of flusher */ @@ -173,6 +216,10 @@ public long getParquetPageSize() { return parquetPageSize; } + public int getParquetOutputBufferMaxSize() { + return parquetOutputBufferMaxSize; + } + /** * Get the parquet compression codec name * @@ -221,7 +268,12 @@ public static Builder builder() { @Override public String toString() { return new StringJoiner(", ", StorageProperties.class.getSimpleName() + "[", "]") + .add("flushOnClose=" + flushOnClose) .add("writeBufferSize=" + writeBufferSize) + .add("writeBufferPermits=" + writeBufferPermits) + .add("writeBufferChunkValuesMax=" + writeBufferChunkValuesMax) + .add("writeBufferChunkValuesMin=" + writeBufferChunkValuesMin) + .add("writeBufferChunkType=" + writeBufferChunkType) .add("writeBufferTimeout=" + writeBufferTimeout) .add("writeBatchSize=" + writeBatchSize) .add("compactPermits=" + compactPermits) @@ -230,6 +282,7 @@ public String toString() { .add("cacheSoftValues=" + cacheSoftValues) .add("parquetRowGroupSize=" + parquetRowGroupSize) .add("parquetPageSize=" + parquetPageSize) + .add("parquetOutputBufferMaxSize=" + parquetOutputBufferMaxSize) .add("parquetCompression='" + parquetCompression + "'") .add("zstdLevel=" + zstdLevel) .add("zstdWorkers=" + zstdWorkers) @@ -241,6 +294,10 @@ public String toString() { public static class Builder { public static final String FLUSH_ON_CLOSE = "close.flush"; public static final String WRITE_BUFFER_SIZE = "write.buffer.size"; + public static final String WRITE_BUFFER_PERMITS = "write.buffer.permits"; + public static final String WRITE_BUFFER_CHUNK_VALUES_MAX = "write.buffer.chunk.values.max"; + public static final String WRITE_BUFFER_CHUNK_VALUES_MIN = "write.buffer.chunk.values.min"; + public static final String WRITE_BUFFER_CHUNK_INDEX = "write.buffer.chunk.index"; public static final String WRITE_BUFFER_TIMEOUT = "write.buffer.timeout"; public static final String WRITE_BATCH_SIZE = "write.batch.size"; public static final String COMPACT_PERMITS = "compact.permits"; @@ -249,6 +306,7 @@ public static class Builder { public static final String CACHE_VALUE_SOFT = "cache.value.soft"; public static final String PARQUET_BLOCK_SIZE = "parquet.block.size"; public static final String PARQUET_PAGE_SIZE = "parquet.page.size"; + public static final String PARQUET_OUTPUT_BUFFER_SIZE = "parquet.output.buffer.size"; public static final String PARQUET_COMPRESSOR = "parquet.compression"; public static final String ZSTD_LEVEL = "zstd.level"; public static final String ZSTD_WORKERS = "zstd.workers"; @@ -256,6 +314,10 @@ public static class Builder { private boolean flushOnClose = true; private long writeBufferSize = 100 * 1024 * 1024; // BYTE + private int writeBufferPermits = 2; + private int writeBufferChunkValuesMax = BaseValueVector.INITIAL_VALUE_ALLOCATION; + private int writeBufferChunkValuesMin = BaseValueVector.INITIAL_VALUE_ALLOCATION; + private IndexedChunkType writeBufferChunkIndex = IndexedChunkType.NONE; private Duration writeBufferTimeout = Duration.ofSeconds(0); private long writeBatchSize = 1024 * 1024; // BYTE private long cacheCapacity = 16 * 1024 * 1024; // BYTE @@ -264,6 +326,7 @@ public static class Builder { private int compactPermits = 2; private long parquetRowGroupSize = 128 * 1024 * 1024; // BYTE private long parquetPageSize = 8 * 1024; // BYTE + private int parquetOutputBufferMaxSize = 256 * 1024; // BYTE private String parquetCompression = "UNCOMPRESSED"; private int zstdLevel = 3; private int zstdWorkers = 0; @@ -294,6 +357,54 @@ public Builder setWriteBufferSize(long writeBufferSize) { return this; } + /** + * Set the shared permits of write buffer, which is used to control the total number of write + * buffer + * + * @param writeBufferPermits the number of write buffer permits + * @return this builder + */ + public Builder setWriteBufferPermits(int writeBufferPermits) { + ParseUtils.checkPositive(writeBufferPermits); + this.writeBufferPermits = writeBufferPermits; + return this; + } + + /** + * Set the max number of write buffer chunk values + * + * @param writeBufferChunkValuesMax the max number of write buffer chunk values + * @return this builder + */ + public Builder setWriteBufferChunkValuesMax(int writeBufferChunkValuesMax) { + ParseUtils.checkPositive(writeBufferChunkValuesMax); + this.writeBufferChunkValuesMax = writeBufferChunkValuesMax; + return this; + } + + /** + * Set the min number of write buffer chunk values + * + * @param writeBufferChunkValuesMin the max number of write buffer chunk values + * @return this builder + */ + public Builder setWriteBufferChunkValuesMin(int writeBufferChunkValuesMin) { + ParseUtils.checkPositive(writeBufferChunkValuesMin); + this.writeBufferChunkValuesMin = writeBufferChunkValuesMin; + return this; + } + + /** + * Set the write buffer chunk index + * + * @param writeBufferChunkIndexName the write buffer chunk index name + * @return this builder + */ + public Builder setWriteBufferChunkIndex(String writeBufferChunkIndexName) { + this.writeBufferChunkIndex = IndexedChunkType.valueOf(writeBufferChunkIndexName); + return this; + } + /** * Set the timeout of write buffer to flush * @@ -359,7 +470,7 @@ public Builder setCacheSoftValues(boolean cacheSoftValues) { * @return this builder */ public Builder setCompactorPermits(int compactorPermits) { - ParseUtils.checkPositive(compactorPermits); + // ParseUtils.checkPositive(compactorPermits); this.compactPermits = compactorPermits; return this; } @@ -388,6 +499,18 @@ public Builder setParquetPageSize(long parquetPageSize) { return this; } + /** + * Set the size of parquet output buffer in bytes + * + * @param parquetOutputBufferMaxSize the size of parquet output buffer, bytes + * @return this builder + */ + public Builder setParquetOutputBufferMaxSize(int parquetOutputBufferMaxSize) { + ParseUtils.checkPositive(parquetOutputBufferMaxSize); + this.parquetOutputBufferMaxSize = parquetOutputBufferMaxSize; + return this; + } + /** * Set the parquet compression codec name * @@ -441,29 +564,19 @@ public Builder setParquetLz4BufferSize(int bufferSize) { * Parse properties to set the properties of StorageProperties * * @param properties the properties to be parsed - *

Supported keys: - *

    - *
  • close.flush: whether to flush on close - *
  • write.buffer.size: the size of write buffer, bytes - *
  • write.buffer.timeout: the timeout of write buffer to flush, iso-8601 - *
  • write.batch.size: the size of write batch, bytes - *
  • compact.permits: the number of flusher permits - *
  • cache.capacity: the capacity of cache, bytes - *
  • cache.timeout: the expiry timeout of cache, iso8601 duration - *
  • cache.value.soft: whether to enable soft values of cache - *
  • parquet.block.size: the size of parquet row group, bytes - *
  • parquet.page.size: the size of parquet page, bytes - *
  • parquet.compression: the parquet compression codec name - *
  • parquet.lz4.buffer.size: the parquet lz4 buffer size, bytes - *
  • zstd.level: the zstd level - *
  • zstd.workers: the zstd workers number - *
- * * @return this builder */ public Builder parse(Map properties) { ParseUtils.getOptionalBoolean(properties, FLUSH_ON_CLOSE).ifPresent(this::setFlushOnClose); ParseUtils.getOptionalLong(properties, WRITE_BUFFER_SIZE).ifPresent(this::setWriteBufferSize); + ParseUtils.getOptionalInteger(properties, WRITE_BUFFER_PERMITS) + .ifPresent(this::setWriteBufferPermits); + ParseUtils.getOptionalInteger(properties, WRITE_BUFFER_CHUNK_VALUES_MAX) + .ifPresent(this::setWriteBufferChunkValuesMax); + ParseUtils.getOptionalInteger(properties, WRITE_BUFFER_CHUNK_VALUES_MIN) + .ifPresent(this::setWriteBufferChunkValuesMin); + ParseUtils.getOptionalString(properties, WRITE_BUFFER_CHUNK_INDEX) + .ifPresent(this::setWriteBufferChunkIndex); ParseUtils.getOptionalDuration(properties, WRITE_BUFFER_TIMEOUT) .ifPresent(this::setWriteBufferTimeout); ParseUtils.getOptionalLong(properties, WRITE_BATCH_SIZE).ifPresent(this::setWriteBatchSize); @@ -476,6 +589,8 @@ public Builder parse(Map properties) { ParseUtils.getOptionalLong(properties, PARQUET_BLOCK_SIZE) .ifPresent(this::setParquetRowGroupSize); ParseUtils.getOptionalLong(properties, PARQUET_PAGE_SIZE).ifPresent(this::setParquetPageSize); + ParseUtils.getOptionalInteger(properties, PARQUET_OUTPUT_BUFFER_SIZE) + .ifPresent(this::setParquetOutputBufferMaxSize); ParseUtils.getOptionalString(properties, PARQUET_COMPRESSOR) .ifPresent(this::setParquetCompression); ParseUtils.getOptionalInteger(properties, ZSTD_LEVEL).ifPresent(this::setZstdLevel); @@ -494,6 +609,10 @@ public StorageProperties build() { return new StorageProperties( flushOnClose, writeBufferSize, + writeBufferPermits, + writeBufferChunkValuesMax, + writeBufferChunkValuesMin, + writeBufferChunkIndex, writeBufferTimeout, writeBatchSize, compactPermits, @@ -502,6 +621,7 @@ public StorageProperties build() { cacheSoftValues, parquetRowGroupSize, parquetPageSize, + parquetOutputBufferMaxSize, parquetCompression, zstdLevel, zstdWorkers, diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/arrow/ArrowFieldTypes.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/arrow/ArrowFieldTypes.java new file mode 100644 index 0000000000..65e864f559 --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/arrow/ArrowFieldTypes.java @@ -0,0 +1,46 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cn.edu.tsinghua.iginx.parquet.util.arrow; + +import cn.edu.tsinghua.iginx.thrift.DataType; +import java.util.Map; +import javax.annotation.Nullable; +import org.apache.arrow.util.Preconditions; +import org.apache.arrow.vector.types.pojo.DictionaryEncoding; +import org.apache.arrow.vector.types.pojo.FieldType; + +public class ArrowFieldTypes { + + public static FieldType of( + boolean nullable, + DataType dataType, + @Nullable DictionaryEncoding dictionary, + @Nullable Map metadata) { + Preconditions.checkNotNull(dataType, "dataType"); + + return new FieldType(nullable, ArrowTypes.of(dataType), dictionary, metadata); + } + + public static FieldType nonnull(DataType dataType, @Nullable Map metadata) { + return of(false, dataType, null, null); + } + + public static FieldType nonnull(DataType dataType) { + return nonnull(dataType, null); + } +} diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/arrow/ArrowFields.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/arrow/ArrowFields.java new file mode 100644 index 0000000000..e29b423848 --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/arrow/ArrowFields.java @@ -0,0 +1,135 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cn.edu.tsinghua.iginx.parquet.util.arrow; + +import cn.edu.tsinghua.iginx.engine.physical.storage.domain.ColumnKey; +import cn.edu.tsinghua.iginx.parquet.db.util.AreaSet; +import cn.edu.tsinghua.iginx.parquet.manager.utils.TagKVUtils; +import cn.edu.tsinghua.iginx.parquet.util.Constants; +import cn.edu.tsinghua.iginx.thrift.DataType; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import javax.annotation.Nullable; +import org.apache.arrow.util.Preconditions; +import org.apache.arrow.vector.types.pojo.DictionaryEncoding; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; + +public class ArrowFields { + + public static Field of( + boolean nullable, + ColumnKey columnKey, + DataType type, + @Nullable DictionaryEncoding dictionaryEncoding, + @Nullable List children) { + Preconditions.checkNotNull(columnKey, "columnKey"); + Preconditions.checkNotNull(type, "type"); + + Map tags = columnKey.getTags(); + Map metadata = tags.isEmpty() ? null : tags; + FieldType fieldType = ArrowFieldTypes.of(nullable, type, dictionaryEncoding, metadata); + return new Field(columnKey.getPath(), fieldType, children); + } + + public static Field of(boolean nullable, ColumnKey columnKey, DataType type) { + return of(nullable, columnKey, type, null, null); + } + + public static Field of(ColumnKey columnKey, DataType type) { + return of(true, columnKey, type); + } + + public static ColumnKey toColumnKey(Field field) { + Preconditions.checkNotNull(field, "field"); + Preconditions.checkArgument(!field.getType().isComplex()); + + return new ColumnKey(field.getName(), field.getMetadata()); + } + + public static Field KEY = of(false, new ColumnKey(Constants.KEY_FIELD_NAME), DataType.LONG); + + public static Set of(Map schema) { + return schema.entrySet().stream() + .map(entry -> of(TagKVUtils.splitFullName(entry.getKey()), entry.getValue())) + .collect(Collectors.toSet()); + } + + public static Map toIginxSchema(Set fields) { + return fields.stream() + .collect( + Collectors.toMap( + field -> TagKVUtils.toFullName(toColumnKey(field)), + field -> ArrowTypes.toIginxType(field.getType()))); + } + + public static AreaSet toInnerAreas(AreaSet areas) { + AreaSet innerAreas = new AreaSet<>(); + innerAreas.add(areas.getKeys()); + Set innerFields = + areas.getFields().stream().map(ArrowFields::toFullName).collect(Collectors.toSet()); + innerAreas.add(innerFields); + areas + .getSegments() + .forEach( + (field, rangeSet) -> { + String innerField = toFullName(field); + innerAreas.add(Collections.singleton(innerField), rangeSet); + }); + return innerAreas; + } + + public static String toFullName(Field field) { + return TagKVUtils.toFullName(toColumnKey(field)); + } + + public static AreaSet of(AreaSet areas, Map schema) { + AreaSet arrowAreas = new AreaSet<>(); + arrowAreas.add(areas.getKeys()); + Set fields = + areas.getFields().stream() + .map(name -> of(TagKVUtils.splitFullName(name), schema.get(name))) + .collect(Collectors.toSet()); + arrowAreas.add(fields); + areas + .getSegments() + .forEach( + (name, rangeSet) -> { + Field arrowField = of(TagKVUtils.splitFullName(name), schema.get(name)); + arrowAreas.add(Collections.singleton(arrowField), rangeSet); + }); + return arrowAreas; + } + + public static Field notNull(Field field) { + return of(false, toColumnKey(field), ArrowTypes.toIginxType(field.getType())); + } + + public static Field nullable(Field field) { + return of(true, toColumnKey(field), ArrowTypes.toIginxType(field.getType())); + } + + public static Set toInnerFields(List fields) { + return fields.stream() + .map(field -> TagKVUtils.toFullName(ArrowFields.toColumnKey(field))) + .collect(Collectors.toSet()); + } +} diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/arrow/ArrowTypes.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/arrow/ArrowTypes.java new file mode 100644 index 0000000000..b563fc4a1c --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/arrow/ArrowTypes.java @@ -0,0 +1,89 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cn.edu.tsinghua.iginx.parquet.util.arrow; + +import cn.edu.tsinghua.iginx.thrift.DataType; +import java.util.Objects; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.ArrowType; + +public class ArrowTypes { + public static ArrowType of(DataType dataType) { + return minorTypeOf(dataType).getType(); + } + + public static DataType toIginxType(ArrowType arrowType) { + Objects.requireNonNull(arrowType, "arrowType"); + switch (arrowType.getTypeID()) { + case Bool: + return DataType.BOOLEAN; + case Int: + return toIginxType((ArrowType.Int) arrowType); + case FloatingPoint: + return toIginxType((ArrowType.FloatingPoint) arrowType); + case Binary: + return DataType.BINARY; + default: + throw new IllegalArgumentException("Unsupported arrow type: " + arrowType); + } + } + + public static DataType toIginxType(ArrowType.Int arrowType) { + Objects.requireNonNull(arrowType, "arrowType"); + switch (arrowType.getBitWidth()) { + case 32: + return DataType.INTEGER; + case 64: + return DataType.LONG; + default: + throw new IllegalArgumentException("Unsupported arrow type: " + arrowType); + } + } + + public static DataType toIginxType(ArrowType.FloatingPoint arrowType) { + Objects.requireNonNull(arrowType, "arrowType"); + switch (arrowType.getPrecision()) { + case SINGLE: + return DataType.FLOAT; + case DOUBLE: + return DataType.DOUBLE; + default: + throw new IllegalArgumentException("Unsupported arrow type: " + arrowType); + } + } + + public static Types.MinorType minorTypeOf(DataType dataType) { + Objects.requireNonNull(dataType, "dataType"); + switch (dataType) { + case BOOLEAN: + return Types.MinorType.BIT; // 1-bit + case INTEGER: + return Types.MinorType.INT; // 32-bit + case LONG: + return Types.MinorType.BIGINT; // 64-bit + case FLOAT: + return Types.MinorType.FLOAT4; // 32-bit + case DOUBLE: + return Types.MinorType.FLOAT8; // 64-bit + case BINARY: + return Types.MinorType.VARBINARY; // variable length + default: + throw new IllegalArgumentException("Unsupported data type: " + dataType); + } + } +} diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/arrow/ArrowVectors.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/arrow/ArrowVectors.java new file mode 100644 index 0000000000..f50d9321fe --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/arrow/ArrowVectors.java @@ -0,0 +1,207 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cn.edu.tsinghua.iginx.parquet.util.arrow; + +import cn.edu.tsinghua.iginx.engine.physical.storage.domain.ColumnKey; +import cn.edu.tsinghua.iginx.thrift.DataType; +import java.util.function.IntPredicate; +import org.apache.arrow.algorithm.sort.DefaultVectorComparators; +import org.apache.arrow.algorithm.sort.IndexSorter; +import org.apache.arrow.algorithm.sort.StableVectorComparator; +import org.apache.arrow.algorithm.sort.VectorValueComparator; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.util.Preconditions; +import org.apache.arrow.vector.*; +import org.apache.arrow.vector.types.Types; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; +import org.apache.arrow.vector.util.VectorBatchAppender; + +public class ArrowVectors { + + public static ValueVector of( + boolean nullable, ColumnKey columnKey, DataType type, BufferAllocator allocator) { + Preconditions.checkNotNull(columnKey, "columnKey"); + Preconditions.checkNotNull(type, "type"); + Preconditions.checkNotNull(allocator, "allocator"); + + Field field = ArrowFields.of(nullable, columnKey, type); + + switch (type) { + case BOOLEAN: + return new BitVector(field, allocator); + case INTEGER: + return new IntVector(field, allocator); + case LONG: + return new BigIntVector(field, allocator); + case FLOAT: + return new Float4Vector(field, allocator); + case DOUBLE: + return new Float8Vector(field, allocator); + case BINARY: + return new VarBinaryVector(field, allocator); + default: + throw new IllegalArgumentException("Unsupported data type: " + type); + } + } + + public static ValueVector nullable( + ColumnKey columnKey, DataType type, BufferAllocator allocator) { + return of(true, columnKey, type, allocator); + } + + public static ValueVector nonnull(ColumnKey columnKey, DataType type, BufferAllocator allocator) { + return of(false, columnKey, type, allocator); + } + + public static BigIntVector key(BufferAllocator allocator) { + return (BigIntVector) nonnull(ColumnKey.KEY, DataType.LONG, allocator); + } + + public static V slice(V vector) { + return slice(vector, 0, vector.getValueCount()); + } + + public static V slice(V vector, BufferAllocator allocator) { + return slice(vector, 0, vector.getValueCount(), allocator); + } + + public static V slice(V vector, int start, int valueCount) { + return slice(vector, start, valueCount, vector.getAllocator()); + } + + @SuppressWarnings("unchecked") + public static V slice( + V vector, int start, int length, BufferAllocator allocator) { + if (length == 0) { + // splitAndTransfer 不能处理 length == 0 的情况 + return like(vector, allocator); + } + TransferPair transferPair = vector.getTransferPair(allocator); + transferPair.splitAndTransfer(start, length); + // TODO: splitAndTransferValidityBuffer() 不能正确地对 ValidityBuffer 进行跨 allocator 的 transfer + // 所以这里只好先 splitAndTransfer 然后再 transfer,个人猜测这是一个需要汇报给 Arrow 的 bug + try (V slice = (V) transferPair.getTo()) { + return transfer(slice, allocator); + } + } + + @SuppressWarnings("unchecked") + public static V transfer(V vector, BufferAllocator allocator) { + TransferPair transferPair = vector.getTransferPair(allocator); + transferPair.transfer(); + return (V) transferPair.getTo(); + } + + @SuppressWarnings("unchecked") + public static V like(V vector, BufferAllocator allocator) { + return (V) vector.getTransferPair(allocator).getTo(); + } + + public static boolean isSorted(ValueVector vector) { + VectorValueComparator comparator = + DefaultVectorComparators.createDefaultComparator(vector); + comparator.attachVector(vector); + for (int i = 1; i < vector.getValueCount(); i++) { + if (comparator.compare(i - 1, i) >= 0) { + return false; + } + } + return true; + } + + public static IntVector range(int start, int end, BufferAllocator allocator) { + IntVector indexes = nonnullIntVector("", allocator); + indexes.allocateNew(end - start); + indexes.setValueCount(end - start); + for (int i = start; i < end; i++) { + indexes.set(i - start, i); + } + return indexes; + } + + public static IntVector stableSortIndexes(ValueVector vector, BufferAllocator allocator) { + VectorValueComparator defaultComparator = + DefaultVectorComparators.createDefaultComparator(vector); + VectorValueComparator stableComparator = + new StableVectorComparator<>(defaultComparator); + + IntVector indexes = nonnullIntVector("", allocator); + indexes.setValueCount(vector.getValueCount()); + + new IndexSorter<>().sort(vector, indexes, stableComparator); + + return indexes; + } + + public static IntVector nonnullIntVector(String name, BufferAllocator allocator) { + return new IntVector(name, FieldType.notNullable(Types.MinorType.INT.getType()), allocator); + } + + public static void dedupSortedIndexes(ValueVector vector, IntVector indexes) { + VectorValueComparator defaultComparator = + DefaultVectorComparators.createDefaultComparator(vector); + defaultComparator.attachVector(vector); + + int count = indexes.getValueCount(); + int unique = 0; + for (int i = 1; i < count; i++) { + if (defaultComparator.compare(indexes.get(i - 1), indexes.get(i)) != 0) { + if (unique != i - 1) { + indexes.set(unique, indexes.get(i - 1)); + } + unique++; + } + } + if (count > 0) { + indexes.set(unique, indexes.get(count - 1)); + unique++; + } + indexes.setValueCount(unique); + } + + public static void append(ValueVector to, ValueVector from) { + if (to.getValueCount() == 0) { + to.allocateNew(); + } + VectorBatchAppender.batchAppend(to, from); + } + + public static void collect(IntVector indexes, Iterable values) { + int offset = indexes.getValueCount(); + for (Integer value : values) { + if (value == null) indexes.setNull(offset); + else indexes.setSafe(offset, value); + offset++; + } + indexes.setValueCount(offset); + } + + public static void filter(IntVector indexes, IntPredicate predicate) { + int count = indexes.getValueCount(); + int offset = 0; + for (int i = 0; i < count; i++) { + if (predicate.test(indexes.get(i))) { + indexes.set(offset, indexes.get(i)); + offset++; + } + } + indexes.setValueCount(offset); + } +} diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/InvalidFieldNameException.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/InvalidFieldNameException.java new file mode 100644 index 0000000000..7b7adf0b8b --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/InvalidFieldNameException.java @@ -0,0 +1,40 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.parquet.util.exception; + +public class InvalidFieldNameException extends SchemaException { + + private final String fieldName; + + private final String reason; + + public InvalidFieldNameException(String fieldName, String reason) { + super(String.format("invalid field name %s, because: ", fieldName, reason)); + this.fieldName = fieldName; + this.reason = reason; + } + + public String getFieldName() { + return fieldName; + } + + public String getReason() { + return reason; + } +} diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/IsClosedException.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/IsClosedException.java new file mode 100644 index 0000000000..0b494a64f0 --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/IsClosedException.java @@ -0,0 +1,26 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.parquet.util.exception; + +public class IsClosedException extends RuntimeException { + + public IsClosedException(String message) { + super(message); + } +} diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/NotIntegrityException.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/NotIntegrityException.java new file mode 100644 index 0000000000..e5599890fe --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/NotIntegrityException.java @@ -0,0 +1,33 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.parquet.util.exception; + +public class NotIntegrityException extends StorageRuntimeException { + public NotIntegrityException(String message) { + super(message); + } + + public NotIntegrityException(String message, Throwable cause) { + super(message, cause); + } + + public NotIntegrityException(Throwable cause) { + super(cause); + } +} diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/SchemaException.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/SchemaException.java new file mode 100644 index 0000000000..61edfc7c16 --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/SchemaException.java @@ -0,0 +1,26 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.parquet.util.exception; + +public class SchemaException extends StorageException { + + public SchemaException(String message) { + super(message); + } +} diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/StorageClosedException.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/StorageClosedException.java new file mode 100644 index 0000000000..6f2dd38608 --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/StorageClosedException.java @@ -0,0 +1,24 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cn.edu.tsinghua.iginx.parquet.util.exception; + +public class StorageClosedException extends StorageException { + public StorageClosedException(String message) { + super(message); + } +} diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/StorageException.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/StorageException.java new file mode 100644 index 0000000000..9670a96b99 --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/StorageException.java @@ -0,0 +1,36 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.parquet.util.exception; + +import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; + +public class StorageException extends PhysicalException { + + public StorageException(Throwable cause) { + super(cause); + } + + public StorageException(String message) { + super(message); + } + + public StorageException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/StorageRuntimeException.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/StorageRuntimeException.java new file mode 100644 index 0000000000..354e9b882c --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/StorageRuntimeException.java @@ -0,0 +1,35 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.parquet.util.exception; + +import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalRuntimeException; + +public class StorageRuntimeException extends PhysicalRuntimeException { + public StorageRuntimeException(String message) { + super(message); + } + + public StorageRuntimeException(String message, Throwable cause) { + super(message, cause); + } + + public StorageRuntimeException(Throwable cause) { + super(cause); + } +} diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/TimeoutException.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/TimeoutException.java new file mode 100644 index 0000000000..42dc9b4738 --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/TimeoutException.java @@ -0,0 +1,29 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.parquet.util.exception; + +public class TimeoutException extends StorageException { + public TimeoutException(String message) { + super(message); + } + + public TimeoutException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/TypeConflictedException.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/TypeConflictedException.java new file mode 100644 index 0000000000..9ea3f3fa8e --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/TypeConflictedException.java @@ -0,0 +1,44 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.parquet.util.exception; + +public class TypeConflictedException extends SchemaException { + private final String field; + private final String type; + private final String oldType; + + public TypeConflictedException(String field, String type, String oldType) { + super(String.format("can't insert %s value into %s column at %s", type, oldType, field)); + this.field = field; + this.type = type; + this.oldType = oldType; + } + + public String getField() { + return field; + } + + public String getType() { + return type; + } + + public String getOldType() { + return oldType; + } +} diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/UnsupportedFilterException.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/UnsupportedFilterException.java new file mode 100644 index 0000000000..ec066b7031 --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/UnsupportedFilterException.java @@ -0,0 +1,39 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.parquet.util.exception; + +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; + +public class UnsupportedFilterException extends StorageException { + private final Filter filter; + + public UnsupportedFilterException(Filter filter) { + super(String.format("unsupported filter %s", filter.toString())); + this.filter = filter; + } + + public UnsupportedFilterException(String message, Filter filter) { + super(message); + this.filter = filter; + } + + public Filter getFilter() { + return filter; + } +} diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/iterator/DedupIterator.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/iterator/DedupIterator.java new file mode 100644 index 0000000000..f3b22cc2ef --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/iterator/DedupIterator.java @@ -0,0 +1,57 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cn.edu.tsinghua.iginx.parquet.util.iterator; + +import com.google.common.collect.Iterators; +import com.google.common.collect.PeekingIterator; +import com.google.common.collect.UnmodifiableIterator; +import java.util.Iterator; +import java.util.function.Function; + +public class DedupIterator extends UnmodifiableIterator { + + private final PeekingIterator iterator; + private final Function keyExtractor; + + public DedupIterator(Iterator iterator, Function keyExtractor) { + this.iterator = Iterators.peekingIterator(iterator); + this.keyExtractor = keyExtractor; + } + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public T next() { + T next = iterator.next(); + Object key = keyExtractor.apply(next); + while (iterator.hasNext()) { + T peek = iterator.peek(); + Object peekKey = keyExtractor.apply(peek); + if (!key.equals(peekKey)) { + break; + } + next = peek; + key = peekKey; + iterator.next(); + } + return next; + } +} diff --git a/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/iterator/StableMergeIterator.java b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/iterator/StableMergeIterator.java new file mode 100644 index 0000000000..f2dbc240ab --- /dev/null +++ b/dataSource/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/iterator/StableMergeIterator.java @@ -0,0 +1,92 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cn.edu.tsinghua.iginx.parquet.util.iterator; + +import com.google.common.collect.Iterators; +import com.google.common.collect.PeekingIterator; +import com.google.common.collect.UnmodifiableIterator; +import java.util.Comparator; +import java.util.Iterator; +import java.util.PriorityQueue; + +public class StableMergeIterator extends UnmodifiableIterator { + private final Comparator> comparator; + private final PriorityQueue> queue; + private Entry activeEntry; + + public StableMergeIterator( + Iterable> iterators, Comparator itemComparator) { + + Comparator> heapComparator = Comparator.comparing(Entry::peek, itemComparator); + this.comparator = heapComparator.thenComparing(Entry::order); + this.queue = new PriorityQueue<>(comparator); + + int order = 0; + for (Iterator iterator : iterators) { + if (iterator.hasNext()) { + this.queue.add(new Entry<>(Iterators.peekingIterator(iterator), order)); + order++; + } + } + if (!this.queue.isEmpty()) { + this.activeEntry = this.queue.remove(); + } + } + + public boolean hasNext() { + return activeEntry != null; + } + + public T next() { + T next = activeEntry.peekingIterator.next(); + refreshActiveEntry(); + return next; + } + + private void refreshActiveEntry() { + if (!activeEntry.peekingIterator.hasNext()) { + activeEntry = this.queue.poll(); + return; + } + if (!queue.isEmpty()) { + Entry nextEntry = queue.peek(); + if (comparator.compare(activeEntry, nextEntry) > 0) { + queue.add(activeEntry); + activeEntry = queue.poll(); + } + } + } + + private static class Entry { + final PeekingIterator peekingIterator; + final int order; + + static T peek(Entry entry) { + return entry.peekingIterator.peek(); + } + + static int order(Entry entry) { + return entry.order; + } + + private Entry(PeekingIterator peekingIterator, int order) { + this.peekingIterator = peekingIterator; + this.order = order; + } + } +} diff --git a/dataSources/parquet/src/test/java/cn/edu/tsinghua/iginx/parquet/db/common/utils/SerializeUtilsTest.java b/dataSource/parquet/src/test/java/cn/edu/tsinghua/iginx/parquet/db/common/utils/SerializeUtilsTest.java similarity index 78% rename from dataSources/parquet/src/test/java/cn/edu/tsinghua/iginx/parquet/db/common/utils/SerializeUtilsTest.java rename to dataSource/parquet/src/test/java/cn/edu/tsinghua/iginx/parquet/db/common/utils/SerializeUtilsTest.java index b8fc070952..ed1824a888 100644 --- a/dataSources/parquet/src/test/java/cn/edu/tsinghua/iginx/parquet/db/common/utils/SerializeUtilsTest.java +++ b/dataSource/parquet/src/test/java/cn/edu/tsinghua/iginx/parquet/db/common/utils/SerializeUtilsTest.java @@ -1,17 +1,19 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.parquet.db.common.utils; diff --git a/dataSources/parquet/src/test/java/cn/edu/tsinghua/iginx/parquet/io/ParquetFormatIOTest.java b/dataSource/parquet/src/test/java/cn/edu/tsinghua/iginx/parquet/io/ParquetFormatIOTest.java similarity index 93% rename from dataSources/parquet/src/test/java/cn/edu/tsinghua/iginx/parquet/io/ParquetFormatIOTest.java rename to dataSource/parquet/src/test/java/cn/edu/tsinghua/iginx/parquet/io/ParquetFormatIOTest.java index 9cbeeae82a..89b3dfefeb 100644 --- a/dataSources/parquet/src/test/java/cn/edu/tsinghua/iginx/parquet/io/ParquetFormatIOTest.java +++ b/dataSource/parquet/src/test/java/cn/edu/tsinghua/iginx/parquet/io/ParquetFormatIOTest.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.parquet.io; import static org.junit.Assert.assertEquals; diff --git a/dataSources/parquet/src/test/java/cn/edu/tsinghua/iginx/parquet/manager/data/FilterRangeUtilsTest.java b/dataSource/parquet/src/test/java/cn/edu/tsinghua/iginx/parquet/manager/data/FilterRangeUtilsTest.java similarity index 64% rename from dataSources/parquet/src/test/java/cn/edu/tsinghua/iginx/parquet/manager/data/FilterRangeUtilsTest.java rename to dataSource/parquet/src/test/java/cn/edu/tsinghua/iginx/parquet/manager/data/FilterRangeUtilsTest.java index 5605222c38..1d4b53b33f 100644 --- a/dataSources/parquet/src/test/java/cn/edu/tsinghua/iginx/parquet/manager/data/FilterRangeUtilsTest.java +++ b/dataSource/parquet/src/test/java/cn/edu/tsinghua/iginx/parquet/manager/data/FilterRangeUtilsTest.java @@ -1,17 +1,19 @@ /* - * Copyright 2024 IGinX of Tsinghua University + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.parquet.manager.data; diff --git a/dataSource/pom.xml b/dataSource/pom.xml new file mode 100644 index 0000000000..d1d25c1b04 --- /dev/null +++ b/dataSource/pom.xml @@ -0,0 +1,113 @@ + + + + 4.0.0 + + cn.edu.tsinghua + iginx + ${revision} + + + iginx-driver + pom + IGinX Driver + + + filesystem + influxdb + iotdb12 + mongodb + parquet + redis + relational + + + + true + + + + + cn.edu.tsinghua + iginx-core + provided + + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + + false + + + driver + + dir + + false + + + driver/${project.artifactId} + + + + + + + + + single + + package + + + + + org.apache.maven.plugins + maven-antrun-plugin + 1.7 + + + copy-driver-into-core + + run + + package + + + + + + + + + + + + + + + diff --git a/dataSource/redis/pom.xml b/dataSource/redis/pom.xml new file mode 100644 index 0000000000..7af0f58773 --- /dev/null +++ b/dataSource/redis/pom.xml @@ -0,0 +1,59 @@ + + + + 4.0.0 + + cn.edu.tsinghua + iginx-driver + ${revision} + + + redis + IGinX Redis + + + + redis.clients + jedis + + + org.slf4j + slf4j-api + + + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + org.apache.maven.plugins + maven-antrun-plugin + + + + + diff --git a/dataSources/redis/src/main/java/cn/edu/tsinghua/iginx/redis/RedisStorage.java b/dataSource/redis/src/main/java/cn/edu/tsinghua/iginx/redis/RedisStorage.java similarity index 70% rename from dataSources/redis/src/main/java/cn/edu/tsinghua/iginx/redis/RedisStorage.java rename to dataSource/redis/src/main/java/cn/edu/tsinghua/iginx/redis/RedisStorage.java index 831a7b9c1c..fb21548a17 100644 --- a/dataSources/redis/src/main/java/cn/edu/tsinghua/iginx/redis/RedisStorage.java +++ b/dataSource/redis/src/main/java/cn/edu/tsinghua/iginx/redis/RedisStorage.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.redis; import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; @@ -18,12 +36,23 @@ import cn.edu.tsinghua.iginx.metadata.entity.KeyInterval; import cn.edu.tsinghua.iginx.metadata.entity.StorageEngineMeta; import cn.edu.tsinghua.iginx.redis.entity.RedisQueryRowStream; -import cn.edu.tsinghua.iginx.redis.tools.*; +import cn.edu.tsinghua.iginx.redis.tools.DataCoder; +import cn.edu.tsinghua.iginx.redis.tools.DataTransformer; +import cn.edu.tsinghua.iginx.redis.tools.DataViewWrapper; +import cn.edu.tsinghua.iginx.redis.tools.FilterUtils; +import cn.edu.tsinghua.iginx.redis.tools.TagKVUtils; import cn.edu.tsinghua.iginx.thrift.DataType; import cn.edu.tsinghua.iginx.thrift.StorageEngineType; import cn.edu.tsinghua.iginx.utils.Pair; import cn.edu.tsinghua.iginx.utils.StringUtils; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; +import java.util.Set; import java.util.function.Consumer; import java.util.stream.Collectors; import org.slf4j.Logger; @@ -60,11 +89,15 @@ public class RedisStorage implements IStorage { private static final String TIMEOUT = "timeout"; + private static final String USERNAME = "username"; + + private static final String PASSWORD = "password"; + private static final String DATA_DB = "data_db"; - private static final String DATA_PASSWORD = "dummy_db"; + private static final String DUMMY_DB = "dummy_db"; - private static final int DEFAULT_TIMEOUT = 5000; + private static final int DEFAULT_TIMEOUT = 10000; private static final int DEFAULT_DATA_DB = 1; @@ -85,11 +118,15 @@ public RedisStorage(StorageEngineMeta meta) throws StorageInitializationExceptio Map extraParams = meta.getExtraParams(); int timeout = Integer.parseInt(extraParams.getOrDefault(TIMEOUT, String.valueOf(DEFAULT_TIMEOUT))); - this.jedisPool = new JedisPool(new JedisPoolConfig(), meta.getIp(), meta.getPort(), timeout); + String username = extraParams.get(USERNAME); + String password = extraParams.get(PASSWORD); + this.jedisPool = + new JedisPool( + new JedisPoolConfig(), meta.getIp(), meta.getPort(), timeout, username, password); this.dataDb = Integer.parseInt(extraParams.getOrDefault(DATA_DB, String.valueOf(DEFAULT_DATA_DB))); this.dummyDb = - Integer.parseInt(extraParams.getOrDefault(DATA_PASSWORD, String.valueOf(DEFAULT_DUMMY_DB))); + Integer.parseInt(extraParams.getOrDefault(DUMMY_DB, String.valueOf(DEFAULT_DUMMY_DB))); this.dataPrefix = meta.getDataPrefix(); if (dataDb == dummyDb) { throw new StorageInitializationException("data db and dummy db should not be the same"); @@ -117,14 +154,8 @@ public boolean isSupportProjectWithSelect() { public TaskExecuteResult executeProjectWithSelect( Project project, Select select, DataArea dataArea) { String storageUnit = dataArea.getStorageUnit(); - List queryPaths; - try { - queryPaths = determinePathList(storageUnit, project.getPatterns(), project.getTagFilter()); - } catch (PhysicalException e) { - LOGGER.error("encounter error when delete path: ", e); - return new TaskExecuteResult( - new PhysicalTaskExecuteFailureException("execute delete path task in redis failure", e)); - } + List queryPaths = + determinePathList(storageUnit, project.getPatterns(), project.getTagFilter()); Filter filter = select.getFilter(); List> keyRanges = FilterUtils.keyRangesFrom(filter); @@ -174,9 +205,6 @@ public TaskExecuteResult executeProjectWithSelect( columns.add(column); } } - } catch (Exception e) { - return new TaskExecuteResult( - new PhysicalTaskExecuteFailureException("execute query path task in redis failure", e)); } return new TaskExecuteResult(new RedisQueryRowStream(columns, filter), null); @@ -231,10 +259,6 @@ public TaskExecuteResult executeProjectDummyWithSelect( LOGGER.warn("unknown key type, type={}", type); } } - } catch (Exception e) { - return new TaskExecuteResult( - new PhysicalTaskExecuteFailureException( - "execute query history task in redis failure", e)); } Filter filter = select.getFilter(); @@ -244,14 +268,8 @@ public TaskExecuteResult executeProjectDummyWithSelect( @Override public TaskExecuteResult executeProject(Project project, DataArea dataArea) { String storageUnit = dataArea.getStorageUnit(); - List queryPaths; - try { - queryPaths = determinePathList(storageUnit, project.getPatterns(), project.getTagFilter()); - } catch (PhysicalException e) { - LOGGER.error("encounter error when delete path: ", e); - return new TaskExecuteResult( - new PhysicalTaskExecuteFailureException("execute delete path task in redis failure", e)); - } + List queryPaths = + determinePathList(storageUnit, project.getPatterns(), project.getTagFilter()); List columns = new ArrayList<>(); try (Jedis jedis = getDataConnection()) { @@ -272,9 +290,6 @@ public TaskExecuteResult executeProject(Project project, DataArea dataArea) { columns.add(column); } } - } catch (Exception e) { - return new TaskExecuteResult( - new PhysicalTaskExecuteFailureException("execute query path task in redis failure", e)); } return new TaskExecuteResult(new RedisQueryRowStream(columns), null); } @@ -327,10 +342,6 @@ public TaskExecuteResult executeProjectDummy(Project project, DataArea dataArea) LOGGER.warn("unknown key type, type={}", type); } } - } catch (Exception e) { - return new TaskExecuteResult( - new PhysicalTaskExecuteFailureException( - "execute query history task in redis failure", e)); } return new TaskExecuteResult(new RedisQueryRowStream(columns), null); } @@ -338,68 +349,71 @@ public TaskExecuteResult executeProjectDummy(Project project, DataArea dataArea) @Override public TaskExecuteResult executeDelete(Delete delete, DataArea dataArea) { String storageUnit = dataArea.getStorageUnit(); - List deletedPaths; - try { - deletedPaths = determinePathList(storageUnit, delete.getPatterns(), delete.getTagFilter()); - } catch (PhysicalException e) { - LOGGER.warn("encounter error when delete path: ", e); - return new TaskExecuteResult( - new PhysicalTaskExecuteFailureException("execute delete path task in redis failure", e)); - } + List deletedKeyRanges = delete.getKeyRanges(); + boolean deleteAllColumn = + delete.getPatterns().stream().anyMatch(STAR::equals) && delete.getTagFilter() == null; + boolean deleteAllRange = delete.getKeyRanges() == null || deletedKeyRanges.isEmpty(); - if (deletedPaths.isEmpty()) { - return new TaskExecuteResult(null, null); - } - - if (delete.getKeyRanges() == null || delete.getKeyRanges().isEmpty()) { - // 没有传任何 time range, 删除全部数据 + if (deleteAllColumn && deleteAllRange) { try (Jedis jedis = getDataConnection()) { - int size = deletedPaths.size(); - String[] deletedPathArray = new String[size * 3]; - for (int i = 0; i < size; i++) { - String path = deletedPaths.get(i); - deletedPathArray[i] = String.format(KEY_FORMAT_HASH_VALUES, storageUnit, path); - deletedPathArray[i + size] = String.format(KEY_FORMAT_ZSET_KEYS, storageUnit, path); - deletedPathArray[i + 2 * size] = String.format(KEY_FORMAT_STRING_PATH, storageUnit, path); - } - jedis.del(deletedPathArray); - jedis.hdel(KEY_DATA_TYPE, deletedPaths.toArray(new String[0])); - } catch (Exception e) { - LOGGER.warn("encounter error when delete path: ", e); - return new TaskExecuteResult( - new PhysicalException("execute delete path in redis failure", e)); + jedis.flushDB(); } } else { - // 删除指定部分数据 - try (Jedis jedis = getDataConnection()) { - for (String path : deletedPaths) { - for (KeyRange keyRange : delete.getKeyRanges()) { - byte[] zSetKey = - DataCoder.encode(String.format(KEY_FORMAT_ZSET_KEYS, storageUnit, path)); - byte[] beginKeyRange = - concat(CLOSED_SIGN, DataCoder.encode(keyRange.getActualBeginKey())); - byte[] endKeyRange = concat(CLOSED_SIGN, DataCoder.encode(keyRange.getActualEndKey())); + List deletedPaths = + determinePathList(storageUnit, delete.getPatterns(), delete.getTagFilter()); + if (!deletedPaths.isEmpty()) { + if (deleteAllRange) { + deleteColumns(deletedPaths, storageUnit); + } else { + deleteColumnsRanges(deletedPaths, deletedKeyRanges, storageUnit); + } + } + } - List keys = jedis.zrangeByLex(zSetKey, beginKeyRange, endKeyRange); - if (!keys.isEmpty()) { - byte[] hashKey = - DataCoder.encode(String.format(KEY_FORMAT_HASH_VALUES, storageUnit, path)); - jedis.hdel(hashKey, keys.toArray(new byte[0][0])); - jedis.zremrangeByLex(zSetKey, beginKeyRange, endKeyRange); - } + return new TaskExecuteResult(null, null); + } + + private void deleteColumnsRanges( + List deletedPaths, List deletedKeyRanges, String storageUnit) { + // 删除指定部分数据 + try (Jedis jedis = getDataConnection()) { + for (String path : deletedPaths) { + for (KeyRange keyRange : deletedKeyRanges) { + byte[] zSetKey = DataCoder.encode(String.format(KEY_FORMAT_ZSET_KEYS, storageUnit, path)); + byte[] beginKeyRange = + concat(CLOSED_SIGN, DataCoder.encode(keyRange.getActualBeginKey())); + byte[] endKeyRange = concat(CLOSED_SIGN, DataCoder.encode(keyRange.getActualEndKey())); + + List keys = jedis.zrangeByLex(zSetKey, beginKeyRange, endKeyRange); + if (!keys.isEmpty()) { + byte[] hashKey = + DataCoder.encode(String.format(KEY_FORMAT_HASH_VALUES, storageUnit, path)); + jedis.hdel(hashKey, keys.toArray(new byte[0][0])); + jedis.zremrangeByLex(zSetKey, beginKeyRange, endKeyRange); } } - } catch (Exception e) { - LOGGER.warn("encounter error when delete path: ", e); - return new TaskExecuteResult( - new PhysicalException("execute delete data in redis failure", e)); } } - return new TaskExecuteResult(null, null); + } + + private void deleteColumns(List deletedPaths, String storageUnit) { + // 没有传任何 time range, 删除全部数据 + try (Jedis jedis = getDataConnection()) { + int size = deletedPaths.size(); + String[] deletedPathArray = new String[size * 3]; + for (int i = 0; i < size; i++) { + String path = deletedPaths.get(i); + deletedPathArray[i] = String.format(KEY_FORMAT_HASH_VALUES, storageUnit, path); + deletedPathArray[i + size] = String.format(KEY_FORMAT_ZSET_KEYS, storageUnit, path); + deletedPathArray[i + 2 * size] = String.format(KEY_FORMAT_STRING_PATH, storageUnit, path); + } + jedis.del(deletedPathArray); + jedis.hdel(KEY_DATA_TYPE, deletedPaths.toArray(new String[0])); + } } private List determinePathList( - String storageUnit, List patterns, TagFilter tagFilter) throws PhysicalException { + String storageUnit, List patterns, TagFilter tagFilter) { boolean hasTagFilter = tagFilter != null; List paths = new ArrayList<>(); try (Jedis jedis = getDataConnection()) { @@ -419,9 +433,6 @@ private List determinePathList( paths.add(path); }); } - } catch (Exception e) { - LOGGER.error("get du time series error, cause by: ", e); - return Collections.emptyList(); } if (!hasTagFilter) { return paths; @@ -459,58 +470,72 @@ public TaskExecuteResult executeInsert(Insert insert, DataArea dataArea) { jedis.hset(KEY_DATA_TYPE, path, type); jedis.set(String.format(KEY_FORMAT_STRING_PATH, storageUnit, path), EMTPY_STRING); - } catch (Exception e) { - return new TaskExecuteResult(new PhysicalException("execute insert in redis error", e)); } } return new TaskExecuteResult(null, null); } @Override - public List getColumns() { - List ret = new ArrayList<>(); - getIginxColumns(ret::add); - getDummyColumns(ret::add); - return ret; + public List getColumns(Set patterns, TagFilter tagFilter) { + try { + List ret = new ArrayList<>(); + getIginxColumns(ret::add, patterns, tagFilter); + getDummyColumns(ret::add, patterns); + return ret; + } catch (PhysicalException e) { + throw new IllegalStateException("get columns error", e); + } } - private void getIginxColumns(Consumer ret) { + private void getIginxColumns(Consumer ret, Set patterns, TagFilter tagFilter) + throws PhysicalException { + List patternList = new ArrayList<>(patterns); + if (patternList.isEmpty()) { + patternList.add("*"); + } + List allPaths = determinePathList("*", patternList, tagFilter); try (Jedis jedis = getDataConnection()) { - Map pathsAndTypes = jedis.hgetAll(KEY_DATA_TYPE); - pathsAndTypes.forEach( - (k, v) -> { - DataType type = DataTransformer.fromStringDataType(v); - Pair> pair = TagKVUtils.splitFullName(k); - ret.accept(new Column(pair.k, type, pair.v)); - }); + for (String path : allPaths) { + String typeStr = jedis.hget(KEY_DATA_TYPE, path); + if (typeStr == null) { + continue; + } + DataType type = DataTransformer.fromStringDataType(typeStr); + Pair> pair = TagKVUtils.splitFullName(path); + ret.accept(new Column(pair.k, type, pair.v)); + } } } - private void getDummyColumns(Consumer ret) { + private void getDummyColumns(Consumer ret, Set patterns) { + List patternList = new ArrayList<>(patterns); + if (patternList.isEmpty()) { + patternList.add("*"); + } try (Jedis jedis = getDummyConnection()) { - String pattern = STAR; - if (dataPrefix != null) { - pattern = dataPrefix + "." + pattern; - } - Set keys = jedis.keys(pattern); - for (String key : keys) { - String type = jedis.type(key); - switch (type) { - case "string": - case "list": - case "set": - case "zset": - ret.accept(new Column(key, DataType.BINARY, Collections.emptyMap(), true)); - break; - case "hash": - ret.accept(new Column(key + SUFFIX_KEY, DataType.BINARY, Collections.emptyMap(), true)); - ret.accept( - new Column(key + SUFFIX_VALUE, DataType.BINARY, Collections.emptyMap(), true)); - break; - case "none": - LOGGER.warn("key {} not exists", key); - default: - LOGGER.warn("unknown key type, type={}", type); + for (String pattern : patternList) { + String redisPattern = TagKVUtils.escapeRedisSpecialCharInPattern(pattern); + Set keys = jedis.keys(redisPattern); + for (String key : keys) { + String type = jedis.type(key); + switch (type) { + case "string": + case "list": + case "set": + case "zset": + ret.accept(new Column(key, DataType.BINARY, Collections.emptyMap(), true)); + break; + case "hash": + ret.accept( + new Column(key + SUFFIX_KEY, DataType.BINARY, Collections.emptyMap(), true)); + ret.accept( + new Column(key + SUFFIX_VALUE, DataType.BINARY, Collections.emptyMap(), true)); + break; + case "none": + LOGGER.warn("key {} not exists", key); + default: + LOGGER.warn("unknown key type, type={}", type); + } } } } @@ -530,12 +555,8 @@ public Pair getBoundaryOfStorage(String prefix) if (prefix != null) { columnsInterval = new ColumnsInterval(prefix); } else { - if (!paths.isEmpty()) { - columnsInterval = - new ColumnsInterval(paths.get(0), StringUtils.nextString(paths.get(paths.size() - 1))); - } else { - columnsInterval = new ColumnsInterval(null, null); - } + columnsInterval = + new ColumnsInterval(paths.get(0), StringUtils.nextString(paths.get(paths.size() - 1))); } KeyInterval keyInterval = KeyInterval.getDefaultKeyInterval(); return new Pair<>(columnsInterval, keyInterval); @@ -546,8 +567,6 @@ private List getKeysByPattern(String pattern) { try (Jedis jedis = jedisPool.getResource()) { Set keys = jedis.keys(pattern); paths.addAll(keys); - } catch (Exception e) { - LOGGER.error("get keys error, cause by: ", e); } return paths; } @@ -560,7 +579,7 @@ private static byte[] concat(byte prefix, byte[] arr) { } @Override - public void release() throws PhysicalException { + public void release() { jedisPool.close(); } } diff --git a/dataSources/redis/src/main/java/cn/edu/tsinghua/iginx/redis/entity/Column.java b/dataSource/redis/src/main/java/cn/edu/tsinghua/iginx/redis/entity/Column.java similarity index 62% rename from dataSources/redis/src/main/java/cn/edu/tsinghua/iginx/redis/entity/Column.java rename to dataSource/redis/src/main/java/cn/edu/tsinghua/iginx/redis/entity/Column.java index 4cb3770b9d..8edb0d2c7b 100644 --- a/dataSources/redis/src/main/java/cn/edu/tsinghua/iginx/redis/entity/Column.java +++ b/dataSource/redis/src/main/java/cn/edu/tsinghua/iginx/redis/entity/Column.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.redis.entity; import cn.edu.tsinghua.iginx.thrift.DataType; diff --git a/dataSources/redis/src/main/java/cn/edu/tsinghua/iginx/redis/entity/RedisQueryRowStream.java b/dataSource/redis/src/main/java/cn/edu/tsinghua/iginx/redis/entity/RedisQueryRowStream.java similarity index 79% rename from dataSources/redis/src/main/java/cn/edu/tsinghua/iginx/redis/entity/RedisQueryRowStream.java rename to dataSource/redis/src/main/java/cn/edu/tsinghua/iginx/redis/entity/RedisQueryRowStream.java index b7761a6045..7cb9789ce5 100644 --- a/dataSources/redis/src/main/java/cn/edu/tsinghua/iginx/redis/entity/RedisQueryRowStream.java +++ b/dataSource/redis/src/main/java/cn/edu/tsinghua/iginx/redis/entity/RedisQueryRowStream.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.redis.entity; import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; diff --git a/dataSources/redis/src/main/java/cn/edu/tsinghua/iginx/redis/tools/DataCoder.java b/dataSource/redis/src/main/java/cn/edu/tsinghua/iginx/redis/tools/DataCoder.java similarity index 60% rename from dataSources/redis/src/main/java/cn/edu/tsinghua/iginx/redis/tools/DataCoder.java rename to dataSource/redis/src/main/java/cn/edu/tsinghua/iginx/redis/tools/DataCoder.java index e964a11f40..323b20d8df 100644 --- a/dataSources/redis/src/main/java/cn/edu/tsinghua/iginx/redis/tools/DataCoder.java +++ b/dataSource/redis/src/main/java/cn/edu/tsinghua/iginx/redis/tools/DataCoder.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.redis.tools; public class DataCoder { diff --git a/dataSources/redis/src/main/java/cn/edu/tsinghua/iginx/redis/tools/DataTransformer.java b/dataSource/redis/src/main/java/cn/edu/tsinghua/iginx/redis/tools/DataTransformer.java similarity index 69% rename from dataSources/redis/src/main/java/cn/edu/tsinghua/iginx/redis/tools/DataTransformer.java rename to dataSource/redis/src/main/java/cn/edu/tsinghua/iginx/redis/tools/DataTransformer.java index 605fab83b8..37b3d96274 100644 --- a/dataSources/redis/src/main/java/cn/edu/tsinghua/iginx/redis/tools/DataTransformer.java +++ b/dataSource/redis/src/main/java/cn/edu/tsinghua/iginx/redis/tools/DataTransformer.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.redis.tools; import cn.edu.tsinghua.iginx.thrift.DataType; diff --git a/dataSources/redis/src/main/java/cn/edu/tsinghua/iginx/redis/tools/DataViewWrapper.java b/dataSource/redis/src/main/java/cn/edu/tsinghua/iginx/redis/tools/DataViewWrapper.java similarity index 79% rename from dataSources/redis/src/main/java/cn/edu/tsinghua/iginx/redis/tools/DataViewWrapper.java rename to dataSource/redis/src/main/java/cn/edu/tsinghua/iginx/redis/tools/DataViewWrapper.java index ed0993074c..30d26730b4 100644 --- a/dataSources/redis/src/main/java/cn/edu/tsinghua/iginx/redis/tools/DataViewWrapper.java +++ b/dataSource/redis/src/main/java/cn/edu/tsinghua/iginx/redis/tools/DataViewWrapper.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.redis.tools; import cn.edu.tsinghua.iginx.engine.shared.data.write.BitmapView; diff --git a/dataSources/redis/src/main/java/cn/edu/tsinghua/iginx/redis/tools/FilterUtils.java b/dataSource/redis/src/main/java/cn/edu/tsinghua/iginx/redis/tools/FilterUtils.java similarity index 88% rename from dataSources/redis/src/main/java/cn/edu/tsinghua/iginx/redis/tools/FilterUtils.java rename to dataSource/redis/src/main/java/cn/edu/tsinghua/iginx/redis/tools/FilterUtils.java index 5c815c44b5..3ad3a615ea 100644 --- a/dataSources/redis/src/main/java/cn/edu/tsinghua/iginx/redis/tools/FilterUtils.java +++ b/dataSource/redis/src/main/java/cn/edu/tsinghua/iginx/redis/tools/FilterUtils.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.redis.tools; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.AndFilter; diff --git a/dataSources/redis/src/main/java/cn/edu/tsinghua/iginx/redis/tools/TagKVUtils.java b/dataSource/redis/src/main/java/cn/edu/tsinghua/iginx/redis/tools/TagKVUtils.java similarity index 82% rename from dataSources/redis/src/main/java/cn/edu/tsinghua/iginx/redis/tools/TagKVUtils.java rename to dataSource/redis/src/main/java/cn/edu/tsinghua/iginx/redis/tools/TagKVUtils.java index c42ffba5d3..9bb2f52977 100644 --- a/dataSources/redis/src/main/java/cn/edu/tsinghua/iginx/redis/tools/TagKVUtils.java +++ b/dataSource/redis/src/main/java/cn/edu/tsinghua/iginx/redis/tools/TagKVUtils.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.redis.tools; @@ -67,7 +66,11 @@ public static Pair> splitFullName(String fullName) { public static String getPattern(String name) { String escaped = COLUMN_KEY_TRANSLATOR.getEscaper().escape(name); - return escaped.replaceAll("[?^{}\\[\\]\\\\]", "\\\\$0"); + return escapeRedisSpecialCharInPattern(escaped); + } + + public static String escapeRedisSpecialCharInPattern(String name) { + return name.replaceAll("[?^{}\\[\\]\\\\]", "\\\\$0"); } public static boolean match(Map tags, TagFilter tagFilter) { diff --git a/dataSources/relational/.gitignore b/dataSource/relational/.gitignore similarity index 100% rename from dataSources/relational/.gitignore rename to dataSource/relational/.gitignore diff --git a/dataSource/relational/pom.xml b/dataSource/relational/pom.xml new file mode 100644 index 0000000000..21d4a6b6ec --- /dev/null +++ b/dataSource/relational/pom.xml @@ -0,0 +1,71 @@ + + + + 4.0.0 + + + cn.edu.tsinghua + iginx-driver + ${revision} + + + relational + IGinX relational + + + + org.postgresql + postgresql + + + com.zaxxer + HikariCP + 2.4.7 + compile + + + org.slf4j + slf4j-api + + + + + mysql + mysql-connector-java + 8.0.26 + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + org.apache.maven.plugins + maven-antrun-plugin + + + + + diff --git a/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/RelationalStorage.java b/dataSource/relational/src/main/java/cn/edu/tsinghua/iginx/relational/RelationalStorage.java similarity index 85% rename from dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/RelationalStorage.java rename to dataSource/relational/src/main/java/cn/edu/tsinghua/iginx/relational/RelationalStorage.java index 484814477d..c3fac8dd48 100644 --- a/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/RelationalStorage.java +++ b/dataSource/relational/src/main/java/cn/edu/tsinghua/iginx/relational/RelationalStorage.java @@ -1,25 +1,34 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.relational; import static cn.edu.tsinghua.iginx.constant.GlobalConstant.SEPARATOR; -import static cn.edu.tsinghua.iginx.relational.tools.Constants.*; +import static cn.edu.tsinghua.iginx.relational.tools.Constants.ADD_COLUMN_STATEMENT; +import static cn.edu.tsinghua.iginx.relational.tools.Constants.BATCH_SIZE; +import static cn.edu.tsinghua.iginx.relational.tools.Constants.CREATE_DATABASE_STATEMENT; +import static cn.edu.tsinghua.iginx.relational.tools.Constants.DATABASE_PREFIX; +import static cn.edu.tsinghua.iginx.relational.tools.Constants.DROP_COLUMN_STATEMENT; +import static cn.edu.tsinghua.iginx.relational.tools.Constants.KEY_NAME; +import static cn.edu.tsinghua.iginx.relational.tools.Constants.PASSWORD; +import static cn.edu.tsinghua.iginx.relational.tools.Constants.QUERY_STATEMENT_WITHOUT_KEYNAME; +import static cn.edu.tsinghua.iginx.relational.tools.Constants.TAGKV_SEPARATOR; +import static cn.edu.tsinghua.iginx.relational.tools.Constants.USERNAME; +import static cn.edu.tsinghua.iginx.relational.tools.Constants.classMap; import static cn.edu.tsinghua.iginx.relational.tools.TagKVUtils.splitFullName; import static cn.edu.tsinghua.iginx.relational.tools.TagKVUtils.toFullName; @@ -42,7 +51,16 @@ import cn.edu.tsinghua.iginx.engine.shared.operator.Insert; import cn.edu.tsinghua.iginx.engine.shared.operator.Project; import cn.edu.tsinghua.iginx.engine.shared.operator.Select; -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.*; +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.AndFilter; +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.BoolFilter; +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.FilterType; +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.KeyFilter; +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.NotFilter; +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Op; +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.OrFilter; +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.PathFilter; +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.ValueFilter; import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; import cn.edu.tsinghua.iginx.metadata.entity.ColumnsInterval; import cn.edu.tsinghua.iginx.metadata.entity.KeyInterval; @@ -60,13 +78,26 @@ import cn.edu.tsinghua.iginx.utils.StringUtils; import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; +import com.zaxxer.hikari.pool.HikariPool; import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.sql.*; -import java.util.*; +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -128,7 +159,7 @@ private Connection getConnection(String databaseName) { HikariDataSource newDataSource = new HikariDataSource(config); connectionPoolMap.put(databaseName, newDataSource); return newDataSource.getConnection(); - } catch (SQLException e) { + } catch (SQLException | HikariPool.PoolInitializationException e) { LOGGER.error("Cannot get connection for database {}", databaseName, e); return null; } @@ -232,7 +263,8 @@ private boolean testConnection() { private List getDatabaseNames() throws SQLException { List databaseNames = new ArrayList<>(); Connection conn = getConnection(relationalMeta.getDefaultDatabaseName()); - ResultSet rs = conn.createStatement().executeQuery(relationalMeta.getDatabaseQuerySql()); + Statement statement = conn.createStatement(); + ResultSet rs = statement.executeQuery(relationalMeta.getDatabaseQuerySql()); while (rs.next()) { String databaseName = rs.getString("DATNAME"); if (relationalMeta.getSystemDatabaseName().contains(databaseName) @@ -242,6 +274,7 @@ private List getDatabaseNames() throws SQLException { databaseNames.add(databaseName); } rs.close(); + statement.close(); conn.close(); return databaseNames; } @@ -335,38 +368,108 @@ private boolean filterContainsType(List types, Filter filter) { } @Override - public List getColumns() throws RelationalTaskExecuteFailureException { + public List getColumns(Set patterns, TagFilter tagFilter) + throws RelationalTaskExecuteFailureException { List columns = new ArrayList<>(); Map extraParams = meta.getExtraParams(); try { + // dummy pattern list + List patternList = new ArrayList<>(); + if (patterns == null || patterns.size() == 0) { + patternList = new ArrayList<>(Collections.singletonList("*.*")); + } + String colPattern; + + // non-dummy for (String databaseName : getDatabaseNames()) { if ((extraParams.get("has_data") == null || extraParams.get("has_data").equals("false")) && !databaseName.startsWith(DATABASE_PREFIX)) { continue; } - - List tables = getTables(databaseName, "%"); - for (String tableName : tables) { - List columnFieldList = getColumns(databaseName, tableName, "%"); - for (ColumnField columnField : columnFieldList) { - String columnName = columnField.columnName; - String typeName = columnField.columnType; - if (columnName.equals(KEY_NAME)) { // key 列不显示 - continue; + boolean isDummy = + extraParams.get("has_data") != null + && extraParams.get("has_data").equalsIgnoreCase("true") + && !databaseName.startsWith(DATABASE_PREFIX); + if (isDummy) { + // find pattern that match .* to avoid creating databases after. + if (patterns == null || patterns.size() == 0) { + continue; + } + for (String p : patterns) { + // dummy path starts with . + if (Pattern.matches( + StringUtils.reformatPath(p.substring(0, p.indexOf("."))), databaseName)) { + patternList.add(p); } - Pair> nameAndTags = splitFullName(columnName); - if (databaseName.startsWith(DATABASE_PREFIX)) { + } + continue; + } + + Map tableAndColPattern = new HashMap<>(); + + if (patterns != null && patterns.size() != 0) { + tableAndColPattern = splitAndMergeQueryPatterns(databaseName, new ArrayList<>(patterns)); + } else { + for (String table : getTables(databaseName, "%")) { + tableAndColPattern.put(table, "%"); + } + } + + for (String tableName : tableAndColPattern.keySet()) { + colPattern = tableAndColPattern.get(tableName); + for (String colName : colPattern.split(", ")) { + List columnFieldList = getColumns(databaseName, tableName, colName); + for (ColumnField columnField : columnFieldList) { + String columnName = columnField.columnName; + String typeName = columnField.columnType; + if (columnName.equals(KEY_NAME)) { // key 列不显示 + continue; + } + Pair> nameAndTags = splitFullName(columnName); + columnName = tableName + SEPARATOR + nameAndTags.k; + if (tagFilter != null && !TagKVUtils.match(nameAndTags.v, tagFilter)) { + continue; + } columns.add( new Column( - tableName + SEPARATOR + nameAndTags.k, + columnName, relationalMeta.getDataTypeTransformer().fromEngineType(typeName), - nameAndTags.v)); - } else { + nameAndTags.v, + isDummy)); + } + } + } + } + + // dummy + Map> dummyRes = splitAndMergeHistoryQueryPatterns(patternList); + Map table2cols; + // seemingly there are 4 nested loops, but it's only the consequence of special data structure + // and reused methods. + // the loops would not affect complexity + for (String databaseName : dummyRes.keySet()) { + table2cols = dummyRes.get(databaseName); + for (String tableName : table2cols.keySet()) { + colPattern = table2cols.get(tableName); + for (String colName : colPattern.split(", ")) { + List columnFieldList = getColumns(databaseName, tableName, colName); + for (ColumnField columnField : columnFieldList) { + String columnName = columnField.columnName; + String typeName = columnField.columnType; + if (columnName.equals(KEY_NAME)) { // key 列不显示 + continue; + } + Pair> nameAndTags = splitFullName(columnName); + columnName = databaseName + SEPARATOR + tableName + SEPARATOR + nameAndTags.k; + if (tagFilter != null && !TagKVUtils.match(nameAndTags.v, tagFilter)) { + continue; + } columns.add( new Column( - databaseName + SEPARATOR + tableName + SEPARATOR + nameAndTags.k, + columnName, relationalMeta.getDataTypeTransformer().fromEngineType(typeName), - nameAndTags.v)); + nameAndTags.v, + true)); } } } @@ -471,13 +574,14 @@ else if (!tableNameToColumnNames.isEmpty()) { // 将columnNames中的列名加上tableName前缀 fullColumnNames.replaceAll( - s -> RelationSchema.getQuotFullName(tableName, s, relationalMeta.getQuote())); + s -> RelationSchema.getQuoteFullName(tableName, s, relationalMeta.getQuote())); fullColumnNamesList.add(fullColumnNames); } StringBuilder fullColumnNames = new StringBuilder(); fullColumnNames.append( - RelationSchema.getQuotFullName(tableNames.get(0), KEY_NAME, relationalMeta.getQuote())); + RelationSchema.getQuoteFullName( + tableNames.get(0), KEY_NAME, relationalMeta.getQuote())); for (List columnNames : fullColumnNamesList) { for (String columnName : columnNames) { fullColumnNames.append(", ").append(columnName); @@ -505,7 +609,7 @@ else if (!tableNameToColumnNames.isEmpty()) { String fullColumnNamesStr = fullColumnNames.toString(); String filterStr = filterTransformer.toString(filter); String orderByKey = - RelationSchema.getQuotFullName(tableNames.get(0), KEY_NAME, relationalMeta.getQuote()); + RelationSchema.getQuoteFullName(tableNames.get(0), KEY_NAME, relationalMeta.getQuote()); if (!relationalMeta.isSupportFullJoin()) { // 如果不支持full join,需要为left join + union模拟的full join表起别名,同时select、where、order by的部分都要调整 fullColumnNamesStr = fullColumnNamesStr.replaceAll("`\\.`", "."); @@ -570,11 +674,11 @@ private String getFullJoinTables(List tableNames, List> ful for (int j = 0; j < i; j++) { fullTableName .append( - RelationSchema.getQuotFullName( + RelationSchema.getQuoteFullName( tableNames.get(i), KEY_NAME, relationalMeta.getQuote())) .append(" = ") .append( - RelationSchema.getQuotFullName( + RelationSchema.getQuoteFullName( tableNames.get(j), KEY_NAME, relationalMeta.getQuote())); if (j != i - 1) { @@ -630,10 +734,43 @@ private String getFullJoinTables(List tableNames, List> ful return fullTableName.toString(); } + /** + * 通过columnNames构建concat语句,由于concat的参数个数有限,所以需要分批次嵌套concat + * 由于pg、mysql一张表最大都不能超过2000列,因此嵌套2层,可达到最大10000列,已满足需求。 + * + * @param columnNames 列名列表 + * @return concat语句 + */ + private String buildConcat(List columnNames) { + int n = columnNames.size(); + assert n > 0; + List> concatList = new ArrayList<>(); + for (int i = 0; i < n / 100 + 1; i++) { + List subList = columnNames.subList(i * 100, Math.min((i + 1) * 100, n)); + if (subList.size() == 0) { + continue; + } + concatList.add(subList); + } + + if (concatList.size() == 1) { + return String.format(" CONCAT(%s) ", String.join(", ", concatList.get(0))); + } + + StringBuilder concat = new StringBuilder(); + concat.append(" CONCAT("); + for (int i = 0; i < concatList.size(); i++) { + concat.append(String.format(" CONCAT(%s) ", String.join(", ", concatList.get(i)))); + if (i != concatList.size() - 1) { + concat.append(", "); + } + } + concat.append(") "); + return concat.toString(); + } + private String getDummyFullJoinTables( - List tableNames, - Map allColumnNameForTable, - List> fullColumnList) { + List tableNames, Map> allColumnNameForTable) { StringBuilder fullTableName = new StringBuilder(); if (relationalMeta.isSupportFullJoin()) { // table之间用FULL OUTER JOIN ON concat(table1所有列) = concat(table2所有列) @@ -646,14 +783,11 @@ private String getDummyFullJoinTables( .append(getQuotName(tableNames.get(i))) .append(" ON "); for (int j = 0; j < i; j++) { - fullTableName - .append("CONCAT(") - .append(allColumnNameForTable.get(tableNames.get(i))) - .append(")") - .append(" = ") - .append("CONCAT(") - .append(allColumnNameForTable.get(tableNames.get(j))) - .append(")"); + fullTableName.append( + String.format( + "%s = %s", + buildConcat(allColumnNameForTable.get(tableNames.get(i))), + buildConcat(allColumnNameForTable.get(tableNames.get(j))))); if (j != i - 1) { fullTableName.append(" AND "); } @@ -662,14 +796,13 @@ private String getDummyFullJoinTables( } } else { // 不支持全连接,就要用Left Join+Union来模拟全连接 - StringBuilder allColumns = new StringBuilder(); - fullColumnList.forEach( - columnList -> - columnList.forEach( - column -> - allColumns.append( - String.format("%s AS %s, ", column, column.replaceAll("`\\.`", "."))))); - allColumns.delete(allColumns.length() - 2, allColumns.length()); + char quote = relationalMeta.getQuote(); + String allColumns = + tableNames.stream() + .map(allColumnNameForTable::get) + .flatMap(List::stream) + .map(s -> s + " AS " + s.replaceAll(quote + "\\." + quote, ".")) + .collect(Collectors.joining(", ")); fullTableName.append("("); for (int i = 0; i < tableNames.size(); i++) { @@ -684,10 +817,10 @@ private String getDummyFullJoinTables( if (i != j) { fullTableName.append( String.format( - " LEFT JOIN %s ON CONCAT(%s) = CONCAT(%s)", + " LEFT JOIN %s ON %s = %s", getQuotName(tableNames.get(j)), - allColumnNameForTable.get(tableNames.get(i)), - allColumnNameForTable.get(tableNames.get(j)))); + buildConcat(allColumnNameForTable.get(tableNames.get(i))), + buildConcat(allColumnNameForTable.get(tableNames.get(j))))); } } if (i != tableNames.size() - 1) { @@ -878,21 +1011,28 @@ private Filter cutFilterDatabaseNameForDummy(Filter filter, String databaseName) return filter; } - private Map getAllColumnNameForTable( + private Map> getAllColumnNameForTable( String databaseName, Map tableNameToColumnNames) throws SQLException { - Map allColumnNameForTable = new HashMap<>(); + Map> allColumnNameForTable = new HashMap<>(); for (Map.Entry entry : tableNameToColumnNames.entrySet()) { String tableName = entry.getKey(); - String columnNames = ""; List columnFieldList = getColumns(databaseName, tableName, "%"); for (ColumnField columnField : columnFieldList) { String columnName = columnField.columnName; - columnNames = - RelationSchema.getQuotFullName(tableName, columnName, relationalMeta.getQuote()); + if (allColumnNameForTable.containsKey(tableName)) { - columnNames = allColumnNameForTable.get(tableName) + ", " + columnNames; + allColumnNameForTable + .get(tableName) + .add( + RelationSchema.getQuoteFullName( + tableName, columnName, relationalMeta.getQuote())); + continue; } + + List columnNames = new ArrayList<>(); + columnNames.add( + RelationSchema.getQuoteFullName(tableName, columnName, relationalMeta.getQuote())); allColumnNameForTable.put(tableName, columnNames); } } @@ -927,29 +1067,34 @@ private TaskExecuteResult executeProjectDummyWithFilter(Project project, Filter } connList.add(conn); + // 这里获取所有table的所有列名,用于concat时生成key列。 + Map> allColumnNameForTable = + getAllColumnNameForTable(databaseName, tableNameToColumnNames); + + // 如果table没有带通配符,那直接简单构建起查询语句即可 if (!filter.toString().contains("*") && !(tableNameToColumnNames.size() > 1 && filterContainsType(Arrays.asList(FilterType.Value, FilterType.Path), filter))) { - Map allColumnNameForTable = - getAllColumnNameForTable(databaseName, tableNameToColumnNames); + for (Map.Entry entry : splitEntry.getValue().entrySet()) { String tableName = entry.getKey(); String fullQuotColumnNames = getQuotColumnNames(entry.getValue()); List fullPathList = Arrays.asList(entry.getValue().split(", ")); fullPathList.replaceAll( - s -> RelationSchema.getQuotFullName(tableName, s, relationalMeta.getQuote())); + s -> RelationSchema.getQuoteFullName(tableName, s, relationalMeta.getQuote())); String filterStr = filterTransformer.toString( dummyFilterSetTrueByColumnNames( cutFilterDatabaseNameForDummy(filter.copy(), databaseName), fullPathList)); + String concatKey = buildConcat(fullPathList); statement = String.format( relationalMeta.getConcatQueryStatement(), - allColumnNameForTable.get(tableName), + concatKey, fullQuotColumnNames, getQuotName(tableName), filterStr.isEmpty() ? "" : "WHERE " + filterStr, - "Concat(" + allColumnNameForTable.get(tableName) + ")"); + concatKey); try { stmt = conn.createStatement(); @@ -967,7 +1112,7 @@ && filterContainsType(Arrays.asList(FilterType.Value, FilterType.Path), filter)) else if (!tableNameToColumnNames.isEmpty()) { List tableNames = new ArrayList<>(); List> fullColumnNamesList = new ArrayList<>(); - List> fullQuotColumnNamesList = new ArrayList<>(); + List> fullQuoteColumnNamesList = new ArrayList<>(); // 将columnNames中的列名加上tableName前缀,带JOIN的查询语句中需要用到 for (Map.Entry entry : tableNameToColumnNames.entrySet()) { @@ -980,40 +1125,16 @@ else if (!tableNameToColumnNames.isEmpty()) { List fullColumnNames = new ArrayList<>(Arrays.asList(entry.getValue().split(", "))); fullColumnNames.replaceAll( - s -> RelationSchema.getQuotFullName(tableName, s, relationalMeta.getQuote())); - fullQuotColumnNamesList.add(fullColumnNames); + s -> RelationSchema.getQuoteFullName(tableName, s, relationalMeta.getQuote())); + fullQuoteColumnNamesList.add(fullColumnNames); } - StringBuilder fullColumnNames = new StringBuilder(); - for (List columnNames : fullQuotColumnNamesList) { - for (String columnName : columnNames) { - fullColumnNames.append(columnName).append(", "); - } - } - fullColumnNames.delete(fullColumnNames.length() - 2, fullColumnNames.length()); - - // 这里获取所有table的所有列名,用于concat时生成key列。 - Map allColumnNameForTable = - getAllColumnNameForTable(databaseName, tableNameToColumnNames); - StringBuilder allColumnNames = new StringBuilder(); - for (String tableName : tableNames) { - allColumnNames.append(allColumnNameForTable.get(tableName)).append(", "); - } - allColumnNames.delete(allColumnNames.length() - 2, allColumnNames.length()); + String fullTableName = getDummyFullJoinTables(tableNames, allColumnNameForTable); - String fullTableName = - getDummyFullJoinTables(tableNames, allColumnNameForTable, fullColumnNamesList); - - Filter copyFilter = cutFilterDatabaseNameForDummy(filter.copy(), databaseName); - - copyFilter = + Filter copyFilter = dummyFilterSetTrueByColumnNames( - copyFilter, - Arrays.asList( - fullColumnNames - .toString() - .replace(String.valueOf(relationalMeta.getQuote()), "") - .split(", "))); + cutFilterDatabaseNameForDummy(filter.copy(), databaseName), + fullColumnNamesList.stream().flatMap(List::stream).collect(Collectors.toList())); // 对通配符做处理,将通配符替换成对应的列名 if (filterTransformer.toString(copyFilter).contains("*")) { @@ -1030,12 +1151,18 @@ else if (!tableNameToColumnNames.isEmpty()) { copyFilter = LogicalFilterUtils.mergeTrue(copyFilter); } - String fullColumnNamesStr = fullColumnNames.toString(); String filterStr = filterTransformer.toString(copyFilter); - String orderByKey = String.format("Concat(%s)", allColumnNames); + String orderByKey = + buildConcat( + fullQuoteColumnNamesList.stream() + .flatMap(List::stream) + .collect(Collectors.toList())); if (!relationalMeta.isSupportFullJoin()) { // 如果不支持full join,需要为left join + union模拟的full join表起别名,同时select、where、order by的部分都要调整 - fullColumnNamesStr = fullColumnNamesStr.replaceAll("`\\.`", "."); + char quote = relationalMeta.getQuote(); + fullQuoteColumnNamesList.forEach( + columnNames -> + columnNames.replaceAll(s -> s.replaceAll(quote + "\\." + quote, "."))); filterStr = filterStr.replaceAll("`\\.`", "."); filterStr = filterStr.replace( @@ -1046,8 +1173,13 @@ else if (!tableNameToColumnNames.isEmpty()) { statement = String.format( relationalMeta.getConcatQueryStatement(), - allColumnNames, - fullColumnNamesStr, + buildConcat( + fullQuoteColumnNamesList.stream() + .flatMap(List::stream) + .collect(Collectors.toList())), + fullQuoteColumnNamesList.stream() + .flatMap(List::stream) + .collect(Collectors.joining(", ")), fullTableName, filterStr.isEmpty() ? "" : "WHERE " + filterStr, orderByKey); @@ -1407,6 +1539,7 @@ private String reformatForJDBC(String path) { private Map> splitAndMergeHistoryQueryPatterns(List patterns) throws SQLException { // > + List databases = getDatabaseNames(); Map> splitResults = new HashMap<>(); String databaseName; String tableName; @@ -1470,6 +1603,9 @@ private Map> splitAndMergeHistoryQueryPatterns(List< } } } else { + if (!databases.contains(databaseName)) { + continue; + } List columnFieldList = getColumns(databaseName, tableName, columnNames); Map tableNameToColumnNames = new HashMap<>(); for (ColumnField columnField : columnFieldList) { @@ -1489,8 +1625,7 @@ private Map> splitAndMergeHistoryQueryPatterns(List< for (Map.Entry entry : tableNameToColumnNames.entrySet()) { String oldColumnNames = oldTableNameToColumnNames.get(entry.getKey()); if (oldColumnNames != null) { - List oldColumnNameList = - Arrays.asList((oldColumnNames + ", " + entry.getValue()).split(", ")); + String[] oldColumnNameList = (oldColumnNames + ", " + entry.getValue()).split(", "); // 对list去重 List newColumnNameList = new ArrayList<>(); for (String columnName : oldColumnNameList) { @@ -1833,7 +1968,7 @@ private void executeBatchInsert( private List> determineDeletedPaths( List paths, TagFilter tagFilter) { try { - List columns = getColumns(); + List columns = getColumns(null, null); List> deletedPaths = new ArrayList<>(); for (Column column : columns) { diff --git a/dataSource/relational/src/main/java/cn/edu/tsinghua/iginx/relational/datatype/transformer/IDataTypeTransformer.java b/dataSource/relational/src/main/java/cn/edu/tsinghua/iginx/relational/datatype/transformer/IDataTypeTransformer.java new file mode 100644 index 0000000000..8a31b66e8e --- /dev/null +++ b/dataSource/relational/src/main/java/cn/edu/tsinghua/iginx/relational/datatype/transformer/IDataTypeTransformer.java @@ -0,0 +1,28 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.relational.datatype.transformer; + +import cn.edu.tsinghua.iginx.thrift.DataType; + +public interface IDataTypeTransformer { + + public DataType fromEngineType(String dataType); + + public String toEngineType(DataType dataType); +} diff --git a/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/datatype/transformer/JDBCDataTypeTransformer.java b/dataSource/relational/src/main/java/cn/edu/tsinghua/iginx/relational/datatype/transformer/JDBCDataTypeTransformer.java similarity index 64% rename from dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/datatype/transformer/JDBCDataTypeTransformer.java rename to dataSource/relational/src/main/java/cn/edu/tsinghua/iginx/relational/datatype/transformer/JDBCDataTypeTransformer.java index be6b23cc35..8aa6951c0c 100644 --- a/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/datatype/transformer/JDBCDataTypeTransformer.java +++ b/dataSource/relational/src/main/java/cn/edu/tsinghua/iginx/relational/datatype/transformer/JDBCDataTypeTransformer.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.relational.datatype.transformer; import cn.edu.tsinghua.iginx.thrift.DataType; diff --git a/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/datatype/transformer/PostgreSQLDataTypeTransformer.java b/dataSource/relational/src/main/java/cn/edu/tsinghua/iginx/relational/datatype/transformer/PostgreSQLDataTypeTransformer.java similarity index 67% rename from dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/datatype/transformer/PostgreSQLDataTypeTransformer.java rename to dataSource/relational/src/main/java/cn/edu/tsinghua/iginx/relational/datatype/transformer/PostgreSQLDataTypeTransformer.java index 33b512390a..dde7c4e4e0 100644 --- a/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/datatype/transformer/PostgreSQLDataTypeTransformer.java +++ b/dataSource/relational/src/main/java/cn/edu/tsinghua/iginx/relational/datatype/transformer/PostgreSQLDataTypeTransformer.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.relational.datatype.transformer; import static cn.edu.tsinghua.iginx.thrift.DataType.*; diff --git a/dataSource/relational/src/main/java/cn/edu/tsinghua/iginx/relational/exception/RelationalException.java b/dataSource/relational/src/main/java/cn/edu/tsinghua/iginx/relational/exception/RelationalException.java new file mode 100644 index 0000000000..92a0f3ea93 --- /dev/null +++ b/dataSource/relational/src/main/java/cn/edu/tsinghua/iginx/relational/exception/RelationalException.java @@ -0,0 +1,36 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.relational.exception; + +import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; + +public class RelationalException extends PhysicalException { + + public RelationalException(Throwable cause) { + super(cause); + } + + public RelationalException(String message) { + super(message); + } + + public RelationalException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/dataSource/relational/src/main/java/cn/edu/tsinghua/iginx/relational/exception/RelationalTaskExecuteFailureException.java b/dataSource/relational/src/main/java/cn/edu/tsinghua/iginx/relational/exception/RelationalTaskExecuteFailureException.java new file mode 100644 index 0000000000..d6bb3c297c --- /dev/null +++ b/dataSource/relational/src/main/java/cn/edu/tsinghua/iginx/relational/exception/RelationalTaskExecuteFailureException.java @@ -0,0 +1,32 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.relational.exception; + +import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalTaskExecuteFailureException; + +public class RelationalTaskExecuteFailureException extends PhysicalTaskExecuteFailureException { + + public RelationalTaskExecuteFailureException(String message) { + super(message); + } + + public RelationalTaskExecuteFailureException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/meta/AbstractRelationalMeta.java b/dataSource/relational/src/main/java/cn/edu/tsinghua/iginx/relational/meta/AbstractRelationalMeta.java similarity index 81% rename from dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/meta/AbstractRelationalMeta.java rename to dataSource/relational/src/main/java/cn/edu/tsinghua/iginx/relational/meta/AbstractRelationalMeta.java index 8225037585..5bedbf4dc6 100644 --- a/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/meta/AbstractRelationalMeta.java +++ b/dataSource/relational/src/main/java/cn/edu/tsinghua/iginx/relational/meta/AbstractRelationalMeta.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.relational.meta; import static cn.edu.tsinghua.iginx.relational.tools.Constants.KEY_NAME; @@ -98,11 +116,7 @@ public String getQueryStatement() { * @return 通过concat生成key的query的SQL语句 */ public String getConcatQueryStatement() { - return "SELECT concat(%s) AS " - + getQuote() - + KEY_NAME - + getQuote() - + ", %s FROM %s %s ORDER BY %s"; + return "SELECT %s AS " + getQuote() + KEY_NAME + getQuote() + ", %s FROM %s %s ORDER BY %s"; } public String getCreateTableStatement() { diff --git a/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/meta/JDBCMeta.java b/dataSource/relational/src/main/java/cn/edu/tsinghua/iginx/relational/meta/JDBCMeta.java similarity index 83% rename from dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/meta/JDBCMeta.java rename to dataSource/relational/src/main/java/cn/edu/tsinghua/iginx/relational/meta/JDBCMeta.java index 66204a301c..5b752c80a0 100644 --- a/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/meta/JDBCMeta.java +++ b/dataSource/relational/src/main/java/cn/edu/tsinghua/iginx/relational/meta/JDBCMeta.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.relational.meta; import cn.edu.tsinghua.iginx.metadata.entity.StorageEngineMeta; diff --git a/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/meta/PostgreSQLMeta.java b/dataSource/relational/src/main/java/cn/edu/tsinghua/iginx/relational/meta/PostgreSQLMeta.java similarity index 77% rename from dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/meta/PostgreSQLMeta.java rename to dataSource/relational/src/main/java/cn/edu/tsinghua/iginx/relational/meta/PostgreSQLMeta.java index 108001d1cb..b6b63e28f9 100644 --- a/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/meta/PostgreSQLMeta.java +++ b/dataSource/relational/src/main/java/cn/edu/tsinghua/iginx/relational/meta/PostgreSQLMeta.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.relational.meta; diff --git a/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/query/entity/RelationQueryRowStream.java b/dataSource/relational/src/main/java/cn/edu/tsinghua/iginx/relational/query/entity/RelationQueryRowStream.java similarity index 94% rename from dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/query/entity/RelationQueryRowStream.java rename to dataSource/relational/src/main/java/cn/edu/tsinghua/iginx/relational/query/entity/RelationQueryRowStream.java index b7d7a3a0f5..bd9345551e 100644 --- a/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/query/entity/RelationQueryRowStream.java +++ b/dataSource/relational/src/main/java/cn/edu/tsinghua/iginx/relational/query/entity/RelationQueryRowStream.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.relational.query.entity; import static cn.edu.tsinghua.iginx.constant.GlobalConstant.SEPARATOR; diff --git a/dataSource/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/ColumnField.java b/dataSource/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/ColumnField.java new file mode 100644 index 0000000000..fae79399fe --- /dev/null +++ b/dataSource/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/ColumnField.java @@ -0,0 +1,31 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.relational.tools; + +public class ColumnField { + public String tableName; + public String columnName; + public String columnType; + + public ColumnField(String tableName, String columnName, String columnType) { + this.tableName = tableName; + this.columnName = columnName; + this.columnType = columnType; + } +} diff --git a/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/Constants.java b/dataSource/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/Constants.java similarity index 58% rename from dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/Constants.java rename to dataSource/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/Constants.java index e7e7cca55f..4a154f3743 100644 --- a/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/Constants.java +++ b/dataSource/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/Constants.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.relational.tools; import java.util.HashMap; diff --git a/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/FilterTransformer.java b/dataSource/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/FilterTransformer.java similarity index 79% rename from dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/FilterTransformer.java rename to dataSource/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/FilterTransformer.java index 19a3fc800f..d1e5c95396 100644 --- a/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/FilterTransformer.java +++ b/dataSource/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/FilterTransformer.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.relational.tools; @@ -90,7 +89,7 @@ private String toString(KeyFilter filter) { private String toString(ValueFilter filter) { RelationSchema schema = new RelationSchema(filter.getPath(), relationalMeta.getQuote()); - String path = schema.getQuotFullName(); + String path = schema.getQuoteFullName(); String op = isLikeOp(filter.getOp()) @@ -127,8 +126,8 @@ private String toString(OrFilter filter) { private String toString(PathFilter filter) { RelationSchema schemaA = new RelationSchema(filter.getPathA(), relationalMeta.getQuote()); RelationSchema schemaB = new RelationSchema(filter.getPathB(), relationalMeta.getQuote()); - String pathA = schemaA.getQuotFullName(); - String pathB = schemaB.getQuotFullName(); + String pathA = schemaA.getQuoteFullName(); + String pathB = schemaB.getQuoteFullName(); String op = Op.op2StrWithoutAndOr(filter.getOp()) diff --git a/dataSource/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/HashUtils.java b/dataSource/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/HashUtils.java new file mode 100644 index 0000000000..4c549f5ee4 --- /dev/null +++ b/dataSource/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/HashUtils.java @@ -0,0 +1,35 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.relational.tools; + +public class HashUtils { + + public static long toHash(String s) { + char c[] = s.toCharArray(); + long hv = 0; + long base = 131; + for (int i = 0; i < c.length; i++) { + hv = hv * base + (long) c[i]; // 利用自然数溢出,即超过 LONG_MAX 自动溢出,节省时间 + } + if (hv < 0) { + return -1 * hv; + } + return hv; + } +} diff --git a/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/RelationSchema.java b/dataSource/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/RelationSchema.java similarity index 64% rename from dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/RelationSchema.java rename to dataSource/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/RelationSchema.java index 052c245cf1..5bc755d668 100644 --- a/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/RelationSchema.java +++ b/dataSource/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/RelationSchema.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.relational.tools; import static cn.edu.tsinghua.iginx.constant.GlobalConstant.SEPARATOR; @@ -30,7 +48,7 @@ public RelationSchema(String path, boolean isDummy, char quote) { this.quote = quote; } - public static String getQuotFullName(String tableName, String columnName, char quote) { + public static String getQuoteFullName(String tableName, String columnName, char quote) { return getQuotName(tableName, quote) + SEPARATOR + getQuotName(columnName, quote); } @@ -50,7 +68,7 @@ public String getColumnName() { return columnName; } - public String getQuotFullName() { + public String getQuoteFullName() { return getQuotName(tableName, quote) + SEPARATOR + getQuotName(columnName, quote); } diff --git a/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/TagKVUtils.java b/dataSource/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/TagKVUtils.java similarity index 68% rename from dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/TagKVUtils.java rename to dataSource/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/TagKVUtils.java index d84b3c8b43..8f6c0b318d 100644 --- a/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/TagKVUtils.java +++ b/dataSource/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/TagKVUtils.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.relational.tools; import static cn.edu.tsinghua.iginx.relational.tools.Constants.*; diff --git a/dataSources/relational/src/main/resources/mysql-meta-template.properties b/dataSource/relational/src/main/resources/mysql-meta-template.properties similarity index 62% rename from dataSources/relational/src/main/resources/mysql-meta-template.properties rename to dataSource/relational/src/main/resources/mysql-meta-template.properties index 118ecaaa88..d52031f1ed 100644 --- a/dataSources/relational/src/main/resources/mysql-meta-template.properties +++ b/dataSource/relational/src/main/resources/mysql-meta-template.properties @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + # 配置MySQL META以供JDBCMeta读取 # 驱动类 diff --git a/dataSources/relational/src/main/resources/oceanbase-meta.properties b/dataSource/relational/src/main/resources/oceanbase-meta.properties similarity index 62% rename from dataSources/relational/src/main/resources/oceanbase-meta.properties rename to dataSource/relational/src/main/resources/oceanbase-meta.properties index f7c63c7f93..6e2fdaaf1b 100644 --- a/dataSources/relational/src/main/resources/oceanbase-meta.properties +++ b/dataSource/relational/src/main/resources/oceanbase-meta.properties @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + # 配置MySQL META以供JDBCMeta读取 # 驱动类 diff --git a/dataSources/filesystem/pom.xml b/dataSources/filesystem/pom.xml deleted file mode 100644 index 8bafd632c0..0000000000 --- a/dataSources/filesystem/pom.xml +++ /dev/null @@ -1,72 +0,0 @@ - - - 4.0.0 - - - cn.edu.tsinghua - iginx-driver - ${revision} - - - filesystem - IGinX FileSystem - - - .. - filesystem - 8 - 8 - - - - - - org.apache.maven.plugins - maven-assembly-plugin - - - org.apache.maven.plugins - maven-dependency-plugin - 2.10 - - - copy-dependencies - - copy-dependencies - - package - - ../../core/target/iginx-core-${project.version}/driver/filesystem/ - provided - - - - - - org.apache.maven.plugins - maven-antrun-plugin - 1.7 - - - copy-native-libraries - - run - - package - - - - - - - - - - - - - - - - diff --git a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exception/FileSystemTaskExecuteFailureException.java b/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exception/FileSystemTaskExecuteFailureException.java deleted file mode 100644 index 79e4784e8f..0000000000 --- a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exception/FileSystemTaskExecuteFailureException.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package cn.edu.tsinghua.iginx.filesystem.exception; - -import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalTaskExecuteFailureException; - -public class FileSystemTaskExecuteFailureException extends PhysicalTaskExecuteFailureException { - - public FileSystemTaskExecuteFailureException(String message) { - super(message); - } - - public FileSystemTaskExecuteFailureException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exception/FilesystemException.java b/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exception/FilesystemException.java deleted file mode 100644 index c4b5c77333..0000000000 --- a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exception/FilesystemException.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2024 IGinX of Tsinghua University - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.edu.tsinghua.iginx.filesystem.exception; - -import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; - -public class FilesystemException extends PhysicalException { - - public FilesystemException(String message) { - super(message); - } - - public FilesystemException(String message, Throwable cause) { - super(message, cause); - } - - public FilesystemException(Throwable cause) { - super(cause); - } -} diff --git a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/query/entity/Record.java b/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/query/entity/Record.java deleted file mode 100644 index 373e44dae3..0000000000 --- a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/query/entity/Record.java +++ /dev/null @@ -1,33 +0,0 @@ -package cn.edu.tsinghua.iginx.filesystem.query.entity; - -import cn.edu.tsinghua.iginx.engine.shared.data.Value; -import cn.edu.tsinghua.iginx.thrift.DataType; - -public class Record { - - private final long key; - - private final Value value; - - public Record(long key, DataType dataType, Object rawData) { - this.key = key; - this.value = new Value(dataType, rawData); - } - - public Record(long key, Object rawData) { - this.key = key; - this.value = new Value(rawData); - } - - public Object getRawData() { - return value.getValue(); - } - - public long getKey() { - return key; - } - - public DataType getDataType() { - return value.getDataType(); - } -} diff --git a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/shared/FileType.java b/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/shared/FileType.java deleted file mode 100644 index 6bbb4dace8..0000000000 --- a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/shared/FileType.java +++ /dev/null @@ -1,8 +0,0 @@ -package cn.edu.tsinghua.iginx.filesystem.shared; - -public enum FileType { - IGINX_FILE, - NORMAL_FILE, - UNKNOWN_FILE, - DIR -} diff --git a/dataSources/influxdb/pom.xml b/dataSources/influxdb/pom.xml deleted file mode 100644 index 266ca425f4..0000000000 --- a/dataSources/influxdb/pom.xml +++ /dev/null @@ -1,79 +0,0 @@ - - - 4.0.0 - - - cn.edu.tsinghua - iginx-driver - ${revision} - - - influxdb - IGinX InfluxDB - - - .. - influxdb - 8 - 8 - - - - - com.influxdb - influxdb-client-java - - - - - - - org.apache.maven.plugins - maven-assembly-plugin - - - org.apache.maven.plugins - maven-dependency-plugin - 2.10 - - - copy-dependencies - - copy-dependencies - - package - - ../../core/target/iginx-core-${project.version}/driver/influxdb/ - provided - - - - - - org.apache.maven.plugins - maven-antrun-plugin - 1.7 - - - copy-native-libraries - - run - - package - - - - - - - - - - - - - - - - diff --git a/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/exception/InfluxDBException.java b/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/exception/InfluxDBException.java deleted file mode 100644 index 084e4dbfe8..0000000000 --- a/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/exception/InfluxDBException.java +++ /dev/null @@ -1,18 +0,0 @@ -package cn.edu.tsinghua.iginx.influxdb.exception; - -import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; - -public class InfluxDBException extends PhysicalException { - - public InfluxDBException(String message) { - super(message); - } - - public InfluxDBException(String message, Throwable cause) { - super(message, cause); - } - - public InfluxDBException(Throwable cause) { - super(cause); - } -} diff --git a/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/exception/InfluxDBTaskExecuteFailureException.java b/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/exception/InfluxDBTaskExecuteFailureException.java deleted file mode 100644 index 1c72331792..0000000000 --- a/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/exception/InfluxDBTaskExecuteFailureException.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package cn.edu.tsinghua.iginx.influxdb.exception; - -import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalTaskExecuteFailureException; - -public class InfluxDBTaskExecuteFailureException extends PhysicalTaskExecuteFailureException { - - public InfluxDBTaskExecuteFailureException(String message) { - super(message); - } - - public InfluxDBTaskExecuteFailureException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/exception/IoTDBException.java b/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/exception/IoTDBException.java deleted file mode 100644 index 75796b76d9..0000000000 --- a/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/exception/IoTDBException.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2024 IGinX of Tsinghua University - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.edu.tsinghua.iginx.iotdb.exception; - -import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; - -public class IoTDBException extends PhysicalException { - - public IoTDBException(String message) { - super(message); - } - - public IoTDBException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/exception/IoTDBTaskExecuteFailureException.java b/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/exception/IoTDBTaskExecuteFailureException.java deleted file mode 100644 index af7cec29fe..0000000000 --- a/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/exception/IoTDBTaskExecuteFailureException.java +++ /dev/null @@ -1,14 +0,0 @@ -package cn.edu.tsinghua.iginx.iotdb.exception; - -import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalTaskExecuteFailureException; - -public class IoTDBTaskExecuteFailureException extends PhysicalTaskExecuteFailureException { - - public IoTDBTaskExecuteFailureException(String message) { - super(message); - } - - public IoTDBTaskExecuteFailureException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/dataSources/mongodb/pom.xml b/dataSources/mongodb/pom.xml deleted file mode 100644 index c23cff5a96..0000000000 --- a/dataSources/mongodb/pom.xml +++ /dev/null @@ -1,79 +0,0 @@ - - - 4.0.0 - - - cn.edu.tsinghua - iginx-driver - ${revision} - - - mongodb - IGinX MongoDB - - - .. - mongodb - 8 - 8 - - - - - org.mongodb - mongodb-driver - - - - - - - org.apache.maven.plugins - maven-assembly-plugin - - - org.apache.maven.plugins - maven-dependency-plugin - 2.10 - - - copy-dependencies - - copy-dependencies - - package - - ../../core/target/iginx-core-${project.version}/driver/mongodb/ - provided - - - - - - org.apache.maven.plugins - maven-antrun-plugin - 1.7 - - - copy-native-libraries - - run - - package - - - - - - - - - - - - - - - - diff --git a/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/NameUtils.java b/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/NameUtils.java deleted file mode 100644 index 927c392979..0000000000 --- a/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/dummy/NameUtils.java +++ /dev/null @@ -1,19 +0,0 @@ -package cn.edu.tsinghua.iginx.mongodb.dummy; - -class NameUtils { - static String getSuffix(String path, String prefix) { - if (path.startsWith(prefix + ".")) { - return path.substring(prefix.length() + 1); - } - throw new IllegalArgumentException(prefix + " is not prefix of " + path); - } - - static boolean containNumberNode(String path) { - for (String node : path.split("\\.")) { - if (node.chars().allMatch(Character::isDigit)) { - return true; - } - } - return false; - } -} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/Database.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/Database.java deleted file mode 100644 index d34e68e53a..0000000000 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/Database.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2024 IGinX of Tsinghua University - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.edu.tsinghua.iginx.parquet.db; - -import cn.edu.tsinghua.iginx.parquet.db.util.AreaSet; -import cn.edu.tsinghua.iginx.parquet.db.util.iterator.Scanner; -import cn.edu.tsinghua.iginx.parquet.util.exception.StorageException; -import java.util.Map; - -public interface Database, F, T, V> extends ImmutableDatabase { - - void upsertRows(Scanner> scanner, Map schema) throws StorageException; - - void upsertColumns(Scanner> scanner, Map schema) throws StorageException; - - void delete(AreaSet areas) throws StorageException; - - void clear() throws StorageException; -} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/ImmutableDatabase.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/ImmutableDatabase.java deleted file mode 100644 index b8f4fcbb8a..0000000000 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/ImmutableDatabase.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2024 IGinX of Tsinghua University - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.edu.tsinghua.iginx.parquet.db; - -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; -import cn.edu.tsinghua.iginx.parquet.db.util.iterator.Scanner; -import cn.edu.tsinghua.iginx.parquet.util.exception.StorageException; -import com.google.common.collect.Range; -import com.google.common.collect.RangeSet; -import java.util.Map; -import java.util.Optional; -import java.util.Set; - -public interface ImmutableDatabase, F, T, V> extends AutoCloseable { - Scanner> query(Set fields, RangeSet ranges, Filter filter) - throws StorageException; - - Optional> range() throws StorageException; - - Map schema() throws StorageException; -} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/OneTierDB.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/OneTierDB.java deleted file mode 100644 index 8df4b6b1ea..0000000000 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/OneTierDB.java +++ /dev/null @@ -1,369 +0,0 @@ -/* - * Copyright 2024 IGinX of Tsinghua University - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.edu.tsinghua.iginx.parquet.db.lsm; - -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; -import cn.edu.tsinghua.iginx.parquet.db.Database; -import cn.edu.tsinghua.iginx.parquet.db.lsm.api.ReadWriter; -import cn.edu.tsinghua.iginx.parquet.db.lsm.buffer.DataBuffer; -import cn.edu.tsinghua.iginx.parquet.db.lsm.table.MemoryTable; -import cn.edu.tsinghua.iginx.parquet.db.lsm.table.Table; -import cn.edu.tsinghua.iginx.parquet.db.lsm.table.TableIndex; -import cn.edu.tsinghua.iginx.parquet.db.lsm.table.TableStorage; -import cn.edu.tsinghua.iginx.parquet.db.util.AreaSet; -import cn.edu.tsinghua.iginx.parquet.db.util.iterator.BatchPlaneScanner; -import cn.edu.tsinghua.iginx.parquet.db.util.iterator.Scanner; -import cn.edu.tsinghua.iginx.parquet.util.Shared; -import cn.edu.tsinghua.iginx.parquet.util.exception.NotIntegrityException; -import cn.edu.tsinghua.iginx.parquet.util.exception.StorageException; -import cn.edu.tsinghua.iginx.parquet.util.exception.StorageRuntimeException; -import com.google.common.collect.Range; -import com.google.common.collect.RangeSet; -import com.google.common.collect.TreeRangeSet; -import com.google.common.util.concurrent.ThreadFactoryBuilder; -import java.io.IOException; -import java.util.*; -import java.util.concurrent.*; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.LongAdder; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class OneTierDB, F, T, V> implements Database { - private static final Logger LOGGER = LoggerFactory.getLogger(OneTierDB.class); - private final Shared shared; - private final TableStorage tableStorage; - private final TableIndex tableIndex; - - private DataBuffer writeBuffer = new DataBuffer<>(); - private String previousTableName = null; - private final AtomicLong bufferDirtiedTime = new AtomicLong(Long.MAX_VALUE); - private final LongAdder bufferInsertedSize = new LongAdder(); - private final Lock checkLock = new ReentrantLock(true); - private final ReadWriteLock deleteLock = new ReentrantReadWriteLock(true); - private final ReadWriteLock commitLock = new ReentrantReadWriteLock(true); - private final ReadWriteLock storageLock = new ReentrantReadWriteLock(true); - - private final ScheduledExecutorService scheduler; - private final String name; - private final long timeout; - - public OneTierDB(String name, Shared shared, ReadWriter readerWriter) - throws IOException { - this.name = name; - this.shared = shared; - this.tableStorage = new TableStorage<>(shared, readerWriter); - this.tableIndex = new TableIndex<>(tableStorage); - - ThreadFactory threadFactory = - new ThreadFactoryBuilder().setNameFormat("OneTierDB-" + name + "-%d").build(); - this.timeout = shared.getStorageProperties().getWriteBufferTimeout().toMillis(); - this.scheduler = Executors.newSingleThreadScheduledExecutor(threadFactory); - if (timeout > 0) { - LOGGER.info("db {} start to check buffer timeout check every {}ms", name, timeout); - this.scheduler.scheduleWithFixedDelay( - this::checkBufferTimeout, timeout, timeout, TimeUnit.MILLISECONDS); - } else { - LOGGER.info("db {} buffer timeout check is immediately after writing", name); - } - } - - @Override - public Scanner> query(Set fields, RangeSet ranges, Filter filter) - throws StorageException { - DataBuffer readBuffer = new DataBuffer<>(); - - commitLock.readLock().lock(); - deleteLock.readLock().lock(); - storageLock.readLock().lock(); - try { - AreaSet areas = new AreaSet<>(); - areas.add(fields, ranges); - Set tables = tableIndex.find(areas); - List sortedTableNames = new ArrayList<>(tables); - sortedTableNames.sort(Comparator.naturalOrder()); - for (String tableName : sortedTableNames) { - Table table = tableStorage.get(tableName); - try (Scanner> scanner = table.scan(fields, ranges)) { - readBuffer.putRows(scanner); - } - } - - for (Range range : ranges.asRanges()) { - readBuffer.putRows(writeBuffer.scanRows(fields, range)); - } - } catch (IOException | StorageException e) { - throw new RuntimeException(e); - } finally { - storageLock.readLock().unlock(); - deleteLock.readLock().unlock(); - commitLock.readLock().unlock(); - } - - return readBuffer.scanRows(fields, Range.all()); - } - - @Override - public Optional> range() throws StorageException { - commitLock.readLock().lock(); - deleteLock.readLock().lock(); - storageLock.readLock().lock(); - try { - TreeRangeSet rangeSet = TreeRangeSet.create(); - for (Range range : tableIndex.ranges().values()) { - rangeSet.add(range); - } - for (Range range : writeBuffer.ranges().values()) { - rangeSet.add(range); - } - if (rangeSet.isEmpty()) { - return Optional.empty(); - } else { - return Optional.of(rangeSet.span()); - } - } finally { - storageLock.readLock().unlock(); - deleteLock.readLock().unlock(); - commitLock.readLock().unlock(); - } - } - - @Override - public Map schema() throws StorageException { - return tableIndex.getType(); - } - - @Override - public void upsertRows(Scanner> scanner, Map schema) - throws StorageException { - try (Scanner>> batchScanner = - new BatchPlaneScanner<>(scanner, shared.getStorageProperties().getWriteBatchSize())) { - while (batchScanner.iterate()) { - beforeWriting(); - try { - tableIndex.declareFields(schema); - try (Scanner> batch = batchScanner.value()) { - writeBuffer.putRows(batch); - bufferInsertedSize.add(batchScanner.key()); - } - } finally { - afterWriting(); - } - } - } - updateDirty(); - if (timeout <= 0) { - checkBufferTimeout(); - } - } - - @Override - public void upsertColumns(Scanner> scanner, Map schema) - throws StorageException { - try (Scanner>> batchScanner = - new BatchPlaneScanner<>(scanner, shared.getStorageProperties().getWriteBatchSize())) { - while (batchScanner.iterate()) { - beforeWriting(); - try { - tableIndex.declareFields(schema); - try (Scanner> batch = batchScanner.value()) { - writeBuffer.putColumns(batch); - bufferInsertedSize.add(batchScanner.key()); - } - } finally { - afterWriting(); - } - } - } - updateDirty(); - if (timeout <= 0) { - checkBufferTimeout(); - } - } - - private void updateDirty() { - long currentTime = System.currentTimeMillis(); - bufferDirtiedTime.updateAndGet(oldTime -> Math.min(oldTime, currentTime)); - } - - private void beforeWriting() { - checkBufferSize(); - commitLock.readLock().lock(); - deleteLock.readLock().lock(); - storageLock.readLock().lock(); - } - - private void afterWriting() { - storageLock.readLock().unlock(); - deleteLock.readLock().unlock(); - commitLock.readLock().unlock(); - } - - private void checkBufferSize() { - checkLock.lock(); - try { - if (bufferInsertedSize.sum() < shared.getStorageProperties().getWriteBufferSize()) { - return; - } - if (LOGGER.isDebugEnabled()) { - LOGGER.debug( - "flushing is triggered when write buffer size {} reaching {}", - bufferInsertedSize.sum(), - shared.getStorageProperties().getWriteBufferSize()); - } - commitMemoryTable(false, new CountDownLatch(1)); - } finally { - checkLock.unlock(); - } - } - - private void checkBufferTimeout() { - CountDownLatch latch = new CountDownLatch(1); - checkLock.lock(); - try { - long interval = System.currentTimeMillis() - bufferDirtiedTime.get(); - if (interval < timeout) { - return; - } - - LOGGER.debug( - "flushing is triggered when write buffer dirtied time {}ms reaching {}ms", - interval, - timeout); - boolean temp = bufferInsertedSize.sum() < shared.getStorageProperties().getWriteBufferSize(); - commitMemoryTable(temp, latch); - } finally { - checkLock.unlock(); - } - LOGGER.debug("waiting for flushing table"); - try { - latch.await(); - } catch (InterruptedException e) { - throw new StorageRuntimeException(e); - } - LOGGER.debug("table is flushed"); - } - - private synchronized void commitMemoryTable(boolean temp, CountDownLatch latch) { - commitLock.writeLock().lock(); - deleteLock.readLock().lock(); - try { - Map types = - tableIndex.getType( - writeBuffer.fields(), - f -> { - throw new NotIntegrityException("field " + f + " is not found in schema"); - }); - - MemoryTable table = new MemoryTable<>(writeBuffer, types); - - String toDelete = previousTableName; - String committedTableName = - tableStorage.flush( - table, - () -> { - if (toDelete != null) { - LOGGER.debug("delete table {}", toDelete); - storageLock.writeLock().lock(); - try { - tableIndex.removeTable(toDelete); - tableStorage.remove(toDelete); - } catch (Throwable e) { - LOGGER.error("failed to delete table {}", toDelete, e); - } finally { - storageLock.writeLock().unlock(); - } - } - latch.countDown(); - }); - - LOGGER.debug( - "submit table {} to flush, with temp={}, latch={}", committedTableName, temp, latch); - tableIndex.addTable(committedTableName, table.getMeta()); - this.bufferDirtiedTime.set(Long.MAX_VALUE); - previousTableName = committedTableName; - if (!temp) { - LOGGER.info("reset write buffer"); - this.writeBuffer = new DataBuffer<>(); - bufferInsertedSize.reset(); - previousTableName = null; - } - } catch (InterruptedException e) { - throw new StorageRuntimeException(e); - } finally { - deleteLock.readLock().unlock(); - commitLock.writeLock().unlock(); - } - } - - @Override - public void delete(AreaSet areas) throws StorageException { - commitLock.readLock().lock(); - deleteLock.writeLock().lock(); - storageLock.readLock().lock(); - try { - LOGGER.info("start to delete {} in {}", areas, name); - writeBuffer.remove(areas); - Set tables = tableIndex.find(areas); - tableStorage.delete(tables, areas); - tableIndex.delete(areas); - } catch (IOException e) { - throw new StorageRuntimeException(e); - } finally { - storageLock.readLock().unlock(); - deleteLock.writeLock().unlock(); - commitLock.readLock().unlock(); - } - } - - @Override - public void clear() throws StorageException { - commitLock.readLock().lock(); - deleteLock.writeLock().lock(); - try { - LOGGER.debug("start to clear {}", name); - previousTableName = null; - tableStorage.clear(); - tableIndex.clear(); - writeBuffer.clear(); - bufferDirtiedTime.set(Long.MAX_VALUE); - bufferInsertedSize.reset(); - } catch (InterruptedException e) { - throw new StorageRuntimeException(e); - } finally { - deleteLock.writeLock().unlock(); - commitLock.readLock().unlock(); - } - } - - @Override - public void close() throws Exception { - scheduler.shutdown(); - if (shared.getStorageProperties().toFlushOnClose()) { - LOGGER.info("flushing is triggered when closing"); - CountDownLatch latch = new CountDownLatch(1); - commitMemoryTable(false, latch); - latch.await(); - } - tableStorage.close(); - tableIndex.close(); - } -} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/api/ReadWriter.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/api/ReadWriter.java deleted file mode 100644 index 3c80d625f5..0000000000 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/api/ReadWriter.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2024 IGinX of Tsinghua University - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.edu.tsinghua.iginx.parquet.db.lsm.api; - -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; -import cn.edu.tsinghua.iginx.parquet.db.util.AreaSet; -import cn.edu.tsinghua.iginx.parquet.db.util.iterator.Scanner; -import com.google.common.collect.RangeSet; -import java.io.IOException; -import java.util.Set; - -public interface ReadWriter, F, T, V> { - - void flush(String name, TableMeta meta, Scanner> scanner) - throws IOException; - - TableMeta readMeta(String name) throws IOException; - - Scanner> scanData( - String name, Set fields, RangeSet ranges, Filter predicate) throws IOException; - - void delete(String name, AreaSet areas) throws IOException; - - void delete(String name); - - Iterable tableNames() throws IOException; - - void clear() throws IOException; -} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/api/TableMeta.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/api/TableMeta.java deleted file mode 100644 index 99ba2944d1..0000000000 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/api/TableMeta.java +++ /dev/null @@ -1,15 +0,0 @@ -package cn.edu.tsinghua.iginx.parquet.db.lsm.api; - -import com.google.common.collect.Range; -import java.util.Map; - -public interface TableMeta, F, T, V> { - Map getSchema(); - - /** - * Get the range of the table. 若某个字段没有数据,则返回的range为null - * - * @return the range of the table, the key is the field name, the value is the range of the field - */ - Map> getRanges(); -} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/compact/Compactor.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/compact/Compactor.java deleted file mode 100644 index 1bd1413de3..0000000000 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/compact/Compactor.java +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright 2024 IGinX of Tsinghua University - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.edu.tsinghua.iginx.parquet.db.lsm.compact; - -public class Compactor {} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/table/DeletedTable.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/table/DeletedTable.java deleted file mode 100644 index 6dd9d85fb5..0000000000 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/table/DeletedTable.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright 2024 IGinX of Tsinghua University - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.edu.tsinghua.iginx.parquet.db.lsm.table; - -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; -import cn.edu.tsinghua.iginx.parquet.db.lsm.api.TableMeta; -import cn.edu.tsinghua.iginx.parquet.db.util.AreaSet; -import cn.edu.tsinghua.iginx.parquet.db.util.iterator.AreaFilterScanner; -import cn.edu.tsinghua.iginx.parquet.db.util.iterator.Scanner; -import com.google.common.collect.RangeSet; -import java.io.IOException; -import java.util.Set; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; - -public class DeletedTable, F, T, V> implements Table { - - private final Table table; - - private final AreaSet deletedAreaSet; - - private final TableMeta meta; - - public DeletedTable(Table table, AreaSet deleted) throws IOException { - this.table = table; - this.deletedAreaSet = deleted; - this.meta = new DeletedTableMeta<>(table.getMeta(), deleted); - } - - @Nonnull - @Override - public TableMeta getMeta() throws IOException { - return meta; - } - - @Nonnull - @Override - public Scanner> scan( - @Nonnull Set fields, @Nonnull RangeSet range, @Nullable Filter predicate) - throws IOException { - return new AreaFilterScanner<>(table.scan(fields, range, predicate), deletedAreaSet); - } -} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/table/DeletedTableMeta.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/table/DeletedTableMeta.java deleted file mode 100644 index a6a32ec8d1..0000000000 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/table/DeletedTableMeta.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright 2024 IGinX of Tsinghua University - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.edu.tsinghua.iginx.parquet.db.lsm.table; - -import cn.edu.tsinghua.iginx.parquet.db.lsm.api.TableMeta; -import cn.edu.tsinghua.iginx.parquet.db.util.AreaSet; -import com.google.common.collect.Range; -import com.google.common.collect.RangeSet; -import com.google.common.collect.TreeRangeSet; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -public class DeletedTableMeta, F, T, V> implements TableMeta { - private final Map schema; - private final Map> ranges; - - public DeletedTableMeta(TableMeta tableMeta, AreaSet tombstone) { - this.schema = new HashMap<>(tableMeta.getSchema()); - schema.keySet().removeAll(tombstone.getFields()); - this.ranges = new HashMap<>(); - - Map> rangeSetMap = new HashMap<>(); - for (Map.Entry> entry : tableMeta.getRanges().entrySet()) { - F field = entry.getKey(); - Range range = entry.getValue(); - rangeSetMap.put(field, TreeRangeSet.create(Collections.singleton(range))); - } - rangeSetMap.keySet().removeAll(tombstone.getFields()); - rangeSetMap.values().forEach(rangeSet -> rangeSet.removeAll(tombstone.getKeys())); - for (Map.Entry> entry : tombstone.getSegments().entrySet()) { - F field = entry.getKey(); - RangeSet rangeSetDeleted = entry.getValue(); - RangeSet rangeSet = rangeSetMap.get(field); - if (rangeSet != null) { - rangeSet.removeAll(rangeSetDeleted); - } - } - for (Map.Entry> entry : rangeSetMap.entrySet()) { - F field = entry.getKey(); - RangeSet rangeSet = entry.getValue(); - if (!rangeSet.isEmpty()) { - ranges.put(field, rangeSet.span()); - } - } - } - - @Override - public Map getSchema() { - return schema; - } - - @Override - public Map> getRanges() { - return ranges; - } -} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/table/EmptyTable.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/table/EmptyTable.java deleted file mode 100644 index 20091f459e..0000000000 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/table/EmptyTable.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright 2024 IGinX of Tsinghua University - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.edu.tsinghua.iginx.parquet.db.lsm.table; - -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; -import cn.edu.tsinghua.iginx.parquet.db.util.iterator.EmptyScanner; -import cn.edu.tsinghua.iginx.parquet.db.util.iterator.Scanner; -import com.google.common.collect.RangeSet; -import java.io.IOException; -import java.util.HashMap; -import java.util.Set; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; - -public class EmptyTable, F, T, V> implements Table { - private static final EmptyTable EMPTY = new EmptyTable<>(); - - @SuppressWarnings("unchecked") - public static , F, T, V> EmptyTable getInstance() { - return (EmptyTable) EMPTY; - } - - @Nonnull - @Override - public MemoryTable.MemoryTableMeta getMeta() { - return new MemoryTable.MemoryTableMeta<>(new HashMap<>(), new HashMap<>()); - } - - @Nonnull - @Override - public Scanner> scan( - @Nonnull Set fields, @Nonnull RangeSet ranges, @Nullable Filter predicate) - throws IOException { - return EmptyScanner.getInstance(); - } -} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/table/FileTable.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/table/FileTable.java deleted file mode 100644 index 19c3e719a7..0000000000 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/table/FileTable.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright 2024 IGinX of Tsinghua University - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.edu.tsinghua.iginx.parquet.db.lsm.table; - -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; -import cn.edu.tsinghua.iginx.parquet.db.lsm.api.ReadWriter; -import cn.edu.tsinghua.iginx.parquet.db.lsm.api.TableMeta; -import cn.edu.tsinghua.iginx.parquet.db.util.iterator.Scanner; -import com.google.common.collect.RangeSet; -import java.io.IOException; -import java.util.Set; -import java.util.StringJoiner; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class FileTable, F, T, V> implements Table { - private static final Logger LOGGER = LoggerFactory.getLogger(FileTable.class); - private final String tableName; - - private final ReadWriter readWriter; - - public FileTable(String tableName, ReadWriter readWriter) { - this.tableName = tableName; - this.readWriter = readWriter; - } - - @Override - @Nonnull - public TableMeta getMeta() throws IOException { - return readWriter.readMeta(tableName); - } - - @Override - @Nonnull - public Scanner> scan( - @Nonnull Set fields, @Nonnull RangeSet ranges, @Nullable Filter predicate) - throws IOException { - LOGGER.debug("read {} where {} & {} from {}", fields, ranges, predicate, tableName); - return readWriter.scanData(tableName, fields, ranges, predicate); - } - - @Override - public String toString() { - return new StringJoiner(", ", FileTable.class.getSimpleName() + "[", "]") - .add("tableName='" + tableName + "'") - .add("readWriter=" + readWriter) - .toString(); - } -} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/table/MemoryTable.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/table/MemoryTable.java deleted file mode 100644 index 4bc6590c62..0000000000 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/table/MemoryTable.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright 2024 IGinX of Tsinghua University - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.edu.tsinghua.iginx.parquet.db.lsm.table; - -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; -import cn.edu.tsinghua.iginx.parquet.db.lsm.api.TableMeta; -import cn.edu.tsinghua.iginx.parquet.db.lsm.buffer.DataBuffer; -import cn.edu.tsinghua.iginx.parquet.db.util.iterator.ConcatScanner; -import cn.edu.tsinghua.iginx.parquet.db.util.iterator.Scanner; -import com.google.common.collect.Range; -import com.google.common.collect.RangeSet; -import java.io.IOException; -import java.util.*; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class MemoryTable, F, T, V> implements Table { - private static final Logger LOGGER = LoggerFactory.getLogger(MemoryTable.class); - - private final DataBuffer buffer; - private final TableMeta meta; - - public MemoryTable(DataBuffer buffer, Map types) { - this.buffer = Objects.requireNonNull(buffer); - Objects.requireNonNull(types); - this.meta = new MemoryTableMeta<>(types, buffer.ranges()); - } - - @Override - public String toString() { - return new StringJoiner(", ", MemoryTable.class.getSimpleName() + "[", "]") - .add("buffer=" + buffer) - .add("meta=" + meta) - .toString(); - } - - @Nonnull - @Override - public TableMeta getMeta() { - return meta; - } - - @Nonnull - @Override - public Scanner> scan( - @Nonnull Set fields, @Nonnull RangeSet ranges, @Nullable Filter predicate) - throws IOException { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("read {} where {} from {},{}", fields, ranges, buffer.fields(), buffer.ranges()); - } - List>> scanners = new ArrayList<>(); - for (Range range : ranges.asRanges()) { - scanners.add(buffer.scanRows(fields, range)); - } - return new ConcatScanner<>(scanners.iterator()); - } - - public static class MemoryTableMeta, F, T, V> - implements TableMeta { - private final Map schema; - private final Map> ranges; - - public MemoryTableMeta(Map schema, Map> ranges) { - this.schema = schema; - this.ranges = ranges; - } - - public Map getSchema() { - return schema; - } - - public Map> getRanges() { - return ranges; - } - - @Override - public String toString() { - return new StringJoiner(", ", MemoryTableMeta.class.getSimpleName() + "[", "]") - .add("schema=" + schema) - .add("ranges=" + ranges) - .toString(); - } - } -} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/table/Table.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/table/Table.java deleted file mode 100644 index 8a5b2370bc..0000000000 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/table/Table.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright 2024 IGinX of Tsinghua University - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.edu.tsinghua.iginx.parquet.db.lsm.table; - -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; -import cn.edu.tsinghua.iginx.parquet.db.lsm.api.TableMeta; -import cn.edu.tsinghua.iginx.parquet.db.util.iterator.Scanner; -import com.google.common.collect.RangeSet; -import java.io.IOException; -import java.util.Set; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; - -public interface Table, F, T, V> { - @Nonnull - TableMeta getMeta() throws IOException; - - @Nonnull - Scanner> scan( - @Nonnull Set fields, @Nonnull RangeSet range, @Nullable Filter predicate) - throws IOException; - - @Nonnull - default Scanner> scan(@Nonnull Set fields, @Nonnull RangeSet ranges) - throws IOException { - return scan(fields, ranges, null); - } -} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/table/TableStorage.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/table/TableStorage.java deleted file mode 100644 index 42bdfbd9b3..0000000000 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/lsm/table/TableStorage.java +++ /dev/null @@ -1,210 +0,0 @@ -/* - * Copyright 2024 IGinX of Tsinghua University - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.edu.tsinghua.iginx.parquet.db.lsm.table; - -import cn.edu.tsinghua.iginx.parquet.db.lsm.api.ReadWriter; -import cn.edu.tsinghua.iginx.parquet.db.lsm.api.TableMeta; -import cn.edu.tsinghua.iginx.parquet.db.util.AreaSet; -import cn.edu.tsinghua.iginx.parquet.db.util.SequenceGenerator; -import cn.edu.tsinghua.iginx.parquet.db.util.iterator.Scanner; -import cn.edu.tsinghua.iginx.parquet.util.Shared; -import cn.edu.tsinghua.iginx.parquet.util.exception.StorageException; -import cn.edu.tsinghua.iginx.parquet.util.exception.StorageRuntimeException; -import com.google.common.collect.ImmutableRangeSet; -import com.google.common.collect.Range; -import java.io.IOException; -import java.util.Comparator; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Semaphore; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.stream.StreamSupport; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -// TODO: merge TableStorage, TableIndex and TombstoneStorage to control concurrent access to the -// storage -public class TableStorage, F, T, V> implements AutoCloseable { - private static final Logger LOGGER = LoggerFactory.getLogger(TableStorage.class); - private final ReadWriteLock lock = new ReentrantReadWriteLock(true); - private final SequenceGenerator seqGen = new SequenceGenerator(); - - private final ExecutorService flusher = Executors.newCachedThreadPool(); - - private final Map> memTables = new HashMap<>(); - private final Map> memTombstones = new HashMap<>(); - private final Shared shared; - private final ReadWriter readWriter; - - private final int localFlusherPermitsTotal; - private final Semaphore localFlusherPermits; - - public TableStorage(Shared shared, ReadWriter readWriter) throws IOException { - this.shared = shared; - this.readWriter = readWriter; - this.localFlusherPermitsTotal = shared.getFlusherPermits().availablePermits(); - this.localFlusherPermits = new Semaphore(localFlusherPermitsTotal, true); - reload(); - } - - private void reload() throws IOException { - Iterable tableNames = readWriter.tableNames(); - String last = - StreamSupport.stream(tableNames.spliterator(), false) - .max(Comparator.naturalOrder()) - .orElse("0"); - seqGen.reset(getSeq(last)); - } - - static long getSeq(String tableName) { - return Long.parseLong(tableName, 10); - } - - static String getTableName(long seq) { - return String.format("%019d", seq); - } - - public String flush(MemoryTable table, Runnable afterFlush) - throws InterruptedException { - shared.getFlusherPermits().acquire(); - localFlusherPermits.acquire(); - lock.writeLock().lock(); - try { - String tableName = getTableName(seqGen.next()); - LOGGER.debug("waiting for flusher permit to flush {}", tableName); - memTables.put(tableName, table); - flusher.submit( - () -> { - LOGGER.debug("task to flush {} started", tableName); - try { - TableMeta meta = table.getMeta(); - try (Scanner> scanner = - table.scan(meta.getSchema().keySet(), ImmutableRangeSet.of(Range.all()))) { - readWriter.flush(tableName, meta, scanner); - } - - commitMemoryTable(tableName); - - LOGGER.debug("{} flushed", tableName); - - afterFlush.run(); - } catch (Throwable e) { - LOGGER.error("failed to flush {}", tableName, e); - } finally { - localFlusherPermits.release(); - shared.getFlusherPermits().release(); - LOGGER.trace("unlock clean lock and released flusher permit"); - } - LOGGER.debug("task to flush {} end", tableName); - }); - return tableName; - } finally { - lock.writeLock().unlock(); - } - } - - private void commitMemoryTable(String name) throws IOException { - lock.writeLock().lock(); - try { - if (memTables.remove(name) != null) { - AreaSet tombstone = memTombstones.remove(name); - if (tombstone != null) { - readWriter.delete(name, tombstone); - } - } else { - readWriter.delete(name); - } - } finally { - lock.writeLock().unlock(); - } - } - - public void remove(String name) { - lock.writeLock().lock(); - try { - if (memTables.remove(name) != null) { - memTombstones.remove(name); - } else { - readWriter.delete(name); - } - } finally { - lock.writeLock().unlock(); - } - } - - public void clear() throws StorageException, InterruptedException { - localFlusherPermits.acquire(localFlusherPermitsTotal); - lock.writeLock().lock(); - try { - memTables.clear(); - memTombstones.clear(); - readWriter.clear(); - seqGen.reset(); - } catch (IOException e) { - throw new StorageRuntimeException(e); - } finally { - localFlusherPermits.release(localFlusherPermitsTotal); - lock.writeLock().unlock(); - } - } - - public Table get(String tableName) throws IOException { - lock.readLock().lock(); - try { - MemoryTable table = memTables.get(tableName); - if (table != null) { - AreaSet tombstone = memTombstones.get(tableName); - if (tombstone == null) { - return table; - } - return new DeletedTable<>(table, tombstone); - } - return new FileTable<>(tableName, readWriter); - } finally { - lock.readLock().unlock(); - } - } - - public void delete(Set tables, AreaSet areas) throws IOException { - lock.writeLock().lock(); - try { - for (String tableName : tables) { - MemoryTable table = memTables.get(tableName); - if (table != null) { - memTombstones.computeIfAbsent(tableName, k -> new AreaSet<>()).addAll(areas); - } else { - readWriter.delete(tableName, areas); - } - } - } finally { - lock.writeLock().unlock(); - } - } - - public Iterable tableNames() throws IOException { - return readWriter.tableNames(); - } - - @Override - public void close() { - flusher.shutdown(); - } -} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/SequenceGenerator.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/SequenceGenerator.java deleted file mode 100644 index 01f15c52ea..0000000000 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/SequenceGenerator.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2024 IGinX of Tsinghua University - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.edu.tsinghua.iginx.parquet.db.util; - -import java.util.concurrent.atomic.AtomicLong; - -public class SequenceGenerator { - - private final AtomicLong current = new AtomicLong(); - - public long next() { - return current.incrementAndGet(); - } - - public void reset() { - reset(new AtomicLong().get()); - } - - public void reset(long last) { - current.set(last); - } -} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/Scanner.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/Scanner.java deleted file mode 100644 index c01fab50d7..0000000000 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/db/util/iterator/Scanner.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2024 IGinX of Tsinghua University - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.edu.tsinghua.iginx.parquet.db.util.iterator; - -import cn.edu.tsinghua.iginx.parquet.util.exception.StorageException; -import javax.annotation.Nonnull; - -public interface Scanner extends AutoCloseable { - @Nonnull - K key(); - - @Nonnull - V value(); - - boolean iterate() throws StorageException; - - @Override - void close() throws StorageException; -} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/FileFormat.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/FileFormat.java deleted file mode 100644 index b0f8cdef9e..0000000000 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/FileFormat.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright 2024 IGinX of Tsinghua University - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.edu.tsinghua.iginx.parquet.io; - -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; -import java.io.IOException; -import java.nio.file.Path; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; - -public interface FileFormat { - - boolean readIsSeekable(); - - boolean writeIsSeekable(); - - /** - * load meta data from file - * - * @param path file path - * @return meta data - * @throws IOException if an I/O error occurs - */ - @Nonnull - FileMeta getMeta(@Nonnull Path path) throws IOException; - - /** - * load data index from file - * - * @param path file path - * @return data index - * @throws IOException if an I/O error occurs - */ - @Nullable - FileIndex getIndex(@Nonnull Path path, @Nonnull FileMeta meta) throws IOException; - - /** - * get reader of specified file - * - * @param path file path - * @param meta meta data loaded from file - * @param index data index loaded from file - * @param filter filter to apply - * @return file reader - * @throws IOException if an I/O error occurs - */ - @Nonnull - FileReader getReader( - @Nonnull Path path, @Nonnull FileMeta meta, @Nonnull FileIndex index, @Nonnull Filter filter) - throws IOException; - - /** - * get writer of specified file - * - * @param path file path - * @param meta meta data to write - * @return file writer - * @throws IOException if an I/O error occurs - */ - @Nonnull - FileWriter getWriter(@Nonnull Path path, @Nonnull FileMeta meta) throws IOException; -} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/FileIndex.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/FileIndex.java deleted file mode 100644 index 8353a5db65..0000000000 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/FileIndex.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright 2024 IGinX of Tsinghua University - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.edu.tsinghua.iginx.parquet.io; - -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; -import cn.edu.tsinghua.iginx.parquet.db.util.iterator.Scanner; -import java.io.IOException; -import javax.annotation.Nonnull; - -public interface FileIndex { - - /** - * estimate size of data index - * - * @return size of data index in bytes - */ - long size(); - - /** - * detect row ranges - * - * @param filter filter - * @return row ranges. key is start row offset, value is row number. - * @throws IOException if an I/O error occurs - */ - @Nonnull - Scanner detect(@Nonnull Filter filter) throws IOException; -} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/FileMeta.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/FileMeta.java deleted file mode 100644 index 42a234f097..0000000000 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/FileMeta.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2024 IGinX of Tsinghua University - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.edu.tsinghua.iginx.parquet.io; - -import cn.edu.tsinghua.iginx.thrift.DataType; -import java.util.Map; -import java.util.Set; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; - -public interface FileMeta { - - /** - * get all fields' name in file - * - * @return all names of fields in file. null if not existed - */ - @Nullable - Set fields(); - - /** - * get type of specified field - * - * @param field field type - * @return type of specified field. null if not existed - */ - @Nullable - DataType getType(@Nonnull String field); - - /** - * get extra information of file - * - * @return extra information of file. null if not existed - */ - @Nullable - Map extra(); -} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/FileReader.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/FileReader.java deleted file mode 100644 index 1391a01d40..0000000000 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/FileReader.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2024 IGinX of Tsinghua University - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.edu.tsinghua.iginx.parquet.io; - -import cn.edu.tsinghua.iginx.parquet.io.common.DataChunk; -import java.io.Closeable; -import java.io.IOException; -import java.util.Map; -import java.util.Set; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; - -public interface FileReader extends Closeable { - - /** - * set start row offset to read - * - * @param start start row offset to read - */ - void seek(long start) throws IOException; - - /** - * load data chunks in current row offset - * - * @param fields fields to load. empty if all fields - * @param limit max number of the row to load - * @return data chunks of each field, null if reader has no more data - */ - @Nullable - Map load(@Nonnull Set fields, long limit) throws IOException; -} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/FileWriter.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/FileWriter.java deleted file mode 100644 index 9f0509c65f..0000000000 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/FileWriter.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright 2024 IGinX of Tsinghua University - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.edu.tsinghua.iginx.parquet.io; - -import cn.edu.tsinghua.iginx.parquet.io.common.DataChunk; -import java.io.Closeable; -import java.io.IOException; -import java.util.Map; - -public interface FileWriter extends Closeable { - - /** - * start row offset to read - * - * @param start start row offset to read - */ - void seek(long start) throws IOException; - - /** - * dump data chunks into current row offset - * - * @param data data chunks to dump. key is field name, value is data chunk. row number of each - * data chunk must be same. - */ - void dump(Map data, long limit) throws IOException; -} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/common/DataChunk.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/common/DataChunk.java deleted file mode 100644 index 1c7ffc217e..0000000000 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/common/DataChunk.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2024 IGinX of Tsinghua University - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.edu.tsinghua.iginx.parquet.io.common; - -import cn.edu.tsinghua.iginx.parquet.db.util.iterator.Scanner; -import javax.annotation.Nonnull; - -public interface DataChunk extends AutoCloseable { - - /** - * get the size of this data chunk - * - * @return size of this data chunk in bytes - */ - long bytes(); - - /** - * get the scanner of this data chunk - * - * @param start begin row offset - * @return scanner of this data chunk - */ - @Nonnull - Scanner scan(long start); -} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/common/DataChunks.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/common/DataChunks.java deleted file mode 100644 index 047543c3d2..0000000000 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/common/DataChunks.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2024 IGinX of Tsinghua University - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.edu.tsinghua.iginx.parquet.io.common; - -import cn.edu.tsinghua.iginx.parquet.db.util.iterator.EmptyScanner; -import cn.edu.tsinghua.iginx.parquet.db.util.iterator.Scanner; -import javax.annotation.Nonnull; - -public class DataChunks { - private DataChunks() {} - - private static class EmptyDataChunk implements DataChunk { - - @Override - public long bytes() { - return 0; - } - - @Nonnull - @Override - public Scanner scan(long position) { - return EmptyScanner.getInstance(); - } - - @Override - public void close() throws Exception {} - } - - private static final DataChunk EMPTY = new EmptyDataChunk(); - - public static DataChunk empty() { - return EMPTY; - } -} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/common/EmptyReader.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/common/EmptyReader.java deleted file mode 100644 index b9de15f41b..0000000000 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/io/common/EmptyReader.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2024 IGinX of Tsinghua University - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.edu.tsinghua.iginx.parquet.io.common; - -import cn.edu.tsinghua.iginx.parquet.io.FileReader; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; -import javax.annotation.Nonnull; - -public class EmptyReader implements FileReader { - private EmptyReader() {} - - public static EmptyReader getInstance() { - return EmptyReader.INSTANCE; - } - - private static final EmptyReader INSTANCE = new EmptyReader(); - - @Nonnull - @Override - public Map load(@Nonnull Set fields, long limit) throws IOException { - Map result = new HashMap<>(); - for (String field : fields) { - result.put(field, DataChunks.empty()); - } - return result; - } - - @Override - public void seek(long start) throws IOException { - throw new IOException("EmptyReader is not seekable"); - } - - @Override - public void close() throws IOException {} -} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/Manager.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/Manager.java deleted file mode 100644 index e072f50b2b..0000000000 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/Manager.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright 2024 IGinX of Tsinghua University - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.edu.tsinghua.iginx.parquet.manager; - -import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; -import cn.edu.tsinghua.iginx.engine.physical.storage.domain.Column; -import cn.edu.tsinghua.iginx.engine.shared.KeyRange; -import cn.edu.tsinghua.iginx.engine.shared.data.read.RowStream; -import cn.edu.tsinghua.iginx.engine.shared.data.write.DataView; -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; -import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; -import cn.edu.tsinghua.iginx.metadata.entity.KeyInterval; -import java.util.List; - -public interface Manager extends AutoCloseable { - - RowStream project(List paths, TagFilter tagFilter, Filter filter) - throws PhysicalException; - - void insert(DataView dataView) throws PhysicalException; - - void delete(List paths, List keyRanges, TagFilter tagFilter) - throws PhysicalException;; - - List getColumns() throws PhysicalException; - - KeyInterval getKeyInterval() throws PhysicalException; -} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/LongFormat.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/LongFormat.java deleted file mode 100644 index 0a3d9a3930..0000000000 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/LongFormat.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright 2024 IGinX of Tsinghua University - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.edu.tsinghua.iginx.parquet.manager.data; - -public class LongFormat implements ObjectFormat { - @Override - public String format(Long value) { - return value.toString(); - } - - @Override - public Long parse(String source) { - return Long.parseLong(source); - } -} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/ObjectFormat.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/ObjectFormat.java deleted file mode 100644 index 538ca3d9b8..0000000000 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/ObjectFormat.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright 2024 IGinX of Tsinghua University - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.edu.tsinghua.iginx.parquet.manager.data; - -public interface ObjectFormat { - - String format(V value); - - V parse(String source); -} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/SizeUtils.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/SizeUtils.java deleted file mode 100644 index 3c205ba400..0000000000 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/SizeUtils.java +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright 2024 IGinX of Tsinghua University - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.edu.tsinghua.iginx.parquet.manager.data; - -public class SizeUtils {} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/StringFormat.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/StringFormat.java deleted file mode 100644 index 21f0130f1d..0000000000 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/data/StringFormat.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright 2024 IGinX of Tsinghua University - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.edu.tsinghua.iginx.parquet.manager.data; - -public class StringFormat implements ObjectFormat { - @Override - public String format(String value) { - return value; - } - - @Override - public String parse(String source) { - return source; - } -} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/utils/TagKVUtils.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/utils/TagKVUtils.java deleted file mode 100644 index 36dca6da5d..0000000000 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/utils/TagKVUtils.java +++ /dev/null @@ -1,38 +0,0 @@ -package cn.edu.tsinghua.iginx.parquet.manager.utils; - -import cn.edu.tsinghua.iginx.engine.physical.storage.domain.ColumnKey; -import cn.edu.tsinghua.iginx.engine.physical.storage.utils.ColumnKeyTranslator; -import cn.edu.tsinghua.iginx.utils.Escaper; -import java.text.ParseException; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -public class TagKVUtils { - private static final ColumnKeyTranslator COLUMN_KEY_TRANSLATOR = - new ColumnKeyTranslator(',', '=', getEscaper()); - - private static Escaper getEscaper() { - Map replacementMap = new HashMap<>(); - replacementMap.put('\\', '\\'); - replacementMap.put(',', ','); - replacementMap.put('=', '='); - return new Escaper('\\', replacementMap); - } - - public static String toFullName(String name, Map tags) { - if (tags == null) { - tags = Collections.emptyMap(); - } - ColumnKey columnKey = new ColumnKey(name, tags); - return COLUMN_KEY_TRANSLATOR.translate(columnKey); - } - - public static ColumnKey splitFullName(String fullName) { - try { - return COLUMN_KEY_TRANSLATOR.translate(fullName); - } catch (ParseException e) { - throw new IllegalStateException("Failed to parse identifier: " + fullName, e); - } - } -} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/Shared.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/Shared.java deleted file mode 100644 index d4b6ddf327..0000000000 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/Shared.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2024 IGinX of Tsinghua University - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.edu.tsinghua.iginx.parquet.util; - -import java.util.concurrent.Semaphore; - -public class Shared { - private final StorageProperties storageProperties; - - private final Semaphore flusherPermits; - - private final CachePool cachePool; - - public Shared( - StorageProperties storageProperties, Semaphore flusherPermits, CachePool cachePool) { - this.storageProperties = storageProperties; - this.flusherPermits = flusherPermits; - this.cachePool = cachePool; - } - - public static Shared of(StorageProperties storageProperties) { - Semaphore flusherPermits = new Semaphore(storageProperties.getCompactPermits(), true); - CachePool cachePool = new CachePool(storageProperties); - return new Shared(storageProperties, flusherPermits, cachePool); - } - - public StorageProperties getStorageProperties() { - return storageProperties; - } - - public Semaphore getFlusherPermits() { - return flusherPermits; - } - - public CachePool getCachePool() { - return cachePool; - } -} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/InvalidFieldNameException.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/InvalidFieldNameException.java deleted file mode 100644 index 46926c296c..0000000000 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/InvalidFieldNameException.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2024 IGinX of Tsinghua University - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.edu.tsinghua.iginx.parquet.util.exception; - -public class InvalidFieldNameException extends SchemaException { - - private final String fieldName; - - private final String reason; - - public InvalidFieldNameException(String fieldName, String reason) { - super(String.format("invalid field name %s, because: ", fieldName, reason)); - this.fieldName = fieldName; - this.reason = reason; - } - - public String getFieldName() { - return fieldName; - } - - public String getReason() { - return reason; - } -} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/IsClosedException.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/IsClosedException.java deleted file mode 100644 index 158e1f1cc7..0000000000 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/IsClosedException.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright 2024 IGinX of Tsinghua University - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.edu.tsinghua.iginx.parquet.util.exception; - -public class IsClosedException extends RuntimeException { - - public IsClosedException(String message) { - super(message); - } -} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/NotIntegrityException.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/NotIntegrityException.java deleted file mode 100644 index 3344488d22..0000000000 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/NotIntegrityException.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2024 IGinX of Tsinghua University - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.edu.tsinghua.iginx.parquet.util.exception; - -public class NotIntegrityException extends StorageRuntimeException { - public NotIntegrityException(String message) { - super(message); - } - - public NotIntegrityException(String message, Throwable cause) { - super(message, cause); - } - - public NotIntegrityException(Throwable cause) { - super(cause); - } -} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/SchemaException.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/SchemaException.java deleted file mode 100644 index a7885cd98d..0000000000 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/SchemaException.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright 2024 IGinX of Tsinghua University - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.edu.tsinghua.iginx.parquet.util.exception; - -public class SchemaException extends StorageException { - - public SchemaException(String message) { - super(message); - } -} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/StorageException.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/StorageException.java deleted file mode 100644 index 6b1376ddb0..0000000000 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/StorageException.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2024 IGinX of Tsinghua University - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.edu.tsinghua.iginx.parquet.util.exception; - -import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; - -public class StorageException extends PhysicalException { - - public StorageException(Throwable cause) { - super(cause); - } - - public StorageException(String message) { - super(message); - } - - public StorageException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/StorageRuntimeException.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/StorageRuntimeException.java deleted file mode 100644 index 26b95af0aa..0000000000 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/StorageRuntimeException.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2024 IGinX of Tsinghua University - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.edu.tsinghua.iginx.parquet.util.exception; - -import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalRuntimeException; - -public class StorageRuntimeException extends PhysicalRuntimeException { - public StorageRuntimeException(String message) { - super(message); - } - - public StorageRuntimeException(String message, Throwable cause) { - super(message, cause); - } - - public StorageRuntimeException(Throwable cause) { - super(cause); - } -} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/TimeoutException.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/TimeoutException.java deleted file mode 100644 index 83014cf257..0000000000 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/TimeoutException.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2024 IGinX of Tsinghua University - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.edu.tsinghua.iginx.parquet.util.exception; - -public class TimeoutException extends StorageException { - public TimeoutException(String message) { - super(message); - } - - public TimeoutException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/TypeConflictedException.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/TypeConflictedException.java deleted file mode 100644 index 18f2ccfd71..0000000000 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/TypeConflictedException.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright 2024 IGinX of Tsinghua University - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.edu.tsinghua.iginx.parquet.util.exception; - -public class TypeConflictedException extends SchemaException { - private final String field; - private final String type; - private final String oldType; - - public TypeConflictedException(String field, String type, String oldType) { - super(String.format("can't insert %s value into %s column at %s", type, oldType, field)); - this.field = field; - this.type = type; - this.oldType = oldType; - } - - public String getField() { - return field; - } - - public String getType() { - return type; - } - - public String getOldType() { - return oldType; - } -} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/UnsupportedFilterException.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/UnsupportedFilterException.java deleted file mode 100644 index ca96d70995..0000000000 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/util/exception/UnsupportedFilterException.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2024 IGinX of Tsinghua University - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.edu.tsinghua.iginx.parquet.util.exception; - -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; - -public class UnsupportedFilterException extends StorageException { - private final Filter filter; - - public UnsupportedFilterException(Filter filter) { - super(String.format("unsupported filter %s", filter.toString())); - this.filter = filter; - } - - public UnsupportedFilterException(String message, Filter filter) { - super(message); - this.filter = filter; - } - - public Filter getFilter() { - return filter; - } -} diff --git a/dataSources/pom.xml b/dataSources/pom.xml deleted file mode 100644 index a1cdc62a91..0000000000 --- a/dataSources/pom.xml +++ /dev/null @@ -1,77 +0,0 @@ - - - - 4.0.0 - - cn.edu.tsinghua - iginx - ${revision} - - - iginx-driver - pom - IGinX Driver - - - filesystem - influxdb - iotdb12 - mongodb - parquet - redis - relational - - - - true - - - - - cn.edu.tsinghua - iginx-core - provided - - - - - - - - org.apache.maven.plugins - maven-assembly-plugin - - false - - ${driver.basedir}/src/assembly/driver.xml - - - - - - single - - package - - - - - - - - diff --git a/dataSources/redis/pom.xml b/dataSources/redis/pom.xml deleted file mode 100644 index d77b50aac4..0000000000 --- a/dataSources/redis/pom.xml +++ /dev/null @@ -1,84 +0,0 @@ - - - 4.0.0 - - cn.edu.tsinghua - iginx-driver - ${revision} - - - redis - IGinX Redis - - - .. - redis - 8 - 8 - - - - - redis.clients - jedis - - - org.slf4j - slf4j-api - - - - - - - - - org.apache.maven.plugins - maven-assembly-plugin - - - org.apache.maven.plugins - maven-dependency-plugin - 2.10 - - - copy-dependencies - - copy-dependencies - - package - - ../../core/target/iginx-core-${project.version}/driver/redis/ - provided - - - - - - org.apache.maven.plugins - maven-antrun-plugin - 1.7 - - - copy-native-libraries - - run - - package - - - - - - - - - - - - - - - - diff --git a/dataSources/relational/pom.xml b/dataSources/relational/pom.xml deleted file mode 100644 index 7d4b4368c3..0000000000 --- a/dataSources/relational/pom.xml +++ /dev/null @@ -1,95 +0,0 @@ - - - 4.0.0 - - - cn.edu.tsinghua - iginx-driver - ${revision} - - - relational - IGinX relational - - - .. - relational - 8 - 8 - - - - org.postgresql - postgresql - - - com.zaxxer - HikariCP - 2.4.7 - compile - - - org.slf4j - slf4j-api - - - - - mysql - mysql-connector-java - 8.0.26 - - - - - - - org.apache.maven.plugins - maven-assembly-plugin - - - org.apache.maven.plugins - maven-dependency-plugin - 2.10 - - - copy-dependencies - - copy-dependencies - - package - - ../../core/target/iginx-core-${project.version}/driver/relational/ - provided - - - - - - org.apache.maven.plugins - maven-antrun-plugin - 1.7 - - - copy-native-libraries - - run - - package - - - - - - - - - - - - - - - - diff --git a/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/datatype/transformer/IDataTypeTransformer.java b/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/datatype/transformer/IDataTypeTransformer.java deleted file mode 100644 index bdf0530397..0000000000 --- a/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/datatype/transformer/IDataTypeTransformer.java +++ /dev/null @@ -1,10 +0,0 @@ -package cn.edu.tsinghua.iginx.relational.datatype.transformer; - -import cn.edu.tsinghua.iginx.thrift.DataType; - -public interface IDataTypeTransformer { - - public DataType fromEngineType(String dataType); - - public String toEngineType(DataType dataType); -} diff --git a/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/exception/RelationalException.java b/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/exception/RelationalException.java deleted file mode 100644 index 2b3bd398f3..0000000000 --- a/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/exception/RelationalException.java +++ /dev/null @@ -1,18 +0,0 @@ -package cn.edu.tsinghua.iginx.relational.exception; - -import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; - -public class RelationalException extends PhysicalException { - - public RelationalException(Throwable cause) { - super(cause); - } - - public RelationalException(String message) { - super(message); - } - - public RelationalException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/exception/RelationalTaskExecuteFailureException.java b/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/exception/RelationalTaskExecuteFailureException.java deleted file mode 100644 index 3c588a5e4c..0000000000 --- a/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/exception/RelationalTaskExecuteFailureException.java +++ /dev/null @@ -1,14 +0,0 @@ -package cn.edu.tsinghua.iginx.relational.exception; - -import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalTaskExecuteFailureException; - -public class RelationalTaskExecuteFailureException extends PhysicalTaskExecuteFailureException { - - public RelationalTaskExecuteFailureException(String message) { - super(message); - } - - public RelationalTaskExecuteFailureException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/ColumnField.java b/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/ColumnField.java deleted file mode 100644 index ca8bc0e0cc..0000000000 --- a/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/ColumnField.java +++ /dev/null @@ -1,13 +0,0 @@ -package cn.edu.tsinghua.iginx.relational.tools; - -public class ColumnField { - public String tableName; - public String columnName; - public String columnType; - - public ColumnField(String tableName, String columnName, String columnType) { - this.tableName = tableName; - this.columnName = columnName; - this.columnType = columnType; - } -} diff --git a/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/HashUtils.java b/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/HashUtils.java deleted file mode 100644 index 45c3845a60..0000000000 --- a/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/HashUtils.java +++ /dev/null @@ -1,17 +0,0 @@ -package cn.edu.tsinghua.iginx.relational.tools; - -public class HashUtils { - - public static long toHash(String s) { - char c[] = s.toCharArray(); - long hv = 0; - long base = 131; - for (int i = 0; i < c.length; i++) { - hv = hv * base + (long) c[i]; // 利用自然数溢出,即超过 LONG_MAX 自动溢出,节省时间 - } - if (hv < 0) { - return -1 * hv; - } - return hv; - } -} diff --git a/dataSources/src/assembly/driver.xml b/dataSources/src/assembly/driver.xml deleted file mode 100644 index 84b424f625..0000000000 --- a/dataSources/src/assembly/driver.xml +++ /dev/null @@ -1,28 +0,0 @@ - - - - driver - - dir - - false - - - driver/${driver.name} - - - \ No newline at end of file diff --git a/dependency/pom.xml b/dependency/pom.xml new file mode 100644 index 0000000000..930901c80f --- /dev/null +++ b/dependency/pom.xml @@ -0,0 +1,68 @@ + + + + 4.0.0 + + cn.edu.tsinghua + iginx + ${revision} + + + iginx-dependency + pom + IGinX Dependency + + + true + + + + + install-jars + + true + + + + + org.apache.maven.plugins + maven-install-plugin + 3.1.2 + + + install-pemjax + + install-file + + generate-sources + + src/main/resources/pemja-0.5-SNAPSHOT.jar + + + + + + + + + + diff --git a/dependency/src/main/resources/iginx-optimizer-extension-0.7.0.jar b/dependency/src/main/resources/iginx-optimizer-extension-0.7.0.jar new file mode 100644 index 0000000000..61b4e81405 Binary files /dev/null and b/dependency/src/main/resources/iginx-optimizer-extension-0.7.0.jar differ diff --git a/dependency/src/main/resources/pemja-0.5-SNAPSHOT.jar b/dependency/src/main/resources/pemja-0.5-SNAPSHOT.jar new file mode 100644 index 0000000000..fa5c673446 Binary files /dev/null and b/dependency/src/main/resources/pemja-0.5-SNAPSHOT.jar differ diff --git a/docker/client/Dockerfile b/docker/client/Dockerfile index 98c47d9b62..e144d98aa6 100644 --- a/docker/client/Dockerfile +++ b/docker/client/Dockerfile @@ -1,10 +1,28 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + FROM maven:3-amazoncorretto-8 AS builder COPY . /root/iginx WORKDIR /root/iginx RUN mvn clean package -pl client -am -Dmaven.test.skip=true -P-format -e FROM openjdk:11-jre-slim -COPY --from=builder /root/iginx/client/target/iginx-client-0.6.0-SNAPSHOT/ /iginx_client/ +COPY --from=builder /root/iginx/client/target/iginx-client-0.8.0-SNAPSHOT/ /iginx_client/ RUN mkdir -p /iginx_client/logs RUN mkdir -p /iginx_client/data diff --git a/docker/client/Dockerfile-no-maven b/docker/client/Dockerfile-no-maven index af0681b03d..298b80a6c8 100644 --- a/docker/client/Dockerfile-no-maven +++ b/docker/client/Dockerfile-no-maven @@ -1,5 +1,5 @@ FROM openjdk:11-jre-slim -COPY ./target/iginx-client-0.6.0-SNAPSHOT/ /iginx_client/ +COPY ./target/iginx-client-0.8.0-SNAPSHOT/ /iginx_client/ RUN mkdir -p /iginx_client/logs RUN mkdir -p /iginx_client/data diff --git a/docker/client/Dockerfile-no-maven-windows b/docker/client/Dockerfile-no-maven-windows index 251678cf37..e7aa37a211 100644 --- a/docker/client/Dockerfile-no-maven-windows +++ b/docker/client/Dockerfile-no-maven-windows @@ -2,7 +2,7 @@ FROM mcr.microsoft.com/windows/servercore:ltsc2022 COPY . C:/iginx WORKDIR C:/iginx -COPY ./target/iginx-client-0.6.0-SNAPSHOT/ C:/iginx_client +COPY ./target/iginx-client-0.8.0-SNAPSHOT/ C:/iginx_client # 设置环境变量 USER ContainerAdministrator diff --git a/docker/client/build-no-maven.bat b/docker/client/build-no-maven.bat index 02015e96a7..e4145ae956 100644 --- a/docker/client/build-no-maven.bat +++ b/docker/client/build-no-maven.bat @@ -1 +1,19 @@ -docker build --file Dockerfile-no-maven-windows -t iginx-client:0.6.0 ../../client \ No newline at end of file +@REM +@REM IGinX - the polystore system with high performance +@REM Copyright (C) Tsinghua University +@REM +@REM This program is free software: you can redistribute it and/or modify +@REM it under the terms of the GNU General Public License as published by +@REM the Free Software Foundation, either version 3 of the License, or +@REM (at your option) any later version. +@REM +@REM This program is distributed in the hope that it will be useful, +@REM but WITHOUT ANY WARRANTY; without even the implied warranty of +@REM MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +@REM GNU General Public License for more details. +@REM +@REM You should have received a copy of the GNU General Public License +@REM along with this program. If not, see . +@REM + +docker build --file Dockerfile-no-maven-windows -t iginx-client:0.8.0-SNAPSHOT ../../client \ No newline at end of file diff --git a/docker/client/build-no-maven.sh b/docker/client/build-no-maven.sh index 961ecc07e8..225f402534 100644 --- a/docker/client/build-no-maven.sh +++ b/docker/client/build-no-maven.sh @@ -1,2 +1,20 @@ #!/bin/bash -docker build --file Dockerfile-no-maven -t iginx-client:0.6.0 ../../client \ No newline at end of file +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +docker build --file Dockerfile-no-maven -t iginx-client:0.8.0-SNAPSHOT ../../client \ No newline at end of file diff --git a/docker/client/build.bat b/docker/client/build.bat index 2861d26462..8952efafba 100644 --- a/docker/client/build.bat +++ b/docker/client/build.bat @@ -1 +1,19 @@ -docker build --file Dockerfile -t iginx-client:0.6.0 ../../client \ No newline at end of file +@REM +@REM IGinX - the polystore system with high performance +@REM Copyright (C) Tsinghua University +@REM +@REM This program is free software: you can redistribute it and/or modify +@REM it under the terms of the GNU General Public License as published by +@REM the Free Software Foundation, either version 3 of the License, or +@REM (at your option) any later version. +@REM +@REM This program is distributed in the hope that it will be useful, +@REM but WITHOUT ANY WARRANTY; without even the implied warranty of +@REM MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +@REM GNU General Public License for more details. +@REM +@REM You should have received a copy of the GNU General Public License +@REM along with this program. If not, see . +@REM + +docker build --file Dockerfile -t iginx-client:0.8.0-SNAPSHOT ../../client \ No newline at end of file diff --git a/docker/client/build.sh b/docker/client/build.sh index 9f0a707b1a..13c3e887fb 100644 --- a/docker/client/build.sh +++ b/docker/client/build.sh @@ -1,2 +1,20 @@ #!/bin/bash -docker build --file Dockerfile -t iginx-client:0.6.0 ../../client \ No newline at end of file +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +docker build --file Dockerfile -t iginx-client:0.8.0-SNAPSHOT ../../client \ No newline at end of file diff --git a/docker/client/run_docker.bat b/docker/client/run_docker.bat index ea633324b5..b020f9ca38 100644 --- a/docker/client/run_docker.bat +++ b/docker/client/run_docker.bat @@ -1,3 +1,21 @@ +@REM +@REM IGinX - the polystore system with high performance +@REM Copyright (C) Tsinghua University +@REM +@REM This program is free software: you can redistribute it and/or modify +@REM it under the terms of the GNU General Public License as published by +@REM the Free Software Foundation, either version 3 of the License, or +@REM (at your option) any later version. +@REM +@REM This program is distributed in the hope that it will be useful, +@REM but WITHOUT ANY WARRANTY; without even the implied warranty of +@REM MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +@REM GNU General Public License for more details. +@REM +@REM You should have received a copy of the GNU General Public License +@REM along with this program. If not, see . +@REM + @echo off setlocal enabledelayedexpansion @@ -34,7 +52,7 @@ if not exist "%datadir%" ( mkdir "%datadir%" ) -set command=docker run --name="%name%" -dit --add-host=host.docker.internal:host-gateway --mount type=bind,source=!datadir!,target=C:/iginx_client/data iginx-client:0.6.0 +set command=docker run --name="%name%" -dit --add-host=host.docker.internal:host-gateway --mount type=bind,source=!datadir!,target=C:/iginx_client/data iginx-client:0.8.0-SNAPSHOT echo %command% %command% diff --git a/docker/client/run_docker.sh b/docker/client/run_docker.sh index 5384ea95c4..7d54fef2e4 100644 --- a/docker/client/run_docker.sh +++ b/docker/client/run_docker.sh @@ -1,4 +1,22 @@ #!/bin/bash +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + datadir="$(pwd)/data" name="iginx-client" @@ -21,6 +39,6 @@ done [ -d "$datadir" ] || mkdir -p "$datadir" -command="docker run --name=\"$name\" --privileged -dit --add-host=host.docker.internal:host-gateway --mount type=bind,source=${datadir},target=/iginx_client/data iginx-client:0.6.0" +command="docker run --name=\"$name\" --privileged -dit --add-host=host.docker.internal:host-gateway --mount type=bind,source=${datadir},target=/iginx_client/data iginx-client:0.8.0-SNAPSHOT" echo $command eval $command diff --git a/docker/oneShot-parquet/Dockerfile b/docker/oneShot-parquet/Dockerfile index 2c1f315d8c..aabf826670 100644 --- a/docker/oneShot-parquet/Dockerfile +++ b/docker/oneShot-parquet/Dockerfile @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + FROM maven:3-amazoncorretto-8 AS builder COPY . /root/IGinX WORKDIR /root/IGinX diff --git a/docker/oneShot-parquet/build_and_run_iginx_docker.bat b/docker/oneShot-parquet/build_and_run_iginx_docker.bat index 3178b9fdc3..ff90426f1f 100644 --- a/docker/oneShot-parquet/build_and_run_iginx_docker.bat +++ b/docker/oneShot-parquet/build_and_run_iginx_docker.bat @@ -1 +1,19 @@ +@REM +@REM IGinX - the polystore system with high performance +@REM Copyright (C) Tsinghua University +@REM +@REM This program is free software: you can redistribute it and/or modify +@REM it under the terms of the GNU General Public License as published by +@REM the Free Software Foundation, either version 3 of the License, or +@REM (at your option) any later version. +@REM +@REM This program is distributed in the hope that it will be useful, +@REM but WITHOUT ANY WARRANTY; without even the implied warranty of +@REM MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +@REM GNU General Public License for more details. +@REM +@REM You should have received a copy of the GNU General Public License +@REM along with this program. If not, see . +@REM + docker compose up --build --detach \ No newline at end of file diff --git a/docker/oneShot-parquet/build_and_run_iginx_docker.sh b/docker/oneShot-parquet/build_and_run_iginx_docker.sh index 3178b9fdc3..544c1868d6 100644 --- a/docker/oneShot-parquet/build_and_run_iginx_docker.sh +++ b/docker/oneShot-parquet/build_and_run_iginx_docker.sh @@ -1 +1,19 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + docker compose up --build --detach \ No newline at end of file diff --git a/docker/oneShot-parquet/docker-compose.yaml b/docker/oneShot-parquet/docker-compose.yaml index a6931ea166..e2e7ee5e4d 100644 --- a/docker/oneShot-parquet/docker-compose.yaml +++ b/docker/oneShot-parquet/docker-compose.yaml @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + --- version: "3.9" services: diff --git a/docker/oneShot/Dockerfile b/docker/oneShot/Dockerfile index 0505817d54..7b2e7da1ba 100644 --- a/docker/oneShot/Dockerfile +++ b/docker/oneShot/Dockerfile @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + FROM maven:3-amazoncorretto-8 AS builder COPY . /root/IGinX WORKDIR /root/IGinX diff --git a/docker/oneShot/build_and_run_iginx_docker.bat b/docker/oneShot/build_and_run_iginx_docker.bat index 3178b9fdc3..ff90426f1f 100644 --- a/docker/oneShot/build_and_run_iginx_docker.bat +++ b/docker/oneShot/build_and_run_iginx_docker.bat @@ -1 +1,19 @@ +@REM +@REM IGinX - the polystore system with high performance +@REM Copyright (C) Tsinghua University +@REM +@REM This program is free software: you can redistribute it and/or modify +@REM it under the terms of the GNU General Public License as published by +@REM the Free Software Foundation, either version 3 of the License, or +@REM (at your option) any later version. +@REM +@REM This program is distributed in the hope that it will be useful, +@REM but WITHOUT ANY WARRANTY; without even the implied warranty of +@REM MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +@REM GNU General Public License for more details. +@REM +@REM You should have received a copy of the GNU General Public License +@REM along with this program. If not, see . +@REM + docker compose up --build --detach \ No newline at end of file diff --git a/docker/oneShot/build_and_run_iginx_docker.sh b/docker/oneShot/build_and_run_iginx_docker.sh index 3178b9fdc3..544c1868d6 100755 --- a/docker/oneShot/build_and_run_iginx_docker.sh +++ b/docker/oneShot/build_and_run_iginx_docker.sh @@ -1 +1,19 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + docker compose up --build --detach \ No newline at end of file diff --git a/docker/oneShot/docker-compose.yaml b/docker/oneShot/docker-compose.yaml index f2fb8ab293..5ad1d772ad 100644 --- a/docker/oneShot/docker-compose.yaml +++ b/docker/oneShot/docker-compose.yaml @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + --- version: "3.9" services: diff --git a/docker/onlyIginx-parquet/Dockerfile-iginx b/docker/onlyIginx-parquet/Dockerfile-iginx index df05c79099..693478bb74 100644 --- a/docker/onlyIginx-parquet/Dockerfile-iginx +++ b/docker/onlyIginx-parquet/Dockerfile-iginx @@ -4,7 +4,7 @@ WORKDIR /root/iginx RUN mvn clean package -DskipTests -P-format -e FROM openjdk:11-jre-slim -COPY --from=builder /root/iginx/core/target/iginx-core-0.6.0-SNAPSHOT /iginx +COPY --from=builder /root/iginx/core/target/iginx-core-0.8.0-SNAPSHOT /iginx # ports will be cast in run.bat diff --git a/docker/onlyIginx-parquet/build_iginx_docker.bat b/docker/onlyIginx-parquet/build_iginx_docker.bat index fb3c1f43d5..a7724a13e5 100644 --- a/docker/onlyIginx-parquet/build_iginx_docker.bat +++ b/docker/onlyIginx-parquet/build_iginx_docker.bat @@ -1 +1,19 @@ -docker build --file Dockerfile-iginx -t iginx:0.6.0 ../.. \ No newline at end of file +@REM +@REM IGinX - the polystore system with high performance +@REM Copyright (C) Tsinghua University +@REM +@REM This program is free software: you can redistribute it and/or modify +@REM it under the terms of the GNU General Public License as published by +@REM the Free Software Foundation, either version 3 of the License, or +@REM (at your option) any later version. +@REM +@REM This program is distributed in the hope that it will be useful, +@REM but WITHOUT ANY WARRANTY; without even the implied warranty of +@REM MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +@REM GNU General Public License for more details. +@REM +@REM You should have received a copy of the GNU General Public License +@REM along with this program. If not, see . +@REM + +docker build --file Dockerfile-iginx -t iginx:0.8.0-SNAPSHOT ../.. \ No newline at end of file diff --git a/docker/onlyIginx-parquet/build_iginx_docker.sh b/docker/onlyIginx-parquet/build_iginx_docker.sh index 9d66c60f9d..05949b3840 100644 --- a/docker/onlyIginx-parquet/build_iginx_docker.sh +++ b/docker/onlyIginx-parquet/build_iginx_docker.sh @@ -1,2 +1,20 @@ #!/bin/bash -docker build --file Dockerfile-iginx -t iginx:0.6.0 ../.. \ No newline at end of file +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +docker build --file Dockerfile-iginx -t iginx:0.8.0-SNAPSHOT ../.. \ No newline at end of file diff --git a/docker/onlyIginx-parquet/run_iginx_docker.bat b/docker/onlyIginx-parquet/run_iginx_docker.bat index 98aabd3787..ae993b795c 100644 --- a/docker/onlyIginx-parquet/run_iginx_docker.bat +++ b/docker/onlyIginx-parquet/run_iginx_docker.bat @@ -1,3 +1,21 @@ +@REM +@REM IGinX - the polystore system with high performance +@REM Copyright (C) Tsinghua University +@REM +@REM This program is free software: you can redistribute it and/or modify +@REM it under the terms of the GNU General Public License as published by +@REM the Free Software Foundation, either version 3 of the License, or +@REM (at your option) any later version. +@REM +@REM This program is distributed in the hope that it will be useful, +@REM but WITHOUT ANY WARRANTY; without even the implied warranty of +@REM MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +@REM GNU General Public License for more details. +@REM +@REM You should have received a copy of the GNU General Public License +@REM along with this program. If not, see . +@REM + @echo off cd /d %~dp0 setlocal EnableDelayedExpansion @@ -162,7 +180,7 @@ if "!network!" neq "null" ( set "configFileConfig=-v !localConfigFile!:/iginx/conf " @REM -set command=docker run --name="%name%" !network!!localIPConfig!!configFileConfig!--privileged -dit -e host_iginx_port=%hostPort% -p %hostPort%:!port! !engineCast!iginx:0.6.0 +set command=docker run --name="%name%" !network!!localIPConfig!!configFileConfig!--privileged -dit -e host_iginx_port=%hostPort% -p %hostPort%:!port! !engineCast!iginx:0.8.0-SNAPSHOT echo %command% %command% diff --git a/docker/onlyIginx-parquet/run_iginx_docker.sh b/docker/onlyIginx-parquet/run_iginx_docker.sh index 5db6a6e0da..627228d8f6 100644 --- a/docker/onlyIginx-parquet/run_iginx_docker.sh +++ b/docker/onlyIginx-parquet/run_iginx_docker.sh @@ -1,4 +1,22 @@ #!/bin/bash +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + # workdir: $IGINX_HOME/docker/onlyiginx-parquet # work descripition: @@ -121,6 +139,6 @@ fi configFileConfig="-v ${localConfigFile}:/iginx/conf/config.properties " -command="docker run --name=${name} ${localIPConfig}${configFileConfig}--add-host=host.docker.internal:host-gateway ${network}--privileged -dit -e host_iginx_port=${hostPort}${portCastParams} iginx:0.6.0" +command="docker run --name=${name} ${localIPConfig}${configFileConfig}--add-host=host.docker.internal:host-gateway ${network}--privileged -dit -e host_iginx_port=${hostPort}${portCastParams} iginx:0.8.0-SNAPSHOT" echo "RUNNING ${command}" # ${command} \ No newline at end of file diff --git a/docker/onlyIginx/Dockerfile-iginx b/docker/onlyIginx/Dockerfile-iginx index 6c37adca7c..5f1169b266 100644 --- a/docker/onlyIginx/Dockerfile-iginx +++ b/docker/onlyIginx/Dockerfile-iginx @@ -4,7 +4,7 @@ WORKDIR /root/iginx RUN mvn clean package -DskipTests -P-format FROM openjdk:11-jre-slim -COPY --from=builder /root/iginx/core/target/iginx-core-0.6.0-SNAPSHOT /iginx +COPY --from=builder /root/iginx/core/target/iginx-core-0.8.0-SNAPSHOT /iginx # 安装 Python 3.8, pip 并安装所需的 Python 包 RUN apt-get update && \ diff --git a/docker/onlyIginx/build_iginx_docker.bat b/docker/onlyIginx/build_iginx_docker.bat index fb3c1f43d5..a7724a13e5 100644 --- a/docker/onlyIginx/build_iginx_docker.bat +++ b/docker/onlyIginx/build_iginx_docker.bat @@ -1 +1,19 @@ -docker build --file Dockerfile-iginx -t iginx:0.6.0 ../.. \ No newline at end of file +@REM +@REM IGinX - the polystore system with high performance +@REM Copyright (C) Tsinghua University +@REM +@REM This program is free software: you can redistribute it and/or modify +@REM it under the terms of the GNU General Public License as published by +@REM the Free Software Foundation, either version 3 of the License, or +@REM (at your option) any later version. +@REM +@REM This program is distributed in the hope that it will be useful, +@REM but WITHOUT ANY WARRANTY; without even the implied warranty of +@REM MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +@REM GNU General Public License for more details. +@REM +@REM You should have received a copy of the GNU General Public License +@REM along with this program. If not, see . +@REM + +docker build --file Dockerfile-iginx -t iginx:0.8.0-SNAPSHOT ../.. \ No newline at end of file diff --git a/docker/onlyIginx/build_iginx_docker.sh b/docker/onlyIginx/build_iginx_docker.sh index fb3c1f43d5..3dbd72181e 100755 --- a/docker/onlyIginx/build_iginx_docker.sh +++ b/docker/onlyIginx/build_iginx_docker.sh @@ -1 +1,19 @@ -docker build --file Dockerfile-iginx -t iginx:0.6.0 ../.. \ No newline at end of file +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +docker build --file Dockerfile-iginx -t iginx:0.8.0-SNAPSHOT ../.. \ No newline at end of file diff --git a/docker/onlyIginx/run_iginx_docker.bat b/docker/onlyIginx/run_iginx_docker.bat index 6e275635fd..183ab516a0 100644 --- a/docker/onlyIginx/run_iginx_docker.bat +++ b/docker/onlyIginx/run_iginx_docker.bat @@ -1,3 +1,21 @@ +@REM +@REM IGinX - the polystore system with high performance +@REM Copyright (C) Tsinghua University +@REM +@REM This program is free software: you can redistribute it and/or modify +@REM it under the terms of the GNU General Public License as published by +@REM the Free Software Foundation, either version 3 of the License, or +@REM (at your option) any later version. +@REM +@REM This program is distributed in the hope that it will be useful, +@REM but WITHOUT ANY WARRANTY; without even the implied warranty of +@REM MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +@REM GNU General Public License for more details. +@REM +@REM You should have received a copy of the GNU General Public License +@REM along with this program. If not, see . +@REM + @echo off set "current_dir=%CD%" @REM 将路径中的单反斜线替换为双反斜线 @@ -11,4 +29,4 @@ set ip=%1 set name=%2 set port=%3 mkdir -p logs/docker_logs -docker run --name="%name%" --privileged -dit --net docker-cluster-iginx --ip %ip% --add-host=host.docker.internal:host-gateway -v %logdir%:/iginx/logs/ -p %port%:6888 iginx:0.6.0 \ No newline at end of file +docker run --name="%name%" --privileged -dit --net docker-cluster-iginx --ip %ip% --add-host=host.docker.internal:host-gateway -v %logdir%:/iginx/logs/ -p %port%:6888 iginx:0.8.0-SNAPSHOT \ No newline at end of file diff --git a/docker/onlyIginx/run_iginx_docker.sh b/docker/onlyIginx/run_iginx_docker.sh index c22cdebe2f..2c689d0688 100755 --- a/docker/onlyIginx/run_iginx_docker.sh +++ b/docker/onlyIginx/run_iginx_docker.sh @@ -1,6 +1,24 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + ip=$1 name=$2 port=$3 logdir="$(pwd)/../../logs/docker_logs" mkdir -p $logdir -docker run --name="${name}" --privileged -dit --net docker-cluster-iginx --ip ${ip} --add-host=host.docker.internal:host-gateway -v ${logdir}:/iginx/logs/ -p ${port}:6888 iginx:0.6.0 \ No newline at end of file +docker run --name="${name}" --privileged -dit --net docker-cluster-iginx --ip ${ip} --add-host=host.docker.internal:host-gateway -v ${logdir}:/iginx/logs/ -p ${port}:6888 iginx:0.8.0-SNAPSHOT \ No newline at end of file diff --git a/docs/pdf/SQLManual.pdf b/docs/pdf/SQLManual.pdf index b7901523bd..4c8f6d5502 100644 Binary files a/docs/pdf/SQLManual.pdf and b/docs/pdf/SQLManual.pdf differ diff --git a/docs/pdf/userManualC.pdf b/docs/pdf/userManualC.pdf index 5b1981cd88..df3c52fad9 100644 Binary files a/docs/pdf/userManualC.pdf and b/docs/pdf/userManualC.pdf differ diff --git a/docs/quickStarts/IGinXByDocker-EnglishVersion.md b/docs/quickStarts/IGinXByDocker-EnglishVersion.md index d72b72f56d..3a948729e8 100644 --- a/docs/quickStarts/IGinXByDocker-EnglishVersion.md +++ b/docs/quickStarts/IGinXByDocker-EnglishVersion.md @@ -255,7 +255,7 @@ The following words are displayed to indicate that the image was built successfu => [builder 2/4] COPY . /root/iginx 1.3s => [builder 3/4] WORKDIR /root/iginx 0.1s => [builder 4/4] RUN mvn clean package -DskipTests -P-format 876.3s -=> [stage-1 2/2] COPY --from=builder /root/iginx/core/target/iginx-core-0.6.0-SNAPSHOT /iginx 0.2s +=> [stage-1 2/2] COPY --from=builder /root/iginx/core/target/iginx-core-0.8.0-SNAPSHOT /iginx 0.2s => exporting to image 0.5s => => exporting layers 0.5s => => writing image sha256:e738348598c9db601dbf39c7a8ca9e1396c5ff51769afeb0fe3da12e2fdcd73a 0.0s diff --git a/docs/quickStarts/IGinXByDocker.md b/docs/quickStarts/IGinXByDocker.md index 2fbf30eb82..1f559a9785 100644 --- a/docs/quickStarts/IGinXByDocker.md +++ b/docs/quickStarts/IGinXByDocker.md @@ -254,7 +254,7 @@ $ ./build_iginx_docker.sh => [builder 2/4] COPY . /root/iginx 1.3s => [builder 3/4] WORKDIR /root/iginx 0.1s => [builder 4/4] RUN mvn clean package -DskipTests -P-format 876.3s -=> [stage-1 2/2] COPY --from=builder /root/iginx/core/target/iginx-core-0.6.0-SNAPSHOT /iginx 0.2s +=> [stage-1 2/2] COPY --from=builder /root/iginx/core/target/iginx-core-0.8.0-SNAPSHOT /iginx 0.2s => exporting to image 0.5s => => exporting layers 0.5s => => writing image sha256:e738348598c9db601dbf39c7a8ca9e1396c5ff51769afeb0fe3da12e2fdcd73a 0.0s diff --git a/docs/quickStarts/IGinXBySource-EnglishVersion.md b/docs/quickStarts/IGinXBySource-EnglishVersion.md index 083c52eca1..0591557544 100644 --- a/docs/quickStarts/IGinXBySource-EnglishVersion.md +++ b/docs/quickStarts/IGinXBySource-EnglishVersion.md @@ -157,8 +157,8 @@ The specific installation method is as follows: ```shell $ cd ~ -$ wget https://mirrors.bfsu.edu.cn/apache/iotdb/0.12.0/apache-iotdb-0.12.0-server-bin.zip -$ unzip apache-iotdb-0.12.0-server-bin.zip +$ wget https://github.com/IGinX-THU/IGinX-resources/raw/main/resources/apache-iotdb-0.12.6-server-bin.zip +$ unzip apache-iotdb-0.12.6-server-bin.zip ``` ### IGinX Installation @@ -180,7 +180,7 @@ The following words are displayed, indicating that the IGinX build is successful ```shell [INFO] ------------------------------------------------------------------------ -[INFO] Reactor Summary for IGinX 0.6.0-SNAPSHOT: +[INFO] Reactor Summary for IGinX 0.8.0-SNAPSHOT: [INFO] [INFO] IGinX .............................................. SUCCESS [ 20.674 s] [INFO] IGinX Thrift ....................................... SUCCESS [01:18 min] @@ -217,7 +217,7 @@ First of all, you need to launch IoTDB. ```shell $ cd ~ -$ cd apache-iotdb-0.12.0-server-bin/ +$ cd apache-iotdb-0.12.6-server-bin/ $ ./sbin/start-server.sh ``` @@ -424,17 +424,17 @@ Below is a short tutorial on how to use it. ### RPC Interface -Since the IGinX 0.5.1 version has not been released to the Maven central repository, if you want to use it, you need to manually install it to the local Maven repository. +Since the IGinX jars have not been released to the Maven central repository, if you want to use it, you need to manually install it to the local Maven repository. The specific installation method is as follows: ```shell -# download iginx 0.4 release version source code package -$ wget https://github.com/IGinX-THU/IGinX/archive/refs/tags/release/v0.5.1.tar.gz +# download the newest IGinX release version source code package +$ wget https://github.com/IGinX-THU/IGinX/archive/refs/tags/v0.7.0.zip # Unzip the source package -$ tar -zxvf v0.5.1.tar.gz +$ tar -zxvf v0.7.0.tar.gz # go to the main project's directory -$ cd IGinX-rc-v0.5.1 +$ cd IGinX-rc-v0.7.0 # Install to local Maven repository $ mvn clean install -DskipTests ``` @@ -445,7 +445,7 @@ Specifically, when using it, you only need to introduce the following dependenci cn.edu.tsinghua iginx-core - 0.6.0-SNAPSHOT + 0.7.0 ``` diff --git a/docs/quickStarts/IGinXBySource.md b/docs/quickStarts/IGinXBySource.md index 0849bd2ab3..396389bb03 100644 --- a/docs/quickStarts/IGinXBySource.md +++ b/docs/quickStarts/IGinXBySource.md @@ -161,8 +161,8 @@ IoTDB 是 Apache 推出的时序数据库,具体安装方式如下: ```shell $ cd ~ -$ wget https://mirrors.bfsu.edu.cn/apache/iotdb/0.12.0/apache-iotdb-0.12.0-server-bin.zip -$ unzip apache-iotdb-0.12.0-server-bin.zip +$ wget https://github.com/IGinX-THU/IGinX-resources/raw/main/resources/apache-iotdb-0.12.6-server-bin.zip +$ unzip apache-iotdb-0.12.6-server-bin.zip ``` ### IGinX 安装 @@ -180,7 +180,7 @@ $ mvn clean install -Dmaven.test.skip=true ```shell [INFO] ------------------------------------------------------------------------ -[INFO] Reactor Summary for IGinX 0.6.0-SNAPSHOT: +[INFO] Reactor Summary for IGinX 0.8.0-SNAPSHOT: [INFO] [INFO] IGinX .............................................. SUCCESS [ 20.674 s] [INFO] IGinX Thrift ....................................... SUCCESS [01:18 min] @@ -217,7 +217,7 @@ $ mvn clean install -Dmaven.test.skip=true ```shell $ cd ~ -$ cd apache-iotdb-0.12.0-server-bin/ +$ cd apache-iotdb-0.12.6-server-bin/ $ ./sbin/start-server.sh ``` @@ -257,7 +257,7 @@ Starting zookeeper ... STARTED ```shell $ cd ~ -$ cd IGinX/core/target/iginx-core-0.6.0-SNAPSHOT +$ cd IGinX/core/target/iginx-core-0.8.0-SNAPSHOT $ chmod +x sbin/start_iginx.sh # 为启动脚本添加启动权限 $ ./sbin/start_iginx.sh ``` @@ -421,15 +421,15 @@ RPC 接口最常见的用法。 下面是一个简短的使用教程。 -由于目前 IGinX 0.5.1 版本还未发布到 maven 中央仓库,因此如需使用的话,需要手动安装到本地的 maven 仓库。具体安装方式如下: +由于目前 IGinX的jar包还未发布到 maven 中央仓库,因此如需使用的话,需要手动安装到本地的 maven 仓库。具体安装方式如下: ```shell -# 下载 IGinX 0.5.1 release 版本源码包 -$ wget https://github.com/IGinX-THU/IGinX/archive/refs/tags/release/v0.5.1.tar.gz +# 下载 IGinX 最新release 版本源码包 +$ wget https://github.com/IGinX-THU/IGinX/archive/refs/tags/v0.7.0.zip # 解压源码包 -$ tar -zxvf v0.5.1.tar.gz +$ tar -zxvf v0.7.0.tar.gz # 进入项目主目录 -$ cd IGinX-rc-v0.5.1 +$ cd IGinX-rc-v0.7.0 # 安装到本地 maven 仓库 $ mvn clean install -DskipTests ``` @@ -440,7 +440,7 @@ $ mvn clean install -DskipTests cn.edu.tsinghua iginx-core - 0.6.0-SNAPSHOT + 0.7.0 ``` diff --git a/docs/quickStarts/IGinXCluster-EnglishVersion.md b/docs/quickStarts/IGinXCluster-EnglishVersion.md index 8496bbb1b6..6980407747 100644 --- a/docs/quickStarts/IGinXCluster-EnglishVersion.md +++ b/docs/quickStarts/IGinXCluster-EnglishVersion.md @@ -91,13 +91,13 @@ The specific installation method is as follows: ```shell $ cd ~ -$ wget https://mirrors.bfsu.edu.cn/apache/iotdb/0.12.0/apache-iotdb-0.12.0-server-bin.zip -$ unzip apache-iotdb-0.12.0-server-bin.zip +$ wget https://github.com/IGinX-THU/IGinX-resources/raw/main/resources/apache-iotdb-0.12.6-server-bin.zip +$ unzip apache-iotdb-0.12.6-server-bin.zip ``` ### IGinX Installation -Go directly to the [IGinX project](https://github.com/IGinX-THU/IGinX) and download the [IGinX project release package](https://github.com/IGinX-THU/IGinX/releases/download/release%2Fv0.5.1/IGinX-release-v0.5.1-bin.tar.gz). That's it. +Go directly to the [IGinX project](https://github.com/IGinX-THU/IGinX) and download the [IGinX project release package](https://github.com/IGinX-THU/IGinX/releases/download/v0.7.0/IGinX-Server-0.7.0.tar.gz). That's it. ## Launch @@ -117,7 +117,7 @@ Start the first instance: ```shell $ cd ~ -$ cd apache-iotdb-0.12.0-server-bin/ +$ cd apache-iotdb-0.12.6-server-bin/ $ ./sbin/start-server.sh # start instance one 127.0.0.1:6667 ``` @@ -337,15 +337,15 @@ In addition to the RESTful interface, IGinX also provides RPC data access interf Below is a short tutorial on how to use it. -Since the IGinX 0.5.1 version has not been released to the maven central repository, if you want to use it, you need to manually install it to the local maven repository. The specific installation method is as follows: +Since the IGinX jars have not been released to the maven central repository, if you want to use it, you need to manually install it to the local maven repository. The specific installation method is as follows: ```shell -# Download IGinX 0.5.1 version source package -$ wget https://github.com/IGinX-THU/IGinX/archive/refs/tags/release/v0.5.1.tar.gz +# Download the newest IGinX version source package +$ wget https://github.com/IGinX-THU/IGinX/archive/refs/tags/v0.7.0.tar.gz # Unzip the source package -$ tar -zxvf v0.5.1.tar.gz +$ tar -zxvf v0.7.0.tar.gz # Enter the project's main directory -$ cd IGinX-release-v0.5.1 +$ cd IGinX-release-v0.7.0 # Install to local maven repository $ mvn clean install -DskipTests ``` @@ -356,7 +356,7 @@ Only when you are using it, you need to introduce the following dependencies in cn.edu.tsinghua iginx-core - 0.6.0-SNAPSHOT + 0.7.0 ``` diff --git a/docs/quickStarts/IGinXCluster.md b/docs/quickStarts/IGinXCluster.md index eb3788c68c..d73bd2d20c 100644 --- a/docs/quickStarts/IGinXCluster.md +++ b/docs/quickStarts/IGinXCluster.md @@ -88,19 +88,19 @@ IoTDB 是 Apache 推出的时序数据库,具体安装方式如下: ```shell $ cd ~ -$ wget https://mirrors.bfsu.edu.cn/apache/iotdb/0.12.0/apache-iotdb-0.12.0-server-bin.zip -$ unzip apache-iotdb-0.12.0-server-bin.zip +$ wget https://github.com/IGinX-THU/IGinX-resources/raw/main/resources/apache-iotdb-0.12.6-server-bin.zip +$ unzip apache-iotdb-0.12.6-server-bin.zip ``` ### IGinX 安装 -直接访问 [IGinX 项目](https://github.com/IGinX-THU/IGinX)下载 [IGinX 项目发布包](https://github.com/IGinX-THU/IGinX/releases/download/release%2Fv0.5.1/IGinX-release-v0.5.1-bin.tar.gz) +直接访问 [IGinX 项目](https://github.com/IGinX-THU/IGinX)下载 [IGinX 项目发布包](https://github.com/IGinX-THU/IGinX/releases/download/v0.7.0/IGinX-Server-0.7.0.tar.gz) 即可。 ```shell $ cd ~ -$ wget https://github.com/IGinX-THU/IGinX/releases/download/release%2Fv0.5.1/IGinX-release-v0.5.1-bin.tar.gz -$ tar -zxvf IGinX-release-v0.5.1-bin.tar.gz +$ wget https://github.com/IGinX-THU/IGinX/releases/download/v0.7.0/IGinX-Server-0.7.0.tar.gz +$ tar -zxvf IGinX-Server-0.7.0.tar.gz ``` ## 启动 @@ -121,7 +121,7 @@ rpc_port=6667 ```shell $ cd ~ -$ cd apache-iotdb-0.12.0-server-bin/ +$ cd apache-iotdb-0.12.6-server-bin/ $ ./sbin/start-server.sh # 启动实例一 127.0.0.1: 6667 ``` @@ -344,15 +344,15 @@ RPC 接口最常见的用法。 下面是一个简短的使用教程。 -由于目前 IGinX 0.5.1 版本还未发布到 maven 中央仓库,因此如需使用的话,需要手动安装到本地的 maven 仓库。具体安装方式如下: +由于目前 IGinX jar包还未发布到 maven 中央仓库,因此如需使用的话,需要手动安装到本地的 maven 仓库。具体安装方式如下: ```shell -# 下载 IGinX 0.5.1 release 版本源码包 -$ wget https://github.com/IGinX-THU/IGinX/archive/refs/tags/release/v0.5.1.tar.gz +# 下载 IGinX 最新release 版本源码包 +$ wget https://github.com/IGinX-THU/IGinX/archive/refs/tags/v0.7.0.tar.gz # 解压源码包 -$ tar -zxvf v0.5.1.tar.gz +$ tar -zxvf v0.7.0.tar.gz # 进入项目主目录 -$ cd IginX-release-v0.5.1 +$ cd IginX-release-v0.7.0 # 安装到本地 maven 仓库 $ mvn clean install -DskipTests ``` @@ -363,7 +363,7 @@ $ mvn clean install -DskipTests cn.edu.tsinghua iginx-core - 0.6.0-SNAPSHOT + 0.7.0 ``` diff --git a/docs/quickStarts/IGinXInOneShot-EnglishVersion.md b/docs/quickStarts/IGinXInOneShot-EnglishVersion.md index 3151c4f49f..ec4c4784e4 100644 --- a/docs/quickStarts/IGinXInOneShot-EnglishVersion.md +++ b/docs/quickStarts/IGinXInOneShot-EnglishVersion.md @@ -55,15 +55,15 @@ IGinX is the main part of the system, and the installation package can be downlo ```shell $ cd ~ -$ wget https://github.com/IGinX-THU/IGinX/releases/download/release%2Fv0.5.1/IGinX-FastDeploy-v0.5.1-bin.tar.gz -$ tar -xzvf IGinX-FastDeploy-v0.5.1-bin.tar.gz +$ wget https://github.com/IGinX-THU/IGinX/releases/download/v0.7.0/IGinX-FastDeploy-0.7.0.tar.gz +$ tar -xzvf IGinX-FastDeploy-0.7.0.tar.gz ``` ## Launch ```shell $ cd ~ -$ cd IGinX-FastDeploy-v0.5.0-bin +$ cd IGinX-FastDeploy-0.7.0 $ chmod +x ./runIginxOn1Host.sh $ ./runIginxOn1Host.sh ``` @@ -251,17 +251,17 @@ Below is a short tutorial on how to use it. ### RPC Interface -Since the IGinX 0.5.1 version has not been released to the Maven central repository, if you want to use it, you need to manually install it to the local Maven repository. +Since the IGinX jars have not been released to the Maven central repository, if you want to use it, you need to manually install it to the local Maven repository. The specific installation method is as follows: ```shell -# download iginx 0.4 release version source code package -$ wget https://github.com/IGinX-THU/IGinX/archive/refs/tags/release/v0.5.1.tar.gz +# download the newest IGinX release version source code package +$ wget https://github.com/IGinX-THU/IGinX/archive/refs/tags/v0.7.0.tar.gz # Unzip the source package -$ tar -zxvf v0.5.1.tar.gz +$ tar -zxvf v0.7.0.tar.gz # go to the main project's directory -$ cd IGinX-release-v0.5.1 +$ cd IGinX-release-v0.7.0 # Install to local Maven repository $ mvn clean install -DskipTests ``` @@ -272,7 +272,7 @@ Specifically, when using it, you only need to introduce the following dependenci cn.edu.tsinghua iginx-core - 0.6.0-SNAPSHOT + 0.7.0 ``` diff --git a/docs/quickStarts/IGinXInOneShot.md b/docs/quickStarts/IGinXInOneShot.md index fb39310c8a..91b1a4424b 100644 --- a/docs/quickStarts/IGinXInOneShot.md +++ b/docs/quickStarts/IGinXInOneShot.md @@ -54,15 +54,15 @@ IGinX 为系统的主体部分,通过一键启动安装包 ```shell $ cd ~ -$ wget https://github.com/IGinX-THU/IGinX/releases/download/release%2Fv0.5.1/IGinX-FastDeploy-v0.5.1-bin.tar.gz -$ tar -xzvf IGinX-FastDeploy-v0.5.1-bin.tar.gz +$ wget https://github.com/IGinX-THU/IGinX/releases/download/v0.7.0/IGinX-FastDeploy-0.7.0.tar.gz +$ tar -xzvf IGinX-FastDeploy-v0.7.0-bin.tar.gz ``` ## 启动 ```shell $ cd ~ -$ cd IGinX-FastDeploy-v0.5.0-bin +$ cd IGinX-FastDeploy-0.7.0 $ chmod +x ./runIginxOn1Host.sh $ ./runIginxOn1Host.sh ``` @@ -247,15 +247,15 @@ RPC 接口最常见的用法。 下面是一个简短的使用教程。 -由于目前 IGinX 0.5.1 版本还未发布到 maven 中央仓库,因此如需使用的话,需要手动安装到本地的 maven 仓库。具体安装方式如下: +由于目前 IGinX jar包还未发布到 maven 中央仓库,因此如需使用的话,需要手动安装到本地的 maven 仓库。具体安装方式如下: ```shell -# 下载 IGinX 0.5.1 release 版本源码包 -$ wget https://github.com/IGinX-THU/IGinX/archive/refs/tags/release/v0.5.1.tar.gz +# 下载 IGinX 最新release 版本源码包 +$ wget https://github.com/IGinX-THU/IGinX/archive/refs/tags/v0.7.0.zip # 解压源码包 -$ tar -zxvf v0.5.1.tar.gz +$ tar -zxvf v0.7.0.tar.gz # 进入项目主目录 -$ cd IGinX-release-v0.5.1 +$ cd IGinX-release-v0.7.0 # 安装到本地 maven 仓库 $ mvn clean install -DskipTests ``` @@ -266,7 +266,7 @@ $ mvn clean install -DskipTests cn.edu.tsinghua iginx-core - 0.6.0-SNAPSHOT + 0.7.0 ``` diff --git a/docs/quickStarts/IGinXManual-EnglishVersion.md b/docs/quickStarts/IGinXManual-EnglishVersion.md index 56b3ebfbfd..d742838af2 100644 --- a/docs/quickStarts/IGinXManual-EnglishVersion.md +++ b/docs/quickStarts/IGinXManual-EnglishVersion.md @@ -136,18 +136,18 @@ The specific installation method is as follows: ```shell $ cd ~ -$ wget https://mirrors.bfsu.edu.cn/apache/iotdb/0.12.0/apache-iotdb-0.12.0-server-bin.zip -$ unzip apache-iotdb-0.12.0-server-bin.zip +$ wget https://github.com/IGinX-THU/IGinX-resources/raw/main/resources/apache-iotdb-0.12.6-server-bin.zip +$ unzip apache-iotdb-0.12.6-server-bin.zip ``` ### Download the binary executables -Go directly to the [IGinX project](https://github.com/IGinX-THU/IGinX) and download the [IGinX project release package](https://github.com/IGinX-THU/IGinX/releases/download/release%2Fv0.5.1/IGinX-release-v0.5.1-bin.tar.gz). +Go directly to the [IGinX project](https://github.com/IGinX-THU/IGinX) and download the [IGinX project release package](https://github.com/IGinX-THU/IGinX/releases/download/v0.7.0/IGinX-Server-0.7.0.tar.gz). ```shell $ cd ~ -$ wget https://github.com/IGinX-THU/IGinX/releases/download/release%2Fv0.5.1/IGinX-release-v0.5.1-bin.tar.gz -$ tar -xzvf IGinX-release-v0.5.1-bin.tar.gz +$ wget https://github.com/IGinX-THU/IGinX/releases/download/v0.7.0/IGinX-Server-0.7.0.tar.gz +$ tar -xzvf IGinX-Server-0.7.0.tar.gz ``` ### Compilation with source code @@ -173,7 +173,7 @@ First of all, you need to launch IoTDB. ```shell $ cd ~ -$ cd apache-iotdb-0.12.0-server-bin/ +$ cd apache-iotdb-0.12.6-server-bin/ $ ./sbin/start-server.sh ``` @@ -191,7 +191,7 @@ The following display of words means the IoTDB installation was successful: #### Launch ZooKeeper -If you are taking a 0.2.0 binary installation package, or if you designate Zookeeper as the metadata management storage backend in the configuration file, you need to launch ZooKeeper. Otherwise, **skip this step entirely**. +If you are taking a binary installation package, or if you designate Zookeeper as the metadata management storage backend in the configuration file, you need to launch ZooKeeper. Otherwise, **skip this step entirely**. ```shell $ cd ~ @@ -213,7 +213,7 @@ Using the release package to launch ```shell $ cd ~ -$ cd IGinX-release-v0.5.1-bin +$ cd IGinX-Server-0.7.0 $ chmod +x startIginX.sh # enable permissions for startup scripts $ ./startIginX.sh ``` @@ -241,7 +241,7 @@ Here is an example of starting two instances with ports 6667 and 7667, respectiv ```shell $ cd ~ -$ cd apache-iotdb-0.12.0-server-bin/ +$ cd apache-iotdb-0.12.6-server-bin/ $ ./sbin/start-server.sh # 启动实例一 127.0.0.1: 6667 ``` @@ -484,15 +484,15 @@ In addition to the RESTful interface, IGinX also provides RPC data access interf Below is a short tutorial on how to use it. -Since the IGinX 0.5.1 version has not been released to the maven central repository, if you want to use it, you need to manually install it to the local maven repository. The specific installation method is as follows: +Since the IGinX jars have not been released to the maven central repository, if you want to use it, you need to manually install it to the local maven repository. The specific installation method is as follows: ```shell -# Download iginx 0.2 rc version source package -$ wget https://github.com/IGinX-THU/IGinX/archive/refs/tags/release/v0.5.1.tar.gz +# Download the newest IGinX version source package +$ wget https://github.com/IGinX-THU/IGinX/archive/refs/tags/v0.7.0.tar.gz # Unzip the source package -$ tar -zxvf v0.5.1.tar.gz +$ tar -zxvf v0.7.0.tar.gz # Enter the project main directory -$ cd IGinX-rc-v0.5.1 +$ cd IGinX-v0.7.0 # Install to local maven repository $ mvn clean install -DskipTests ``` @@ -503,7 +503,7 @@ Only when you are using it, you need to introduce the following dependencies in cn.edu.tsinghua iginx-core - 0.6.0-SNAPSHOT + 0.7.0 ``` @@ -639,7 +639,7 @@ For the full version of the code, please refer to: https://github.com/IGinX-THU/ cn.edu.tsinghua iginx-session - 0.5.1 + 0.7.0 diff --git a/docs/quickStarts/IGinXManual.md b/docs/quickStarts/IGinXManual.md index 12eeab3b7d..0ed5859982 100644 --- a/docs/quickStarts/IGinXManual.md +++ b/docs/quickStarts/IGinXManual.md @@ -133,19 +133,19 @@ IoTDB 是 Apache 推出的时序数据库,具体安装方式如下: ```shell $ cd ~ -$ wget https://mirrors.bfsu.edu.cn/apache/iotdb/0.12.0/apache-iotdb-0.12.0-server-bin.zip -$ unzip apache-iotdb-0.12.0-server-bin.zip +$ wget https://github.com/IGinX-THU/IGinX-resources/raw/main/resources/apache-iotdb-0.12.6-server-bin.zip +$ unzip apache-iotdb-0.12.6-server-bin.zip ``` ### 下载二进制可执行文件 -直接访问 [IGinX 项目](https://github.com/IGinX-THU/IGinX)下载 [IGinX 项目发布包](https://github.com/IGinX-THU/IGinX/releases/download/release%2Fv0.5.1/IGinX-release-v0.5.1-bin.tar.gz) +直接访问 [IGinX 项目](https://github.com/IGinX-THU/IGinX)下载 [IGinX 项目发布包](https://github.com/IGinX-THU/IGinX/releases/download/v0.7.0/IGinX-Server-0.7.0.tar.gz) 即可 ```shell $ cd ~ -$ wget https://github.com/IGinX-THU/IGinX/releases/download/release%2Fv0.5.1/IGinX-release-v0.5.1-bin.tar.gz -$ tar -zxvf IGinX-release-v0.5.1-bin.tar.gz +$ wget https://github.com/IGinX-THU/IGinX/releases/download/v0.7.0/IGinX-Server-0.7.0.tar.gz +$ tar -zxvf IGinX-Server-0.7.0.tar.gz ``` ### 使用源码编译 @@ -171,7 +171,7 @@ $ mvn clean install -Dmaven.test.skip=true ```shell $ cd ~ -$ cd apache-iotdb-0.12.0-server-bin/ +$ cd apache-iotdb-0.12.6-server-bin/ $ ./sbin/start-server.sh ``` @@ -189,7 +189,7 @@ $ ./sbin/start-server.sh #### 启动 ZooKeeper -如果您采取的是 0.5.1 的二进制安装包,或者在配置文件中指定 ZooKeeper 为元数据管理存储后端,需要启动ZooKeeper。否则,**直接跳过此步骤** +如果您采取的是IGinX的快速部署安装包,或者在配置文件中指定 ZooKeeper 为元数据管理存储后端,需要启动ZooKeeper。否则,**直接跳过此步骤** ```shell $ cd ~ @@ -211,7 +211,7 @@ Starting zookeeper ... STARTED ```shell $ cd ~ -$ cd IGinX-release-v0.5.1-bin +$ cd IGinX-Server-0.7.0 $ chmod +x startIginX.sh # 为启动脚本添加启动权限 $ ./startIginX.sh ``` @@ -239,7 +239,7 @@ IGinX 元数据存储管理现在有 ZooKeeper 存储、写本地文件两种方 ```shell $ cd ~ -$ cd apache-iotdb-0.12.0-server-bin/ +$ cd apache-iotdb-0.12.6-server-bin/ $ ./sbin/start-server.sh # 启动实例一 127.0.0.1: 6667 ``` @@ -485,15 +485,15 @@ RPC 接口最常见的用法。 下面是一个简短的使用教程。 -由于目前 IGinX 0.5.1 版本还未发布到 maven 中央仓库,因此如需使用的话,需要手动安装到本地的 maven 仓库。具体安装方式如下: +由于目前 IGinX jar包还未发布到 maven 中央仓库,因此如需使用的话,需要手动安装到本地的 maven 仓库。具体安装方式如下: ```shell -# 下载 IGinX 0.5.1 release 版本源码包 -$ wget https://github.com/IGinX-THU/IGinX/archive/refs/tags/release/v0.5.1.tar.gz +# 下载 IGinX 最新release 版本源码包 +$ wget https://github.com/IGinX-THU/IGinX/archive/refs/tags/v0.7.0.tar.gz # 解压源码包 -$ tar -zxvf v0.5.1.tar.gz +$ tar -zxvf v0.7.0.tar.gz # 进入项目主目录 -$ cd IGinX-release-v0.5.1 +$ cd IGinX-release-v0.7.0 # 安装到本地 maven 仓库 $ mvn clean install -DskipTests ``` @@ -504,7 +504,7 @@ $ mvn clean install -DskipTests cn.edu.tsinghua iginx-core - 0.6.0-SNAPSHOT + 0.7.0 ``` @@ -641,7 +641,7 @@ session.closeSession(); cn.edu.tsinghua iginx-session - 0.5.1 + 0.7.0 diff --git a/docs/quickStarts/IGinXZeppelin-EnglishVersion.md b/docs/quickStarts/IGinXZeppelin-EnglishVersion.md index b1d34754e1..fd75aba78b 100644 --- a/docs/quickStarts/IGinXZeppelin-EnglishVersion.md +++ b/docs/quickStarts/IGinXZeppelin-EnglishVersion.md @@ -10,7 +10,7 @@ Navigate to the IGinX directory and execute the following command to build the I mvn clean package -DskipTests -P get-jar-with-dependencies ``` -Upon successful compilation, you will find the `zeppelin-iginx-0.6.0-SNAPSHOT-jar-with-dependencies.jar` package in the `IGinX/zeppelin-interpreter/target/` directory. We will need this package for the next steps of deploying Zeppelin. +Upon successful compilation, you will find the `zeppelin-iginx-0.8.0-SNAPSHOT-jar-with-dependencies.jar` package in the `IGinX/zeppelin-interpreter/target/` directory. We will need this package for the next steps of deploying Zeppelin. ## Deploying Zeppelin @@ -56,7 +56,7 @@ export JAVA_HOME= #### Integrating IGinX Zeppelin Interpreter -In the `zeppelin-0.10.1-bin-netinst/interpreter/` folder, create a new folder named `IGinX`. Place the compiled `zeppelin-iginx-0.6.0-SNAPSHOT-jar-with-dependencies.jar` package inside this folder. +In the `zeppelin-0.10.1-bin-netinst/interpreter/` folder, create a new folder named `IGinX`. Place the compiled `zeppelin-iginx-0.8.0-SNAPSHOT-jar-with-dependencies.jar` package inside this folder. #### Starting IGinX @@ -86,7 +86,7 @@ Before deploying Zeppelin, start IGinX. Prepare a folder to place the IGinX Zeppelin Interpreter. For example, let's name the folder `zeppelin-interpreter` with the absolute path `~/code/zeppelin-interpreter/`. -Place the `zeppelin-iginx-0.6.0-SNAPSHOT-jar-with-dependencies.jar` package inside this prepared `zeppelin-interpreter` folder. +Place the `zeppelin-iginx-0.8.0-SNAPSHOT-jar-with-dependencies.jar` package inside this prepared `zeppelin-interpreter` folder. #### Starting Docker Container Using Commands diff --git a/docs/quickStarts/IGinXZeppelin.md b/docs/quickStarts/IGinXZeppelin.md index 542f93d0ad..17a7eb7968 100644 --- a/docs/quickStarts/IGinXZeppelin.md +++ b/docs/quickStarts/IGinXZeppelin.md @@ -10,7 +10,7 @@ mvn clean package -DskipTests -P get-jar-with-dependencies ``` -构建成功后,在`IGinX/zeppelin-interpreter/target/`路径下找到`zeppelin-iginx-0.6.0-SNAPSHOT-jar-with-dependencies.jar`包。 +构建成功后,在`IGinX/zeppelin-interpreter/target/`路径下找到`zeppelin-iginx-0.8.0-SNAPSHOT-jar-with-dependencies.jar`包。 在下一步部署Zeppelin时我们需要用到这个包。 @@ -60,7 +60,7 @@ export JAVA_HOME= #### 接入IGinX Zeppelin Interpreter -在`zeppelin-0.10.1-bin-netinst/interpreter/`文件夹下新建一个文件夹`IGinX`,将构建好的`zeppelin-iginx-0.6.0-SNAPSHOT-jar-with-dependencies.jar`包放入其中即可。 +在`zeppelin-0.10.1-bin-netinst/interpreter/`文件夹下新建一个文件夹`IGinX`,将构建好的`zeppelin-iginx-0.8.0-SNAPSHOT-jar-with-dependencies.jar`包放入其中即可。 #### 启动IGinX @@ -90,7 +90,7 @@ export JAVA_HOME= 我们需要准备一个文件夹,用于放置IGinX Zeppelin Interpreter。例如我们准备一个文件夹名为`zeppelin-interpreter`,其绝对路径为`~/code/zeppelin-interpreter/`。 -将`zeppelin-iginx-0.6.0-SNAPSHOT-jar-with-dependencies.jar`包放入我们准备好的`zeppelin-interpreter`文件夹内即可。 +将`zeppelin-iginx-0.8.0-SNAPSHOT-jar-with-dependencies.jar`包放入我们准备好的`zeppelin-interpreter`文件夹内即可。 #### 使用命令启动Docker容器 diff --git a/docs/quickStarts_Parquet/IGinXByDocker-EnglishVersion.md b/docs/quickStarts_Parquet/IGinXByDocker-EnglishVersion.md index 555f039d90..d9b067d89a 100644 --- a/docs/quickStarts_Parquet/IGinXByDocker-EnglishVersion.md +++ b/docs/quickStarts_Parquet/IGinXByDocker-EnglishVersion.md @@ -250,7 +250,7 @@ The following words are displayed to indicate that the image was built successfu => [builder 2/4] COPY . /root/iginx 1.3s => [builder 3/4] WORKDIR /root/iginx 0.1s => [builder 4/4] RUN mvn clean package -DskipTests -P passFormat 876.3s -=> [stage-1 2/2] COPY --from=builder /root/iginx/core/target/iginx-core-0.6.0-SNAPSHOT /iginx 0.2s +=> [stage-1 2/2] COPY --from=builder /root/iginx/core/target/iginx-core-0.8.0-SNAPSHOT /iginx 0.2s => exporting to image 0.5s => => exporting layers 0.5s => => writing image sha256:e738348598c9db601dbf39c7a8ca9e1396c5ff51769afeb0fe3da12e2fdcd73a 0.0s diff --git a/docs/quickStarts_Parquet/IGinXByDocker.md b/docs/quickStarts_Parquet/IGinXByDocker.md index e8c0dffa31..e8551666d2 100644 --- a/docs/quickStarts_Parquet/IGinXByDocker.md +++ b/docs/quickStarts_Parquet/IGinXByDocker.md @@ -249,7 +249,7 @@ $ ./build_iginx_docker.sh => [builder 2/4] COPY . /root/iginx 1.3s => [builder 3/4] WORKDIR /root/iginx 0.1s => [builder 4/4] RUN mvn clean package -DskipTests -P passFormat 876.3s -=> [stage-1 2/2] COPY --from=builder /root/iginx/core/target/iginx-core-0.6.0-SNAPSHOT /iginx 0.2s +=> [stage-1 2/2] COPY --from=builder /root/iginx/core/target/iginx-core-0.8.0-SNAPSHOT /iginx 0.2s => exporting to image 0.5s => => exporting layers 0.5s => => writing image sha256:e738348598c9db601dbf39c7a8ca9e1396c5ff51769afeb0fe3da12e2fdcd73a 0.0s diff --git a/docs/quickStarts_Parquet/IGinXBySource-EnglishVersion.md b/docs/quickStarts_Parquet/IGinXBySource-EnglishVersion.md index 90266f51b1..bd1dcd0c3a 100644 --- a/docs/quickStarts_Parquet/IGinXBySource-EnglishVersion.md +++ b/docs/quickStarts_Parquet/IGinXBySource-EnglishVersion.md @@ -168,7 +168,7 @@ The following words are displayed, indicating that the IGinX build is successful ```shell [INFO] ------------------------------------------------------------------------ -[INFO] Reactor Summary for IGinX 0.6.0-SNAPSHOT: +[INFO] Reactor Summary for IGinX 0.8.0-SNAPSHOT: [INFO] [INFO] IGinX .............................................. SUCCESS [ 20.674 s] [INFO] IGinX Thrift ....................................... SUCCESS [01:18 min] @@ -396,17 +396,17 @@ Below is a short tutorial on how to use it. ### RPC Interface -Since the IGinX 0.5.1 version has not been released to the Maven central repository, if you want to use it, you need to manually install it to the local Maven repository. +Since the IGinX jars have not been released to the Maven central repository, if you want to use it, you need to manually install it to the local Maven repository. The specific installation method is as follows: ```shell -# download iginx 0.4 release version source code package -$ wget https://github.com/IGinX-THU/IGinX/archive/refs/tags/release/v0.5.1.tar.gz +# download the newest iginx release version source code package +$ wget https://github.com/IGinX-THU/IGinX/releases/download/v0.7.0/IGinX-FastDeploy-0.7.0.tar.gz # Unzip the source package -$ tar -zxvf v0.5.1.tar.gz +$ tar -zxvf v0.7.0.tar.gz # go to the main project's directory -$ cd IGinX-rc-v0.5.1 +$ cd IGinX-rc-v0.7.0 # Install to local Maven repository $ mvn clean install -DskipTests ``` @@ -417,7 +417,7 @@ Specifically, when using it, you only need to introduce the following dependenci cn.edu.tsinghua iginx-core - 0.6.0-SNAPSHOT + 0.7.0 ``` diff --git a/docs/quickStarts_Parquet/IGinXBySource.md b/docs/quickStarts_Parquet/IGinXBySource.md index 67e272ff85..f7fc887b6f 100644 --- a/docs/quickStarts_Parquet/IGinXBySource.md +++ b/docs/quickStarts_Parquet/IGinXBySource.md @@ -170,7 +170,7 @@ $ mvn clean install -Dmaven.test.skip=true ```shell [INFO] ------------------------------------------------------------------------ -[INFO] Reactor Summary for IGinX 0.6.0-SNAPSHOT: +[INFO] Reactor Summary for IGinX 0.8.0-SNAPSHOT: [INFO] [INFO] IGinX .............................................. SUCCESS [ 20.674 s] [INFO] IGinX Thrift ....................................... SUCCESS [01:18 min] @@ -232,7 +232,7 @@ storageEngineList=127.0.0.1#6667#parquet#dir=parquetData#has_data=false#is_read_ ```shell $ cd ~ -$ cd IGinX/core/target/iginx-core-0.6.0-SNAPSHOT +$ cd IGinX/core/target/iginx-core-0.8.0-SNAPSHOT $ chmod +x sbin/start_iginx.sh # 为启动脚本添加启动权限 $ ./sbin/start_iginx.sh ``` @@ -395,15 +395,15 @@ RPC 接口最常见的用法。 下面是一个简短的使用教程。 -由于目前 IGinX 0.5.1 版本还未发布到 maven 中央仓库,因此如需使用的话,需要手动安装到本地的 maven 仓库。具体安装方式如下: +由于目前 IGinX jar包还未发布到 maven 中央仓库,因此如需使用的话,需要手动安装到本地的 maven 仓库。具体安装方式如下: ```shell -# 下载 IGinX 0.5.1 release 版本源码包 -$ wget https://github.com/IGinX-THU/IGinX/archive/refs/tags/release/v0.5.1.tar.gz +# 下载 IGinX 最新release 版本源码包 +$ wget https://github.com/IGinX-THU/IGinX/archive/refs/tags/v0.7.0.tar.gz # 解压源码包 -$ tar -zxvf v0.5.1.tar.gz +$ tar -zxvf v0.7.0.tar.gz # 进入项目主目录 -$ cd IGinX-rc-v0.5.1 +$ cd IGinX-rc-v0.7.0 # 安装到本地 maven 仓库 $ mvn clean install -DskipTests ``` @@ -414,7 +414,7 @@ $ mvn clean install -DskipTests cn.edu.tsinghua iginx-core - 0.6.0-SNAPSHOT + 0.7.0 ``` diff --git a/docs/quickStarts_Parquet/IGinXCluster-EnglishVersion.md b/docs/quickStarts_Parquet/IGinXCluster-EnglishVersion.md index 12bbcd0158..9aa22cded6 100644 --- a/docs/quickStarts_Parquet/IGinXCluster-EnglishVersion.md +++ b/docs/quickStarts_Parquet/IGinXCluster-EnglishVersion.md @@ -85,12 +85,12 @@ dataDir=data ### IGinX Installation -Go directly to the [IGinX project](https://github.com/IGinX-THU/IGinX) and download the [IGinX project release package](https://github.com/IGinX-THU/IGinX/releases/download/release%2Fv0.5.1/IGinX-release-v0.5.1-bin.tar.gz). That's it. +Go directly to the [IGinX project](https://github.com/IGinX-THU/IGinX) and download the [IGinX project release package](https://github.com/IGinX-THU/IGinX/releases/download/v0.7.0/IGinX-Server-0.7.0.tar.gz). That's it. ```shell $ cd ~ -$ wget https://github.com/IGinX-THU/IGinX/releases/download/release%2Fv0.5.1/IGinX-release-v0.5.1-bin.tar.gz -$ tar -zxvf IGinX-release-v0.5.1-bin.tar.gz +$ wget https://github.com/IGinX-THU/IGinX/releases/download/v0.7.0/IGinX-Server-0.7.0.tar.gz +$ tar -zxvf IGinX-Server-0.7.0.tar.gz ``` ## Launch @@ -303,15 +303,15 @@ In addition to the RESTful interface, IGinX also provides RPC data access interf Below is a short tutorial on how to use it. -Since the IGinX 0.5.1 version has not been released to the maven central repository, if you want to use it, you need to manually install it to the local maven repository. The specific installation method is as follows: +Since the IGinX jars have not been released to the maven central repository, if you want to use it, you need to manually install it to the local maven repository. The specific installation method is as follows: ```shell -# Download IGinX 0.5.1 version source package -$ wget https://github.com/IGinX-THU/IGinX/archive/refs/tags/release/v0.5.1.tar.gz +# Download the newest IGinX version source package +$ wget https://github.com/IGinX-THU/IGinX/archive/refs/tags/v0.7.0.tar.gz # Unzip the source package -$ tar -zxvf v0.5.1.tar.gz +$ tar -zxvf v0.7.0.tar.gz # Enter the project's main directory -$ cd IGinX-release-v0.5.1 +$ cd IGinX-release-v0.7.0 # Install to local maven repository $ mvn clean install -DskipTests ``` @@ -322,7 +322,7 @@ Only when you are using it, you need to introduce the following dependencies in cn.edu.tsinghua iginx-core - 0.6.0-SNAPSHOT + 0.7.0 ``` diff --git a/docs/quickStarts_Parquet/IGinXCluster.md b/docs/quickStarts_Parquet/IGinXCluster.md index 57713afa2a..1fbdc61e04 100644 --- a/docs/quickStarts_Parquet/IGinXCluster.md +++ b/docs/quickStarts_Parquet/IGinXCluster.md @@ -84,13 +84,13 @@ dataDir=data ### IGinX 安装 -直接访问 [IGinX 项目](https://github.com/IGinX-THU/IGinX)下载 [IGinX 项目发布包](https://github.com/IGinX-THU/IGinX/releases/download/release%2Fv0.5.1/IGinX-release-v0.5.1-bin.tar.gz) +直接访问 [IGinX 项目](https://github.com/IGinX-THU/IGinX)下载 [IGinX 项目发布包](https://github.com/IGinX-THU/IGinX/releases/download/v0.7.0/IGinX-Server-0.7.0.tar.gz) 即可。 ```shell $ cd ~ -$ wget https://github.com/IGinX-THU/IGinX/releases/download/release%2Fv0.5.1/IGinX-release-v0.5.1-bin.tar.gz -$ tar -zxvf IGinX-release-v0.5.1-bin.tar.gz +$ wget https://github.com/IGinX-THU/IGinX/releases/download/v0.7.0/IGinX-Server-0.7.0.tar.gz +$ tar -zxvf IGinX-Server-0.7.0.tar.gz ``` ## 启动 @@ -306,15 +306,15 @@ RPC 接口最常见的用法。 下面是一个简短的使用教程。 -由于目前 IGinX 0.5.1 版本还未发布到 maven 中央仓库,因此如需使用的话,需要手动安装到本地的 maven 仓库。具体安装方式如下: +由于目前 IGinX jar包还未发布到 maven 中央仓库,因此如需使用的话,需要手动安装到本地的 maven 仓库。具体安装方式如下: ```shell -# 下载 IGinX 0.5.1 release 版本源码包 -$ wget https://github.com/IGinX-THU/IGinX/archive/refs/tags/release/v0.5.1.tar.gz +# 下载 IGinX 最新release 版本源码包 +$ wget https://github.com/IGinX-THU/IGinX/archive/refs/tags/v0.7.0.tar.gz # 解压源码包 -$ tar -zxvf v0.5.1.tar.gz +$ tar -zxvf v0.7.0.tar.gz # 进入项目主目录 -$ cd IginX-release-v0.5.1 +$ cd IginX-release-v0.7.0 # 安装到本地 maven 仓库 $ mvn clean install -DskipTests ``` @@ -325,7 +325,7 @@ $ mvn clean install -DskipTests cn.edu.tsinghua iginx-core - 0.6.0-SNAPSHOT + 0.7.0 ``` diff --git a/docs/quickStarts_Parquet/IGinXInOneShot-EnglishVersion.md b/docs/quickStarts_Parquet/IGinXInOneShot-EnglishVersion.md index 68954fac37..e027d9d501 100644 --- a/docs/quickStarts_Parquet/IGinXInOneShot-EnglishVersion.md +++ b/docs/quickStarts_Parquet/IGinXInOneShot-EnglishVersion.md @@ -55,15 +55,15 @@ IGinX is the main part of the system, and the installation package can be downlo ```shell $ cd ~ -$ wget https://github.com/IGinX-THU/IGinX/releases/download/release%2Fv0.5.1/IGinX-FastDeploy-v0.5.1-bin.tar.gz -$ tar -xzvf IGinX-FastDeploy-v0.5.1-bin.tar.gz +$ wget https://github.com/IGinX-THU/IGinX/releases/download/v0.7.0/IGinX-FastDeploy-0.7.0.tar.gz +$ tar -xzvf IGinX-FastDeploy-0.7.0.tar.gz ``` ## Launch ```shell $ cd ~ -$ cd IGinX-FastDeploy-v0.5.0-bin +$ cd IGinX-FastDeploy-0.7.0 $ chmod +x ./runIginxOn1Host.sh $ ./runIginxOn1Host.sh ``` @@ -233,17 +233,17 @@ Below is a short tutorial on how to use it. ### RPC Interface -Since the IGinX 0.5.1 version has not been released to the Maven central repository, if you want to use it, you need to manually install it to the local Maven repository. +Since the IGinX jars have not been released to the Maven central repository, if you want to use it, you need to manually install it to the local Maven repository. The specific installation method is as follows: ```shell -# download iginx 0.4 release version source code package -$ wget https://github.com/IGinX-THU/IGinX/archive/refs/tags/release/v0.5.1.tar.gz +# download the newest iginx release version source code package +$ wget https://github.com/IGinX-THU/IGinX/archive/refs/tags/v0.7.0.tar.gz # Unzip the source package -$ tar -zxvf v0.5.1.tar.gz +$ tar -zxvf v0.7.0.tar.gz # go to the main project's directory -$ cd IGinX-release-v0.5.1 +$ cd IGinX-release-v0.7.0 # Install to local Maven repository $ mvn clean install -DskipTests ``` @@ -254,7 +254,7 @@ Specifically, when using it, you only need to introduce the following dependenci cn.edu.tsinghua iginx-core - 0.6.0-SNAPSHOT + 0.7.0 ``` diff --git a/docs/quickStarts_Parquet/IGinXInOneShot.md b/docs/quickStarts_Parquet/IGinXInOneShot.md index ce08c36de7..8549270990 100644 --- a/docs/quickStarts_Parquet/IGinXInOneShot.md +++ b/docs/quickStarts_Parquet/IGinXInOneShot.md @@ -54,15 +54,15 @@ IGinX 为系统的主体部分,通过一键启动安装包 ```shell $ cd ~ -$ wget https://github.com/IGinX-THU/IGinX/releases/download/release%2Fv0.5.1/IGinX-FastDeploy-v0.5.1-bin.tar.gz -$ tar -xzvf IGinX-FastDeploy-v0.5.1-bin.tar.gz +$ wget https://github.com/IGinX-THU/IGinX/releases/download/v0.7.0/IGinX-FastDeploy-0.7.0.tar.gz +$ tar -xzvf IGinX-FastDeploy-0.7.0.tar.gz ``` ## 启动 ```shell $ cd ~ -$ cd IGinX-FastDeploy-v0.5.0-bin +$ cd IGinX-FastDeploy-0.7.0 $ chmod +x ./runIginxOn1Host.sh $ ./runIginxOn1Host.sh ``` @@ -228,15 +228,15 @@ RPC 接口最常见的用法。 下面是一个简短的使用教程。 -由于目前 IGinX 0.5.1 版本还未发布到 maven 中央仓库,因此如需使用的话,需要手动安装到本地的 maven 仓库。具体安装方式如下: +由于目前 IGinX jar包还未发布到 maven 中央仓库,因此如需使用的话,需要手动安装到本地的 maven 仓库。具体安装方式如下: ```shell -# 下载 IGinX 0.5.1 release 版本源码包 -$ wget https://github.com/IGinX-THU/IGinX/archive/refs/tags/release/v0.5.1.tar.gz +# 下载 IGinX 最新release 版本源码包 +$ wget https://github.com/IGinX-THU/IGinX/archive/refs/tags/v0.7.0.tar.gz # 解压源码包 -$ tar -zxvf v0.5.1.tar.gz +$ tar -zxvf v0.7.0.tar.gz # 进入项目主目录 -$ cd IGinX-release-v0.5.1 +$ cd IGinX-release-v0.7.0 # 安装到本地 maven 仓库 $ mvn clean install -DskipTests ``` @@ -247,7 +247,7 @@ $ mvn clean install -DskipTests cn.edu.tsinghua iginx-core - 0.6.0-SNAPSHOT + 0.7.0 ``` diff --git a/example/pom.xml b/example/pom.xml index 44ca4adf64..a3342c7f33 100644 --- a/example/pom.xml +++ b/example/pom.xml @@ -1,4 +1,23 @@ + 4.0.0 @@ -7,15 +26,12 @@ cn.edu.tsinghua iginx ${revision} - ../pom.xml iginx-example IGinX Example - 8 - 8 true diff --git a/example/src/main/java/cn/edu/tsinghua/iginx/session/IginXStreamSessionExample.java b/example/src/main/java/cn/edu/tsinghua/iginx/session/IginXStreamSessionExample.java index dbcc45db57..4982b500a3 100644 --- a/example/src/main/java/cn/edu/tsinghua/iginx/session/IginXStreamSessionExample.java +++ b/example/src/main/java/cn/edu/tsinghua/iginx/session/IginXStreamSessionExample.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.session; import cn.edu.tsinghua.iginx.thrift.DataType; diff --git a/example/src/main/java/cn/edu/tsinghua/iginx/session/InfluxDBSessionExample.java b/example/src/main/java/cn/edu/tsinghua/iginx/session/InfluxDBSessionExample.java index 8c04a18e77..c94b77b068 100644 --- a/example/src/main/java/cn/edu/tsinghua/iginx/session/InfluxDBSessionExample.java +++ b/example/src/main/java/cn/edu/tsinghua/iginx/session/InfluxDBSessionExample.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session; diff --git a/example/src/main/java/cn/edu/tsinghua/iginx/session/IoTDBAfterDilatationExample.java b/example/src/main/java/cn/edu/tsinghua/iginx/session/IoTDBAfterDilatationExample.java index b9f04a6a52..96e45ddaa6 100644 --- a/example/src/main/java/cn/edu/tsinghua/iginx/session/IoTDBAfterDilatationExample.java +++ b/example/src/main/java/cn/edu/tsinghua/iginx/session/IoTDBAfterDilatationExample.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session; diff --git a/example/src/main/java/cn/edu/tsinghua/iginx/session/IoTDBBeforeDilatationExample.java b/example/src/main/java/cn/edu/tsinghua/iginx/session/IoTDBBeforeDilatationExample.java index f248cbd411..48b6755e19 100644 --- a/example/src/main/java/cn/edu/tsinghua/iginx/session/IoTDBBeforeDilatationExample.java +++ b/example/src/main/java/cn/edu/tsinghua/iginx/session/IoTDBBeforeDilatationExample.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session; diff --git a/example/src/main/java/cn/edu/tsinghua/iginx/session/IoTDBSessionExample.java b/example/src/main/java/cn/edu/tsinghua/iginx/session/IoTDBSessionExample.java index 6cc6fb6429..832e37cf0b 100644 --- a/example/src/main/java/cn/edu/tsinghua/iginx/session/IoTDBSessionExample.java +++ b/example/src/main/java/cn/edu/tsinghua/iginx/session/IoTDBSessionExample.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session; diff --git a/example/src/main/java/cn/edu/tsinghua/iginx/session/IoTDBSessionPoolExample.java b/example/src/main/java/cn/edu/tsinghua/iginx/session/IoTDBSessionPoolExample.java index 601e97f206..0cb10d224f 100644 --- a/example/src/main/java/cn/edu/tsinghua/iginx/session/IoTDBSessionPoolExample.java +++ b/example/src/main/java/cn/edu/tsinghua/iginx/session/IoTDBSessionPoolExample.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.session; import cn.edu.tsinghua.iginx.exception.SessionException; diff --git a/example/src/main/java/cn/edu/tsinghua/iginx/session/NewSessionExample.java b/example/src/main/java/cn/edu/tsinghua/iginx/session/NewSessionExample.java index fa94688c5e..d49d83d56b 100644 --- a/example/src/main/java/cn/edu/tsinghua/iginx/session/NewSessionExample.java +++ b/example/src/main/java/cn/edu/tsinghua/iginx/session/NewSessionExample.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session; diff --git a/example/src/main/java/cn/edu/tsinghua/iginx/session/NewSessionNotSupportedQueryExample.java b/example/src/main/java/cn/edu/tsinghua/iginx/session/NewSessionNotSupportedQueryExample.java index 2e5bf69e3d..0031467e13 100644 --- a/example/src/main/java/cn/edu/tsinghua/iginx/session/NewSessionNotSupportedQueryExample.java +++ b/example/src/main/java/cn/edu/tsinghua/iginx/session/NewSessionNotSupportedQueryExample.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session; diff --git a/example/src/main/java/cn/edu/tsinghua/iginx/session/NewSessionNotSupportedWriteExample.java b/example/src/main/java/cn/edu/tsinghua/iginx/session/NewSessionNotSupportedWriteExample.java index 481220bed4..a5d7939b14 100644 --- a/example/src/main/java/cn/edu/tsinghua/iginx/session/NewSessionNotSupportedWriteExample.java +++ b/example/src/main/java/cn/edu/tsinghua/iginx/session/NewSessionNotSupportedWriteExample.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session; diff --git a/example/src/main/java/cn/edu/tsinghua/iginx/session/OpenTSDBSessionExample.java b/example/src/main/java/cn/edu/tsinghua/iginx/session/OpenTSDBSessionExample.java index f1d7c570c8..c24f7f1ca1 100644 --- a/example/src/main/java/cn/edu/tsinghua/iginx/session/OpenTSDBSessionExample.java +++ b/example/src/main/java/cn/edu/tsinghua/iginx/session/OpenTSDBSessionExample.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.session; import cn.edu.tsinghua.iginx.exception.SessionException; diff --git a/example/src/main/java/cn/edu/tsinghua/iginx/session/ParquetServerExample.java b/example/src/main/java/cn/edu/tsinghua/iginx/session/ParquetServerExample.java index ee31bfcae1..63a066c6c8 100644 --- a/example/src/main/java/cn/edu/tsinghua/iginx/session/ParquetServerExample.java +++ b/example/src/main/java/cn/edu/tsinghua/iginx/session/ParquetServerExample.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.session; import cn.edu.tsinghua.iginx.exception.SessionException; diff --git a/example/src/main/java/cn/edu/tsinghua/iginx/session/ParquetSessionExample.java b/example/src/main/java/cn/edu/tsinghua/iginx/session/ParquetSessionExample.java index a4a39846d0..60ccda6bbe 100644 --- a/example/src/main/java/cn/edu/tsinghua/iginx/session/ParquetSessionExample.java +++ b/example/src/main/java/cn/edu/tsinghua/iginx/session/ParquetSessionExample.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.session; import cn.edu.tsinghua.iginx.exception.SessionException; diff --git a/example/src/main/java/cn/edu/tsinghua/iginx/session/PostgreSQLSessionExample.java b/example/src/main/java/cn/edu/tsinghua/iginx/session/PostgreSQLSessionExample.java index fddc870791..058842ffce 100644 --- a/example/src/main/java/cn/edu/tsinghua/iginx/session/PostgreSQLSessionExample.java +++ b/example/src/main/java/cn/edu/tsinghua/iginx/session/PostgreSQLSessionExample.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session; diff --git a/example/src/main/java/cn/edu/tsinghua/iginx/session/SQLSessionExample.java b/example/src/main/java/cn/edu/tsinghua/iginx/session/SQLSessionExample.java index 6b539c6de8..396204fce7 100644 --- a/example/src/main/java/cn/edu/tsinghua/iginx/session/SQLSessionExample.java +++ b/example/src/main/java/cn/edu/tsinghua/iginx/session/SQLSessionExample.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.session; import cn.edu.tsinghua.iginx.exception.SessionException; diff --git a/example/src/main/java/cn/edu/tsinghua/iginx/session/TagKVSessionExample.java b/example/src/main/java/cn/edu/tsinghua/iginx/session/TagKVSessionExample.java index af66050836..3a56d64d6c 100644 --- a/example/src/main/java/cn/edu/tsinghua/iginx/session/TagKVSessionExample.java +++ b/example/src/main/java/cn/edu/tsinghua/iginx/session/TagKVSessionExample.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.session; import cn.edu.tsinghua.iginx.exception.SessionException; diff --git a/example/src/main/java/cn/edu/tsinghua/iginx/session/TimescaleDBSessionExample.java b/example/src/main/java/cn/edu/tsinghua/iginx/session/TimescaleDBSessionExample.java index a09d81cfe9..6e119cbfc0 100644 --- a/example/src/main/java/cn/edu/tsinghua/iginx/session/TimescaleDBSessionExample.java +++ b/example/src/main/java/cn/edu/tsinghua/iginx/session/TimescaleDBSessionExample.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session; diff --git a/example/src/main/java/cn/edu/tsinghua/iginx/session/TransformCompare.java b/example/src/main/java/cn/edu/tsinghua/iginx/session/TransformCompare.java index e8fa6b582f..6b5fa3ca32 100644 --- a/example/src/main/java/cn/edu/tsinghua/iginx/session/TransformCompare.java +++ b/example/src/main/java/cn/edu/tsinghua/iginx/session/TransformCompare.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.session; import cn.edu.tsinghua.iginx.exception.SessionException; diff --git a/example/src/main/java/cn/edu/tsinghua/iginx/session/TransformExample.java b/example/src/main/java/cn/edu/tsinghua/iginx/session/TransformExample.java index 0c0a8df6f9..e40f0ac9eb 100644 --- a/example/src/main/java/cn/edu/tsinghua/iginx/session/TransformExample.java +++ b/example/src/main/java/cn/edu/tsinghua/iginx/session/TransformExample.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.session; import cn.edu.tsinghua.iginx.exception.SessionException; diff --git a/example/src/main/java/cn/edu/tsinghua/iginx/session/UDFCompareToSys.java b/example/src/main/java/cn/edu/tsinghua/iginx/session/UDFCompareToSys.java index 2f46a097f7..c5e480fe40 100644 --- a/example/src/main/java/cn/edu/tsinghua/iginx/session/UDFCompareToSys.java +++ b/example/src/main/java/cn/edu/tsinghua/iginx/session/UDFCompareToSys.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.session; import cn.edu.tsinghua.iginx.exception.SessionException; diff --git a/example/src/main/java/cn/edu/tsinghua/iginx/session/UDFExample.java b/example/src/main/java/cn/edu/tsinghua/iginx/session/UDFExample.java index 24047bb5c1..7c5f1b5515 100644 --- a/example/src/main/java/cn/edu/tsinghua/iginx/session/UDFExample.java +++ b/example/src/main/java/cn/edu/tsinghua/iginx/session/UDFExample.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.session; import cn.edu.tsinghua.iginx.exception.SessionException; diff --git a/example/src/main/resources/TransformJobExample.yaml b/example/src/main/resources/TransformJobExample.yaml index 5ac0245242..acd1abb010 100644 --- a/example/src/main/resources/TransformJobExample.yaml +++ b/example/src/main/resources/TransformJobExample.yaml @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + --- taskList: - taskType: "iginx" diff --git a/example/src/main/resources/transformer_add_one.py b/example/src/main/resources/transformer_add_one.py index 6315786272..4fe45d624a 100644 --- a/example/src/main/resources/transformer_add_one.py +++ b/example/src/main/resources/transformer_add_one.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + import pandas as pd diff --git a/example/src/main/resources/transformer_avg.py b/example/src/main/resources/transformer_avg.py index dbd319c3d9..90e78e8f8b 100644 --- a/example/src/main/resources/transformer_avg.py +++ b/example/src/main/resources/transformer_avg.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + import pandas as pd diff --git a/example/src/main/resources/transformer_count.py b/example/src/main/resources/transformer_count.py index e2ab063333..259d6ef1da 100644 --- a/example/src/main/resources/transformer_count.py +++ b/example/src/main/resources/transformer_count.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + import pandas as pd diff --git a/example/src/main/resources/transformer_max.py b/example/src/main/resources/transformer_max.py index 594ce29a15..5ad89df329 100644 --- a/example/src/main/resources/transformer_max.py +++ b/example/src/main/resources/transformer_max.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + import pandas as pd diff --git a/example/src/main/resources/transformer_min.py b/example/src/main/resources/transformer_min.py index 31003e10e4..8e886ba5a3 100644 --- a/example/src/main/resources/transformer_min.py +++ b/example/src/main/resources/transformer_min.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + import pandas as pd diff --git a/example/src/main/resources/transformer_row_sum.py b/example/src/main/resources/transformer_row_sum.py index fea1fcff92..0f38eabba1 100644 --- a/example/src/main/resources/transformer_row_sum.py +++ b/example/src/main/resources/transformer_row_sum.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + import pandas as pd import numpy as np diff --git a/example/src/main/resources/transformer_sum.py b/example/src/main/resources/transformer_sum.py index 63dea29ddb..e4ef01dd18 100644 --- a/example/src/main/resources/transformer_sum.py +++ b/example/src/main/resources/transformer_sum.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + import pandas as pd diff --git a/example/src/main/resources/udaf_count.py b/example/src/main/resources/udaf_count.py index b8884e7b3c..623278fe94 100644 --- a/example/src/main/resources/udaf_count.py +++ b/example/src/main/resources/udaf_count.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + class UDFCount: def __init__(self): pass diff --git a/example/src/main/resources/udtf_sin.py b/example/src/main/resources/udtf_sin.py index b66329b91c..51cf539c44 100644 --- a/example/src/main/resources/udtf_sin.py +++ b/example/src/main/resources/udtf_sin.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + import math diff --git a/jdbc/pom.xml b/jdbc/pom.xml index 7daf87ceed..63b65b6269 100644 --- a/jdbc/pom.xml +++ b/jdbc/pom.xml @@ -1,4 +1,23 @@ + 4.0.0 @@ -6,17 +25,11 @@ cn.edu.tsinghua iginx ${revision} - ../pom.xml iginx-jdbc IGinX JDBC - - 8 - 8 - - cn.edu.tsinghua diff --git a/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/Config.java b/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/Config.java index 52673440a8..95ee63baf3 100644 --- a/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/Config.java +++ b/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/Config.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.jdbc; public class Config { diff --git a/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/Constant.java b/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/Constant.java index 0c6be18ffd..e307079037 100644 --- a/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/Constant.java +++ b/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/Constant.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.jdbc; public class Constant { diff --git a/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/IginXConnection.java b/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/IginXConnection.java index da87be06cb..e5bf0bf5bd 100644 --- a/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/IginXConnection.java +++ b/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/IginXConnection.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.jdbc; import cn.edu.tsinghua.iginx.exception.SessionException; diff --git a/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/IginXConnectionParams.java b/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/IginXConnectionParams.java index 2d1f281f01..3d8e036c06 100644 --- a/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/IginXConnectionParams.java +++ b/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/IginXConnectionParams.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.jdbc; public class IginXConnectionParams { diff --git a/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/IginXDataSource.java b/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/IginXDataSource.java index e7a17c459d..82c8fc47e0 100644 --- a/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/IginXDataSource.java +++ b/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/IginXDataSource.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.jdbc; import cn.edu.tsinghua.iginx.exception.SessionException; diff --git a/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/IginXDatabaseMetadata.java b/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/IginXDatabaseMetadata.java index 115fd725ab..2f98301fe1 100644 --- a/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/IginXDatabaseMetadata.java +++ b/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/IginXDatabaseMetadata.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.jdbc; import cn.edu.tsinghua.iginx.session.Session; diff --git a/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/IginXDriver.java b/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/IginXDriver.java index b66f9fb876..5b60685ab6 100644 --- a/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/IginXDriver.java +++ b/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/IginXDriver.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.jdbc; import java.sql.*; diff --git a/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/IginXPreparedStatement.java b/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/IginXPreparedStatement.java index a4c3035490..11df856441 100644 --- a/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/IginXPreparedStatement.java +++ b/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/IginXPreparedStatement.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.jdbc; import cn.edu.tsinghua.iginx.session.Session; diff --git a/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/IginXResultMetadata.java b/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/IginXResultMetadata.java index 40a1e20a6f..19cb84b0c4 100644 --- a/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/IginXResultMetadata.java +++ b/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/IginXResultMetadata.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.jdbc; import cn.edu.tsinghua.iginx.thrift.DataType; diff --git a/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/IginXResultSet.java b/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/IginXResultSet.java index 7beb610ad2..0d947d1e0c 100644 --- a/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/IginXResultSet.java +++ b/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/IginXResultSet.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.jdbc; import cn.edu.tsinghua.iginx.constant.GlobalConstant; diff --git a/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/IginXStatement.java b/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/IginXStatement.java index 8b95433f24..0252120ab2 100644 --- a/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/IginXStatement.java +++ b/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/IginXStatement.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.jdbc; import cn.edu.tsinghua.iginx.exception.SessionException; diff --git a/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/IginxUrlException.java b/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/IginxUrlException.java index a83df88b58..26321f3d27 100644 --- a/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/IginxUrlException.java +++ b/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/IginxUrlException.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.jdbc; import java.sql.SQLException; diff --git a/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/Utils.java b/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/Utils.java index e69c1a4cc4..f64653c850 100644 --- a/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/Utils.java +++ b/jdbc/src/main/java/cn/edu/tsinghua/iginx/jdbc/Utils.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.jdbc; import java.sql.Date; diff --git a/jdbc/src/test/java/BatchTest.java b/jdbc/src/test/java/BatchTest.java index 4568232f6d..a967a66fb6 100644 --- a/jdbc/src/test/java/BatchTest.java +++ b/jdbc/src/test/java/BatchTest.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + import cn.edu.tsinghua.iginx.jdbc.IginXStatement; import java.sql.SQLException; import java.util.ArrayList; diff --git a/jdbc/src/test/java/ConnectionParamsTest.java b/jdbc/src/test/java/ConnectionParamsTest.java index 7bcc302e7e..fb898160fa 100644 --- a/jdbc/src/test/java/ConnectionParamsTest.java +++ b/jdbc/src/test/java/ConnectionParamsTest.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + import static org.junit.Assert.assertEquals; import cn.edu.tsinghua.iginx.jdbc.Config; diff --git a/jdbc/src/test/java/ExampleTest.java b/jdbc/src/test/java/ExampleTest.java index 2a28406772..801ce32737 100644 --- a/jdbc/src/test/java/ExampleTest.java +++ b/jdbc/src/test/java/ExampleTest.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + import cn.edu.tsinghua.iginx.jdbc.IginXPreparedStatement; import java.sql.*; import org.apache.commons.lang3.RandomStringUtils; diff --git a/jdbc/src/test/java/PreparedStatementTest.java b/jdbc/src/test/java/PreparedStatementTest.java index 9a54307891..f2bb27cb5f 100644 --- a/jdbc/src/test/java/PreparedStatementTest.java +++ b/jdbc/src/test/java/PreparedStatementTest.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + import cn.edu.tsinghua.iginx.jdbc.IginXPreparedStatement; import java.sql.SQLException; import org.junit.Assert; diff --git a/jdbc/src/test/java/ResultMetadataTest.java b/jdbc/src/test/java/ResultMetadataTest.java index c699a50203..c2daabe903 100644 --- a/jdbc/src/test/java/ResultMetadataTest.java +++ b/jdbc/src/test/java/ResultMetadataTest.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; diff --git a/jdbc/src/test/java/ResultSetTest.java b/jdbc/src/test/java/ResultSetTest.java index f19fd32c1f..70b052602d 100644 --- a/jdbc/src/test/java/ResultSetTest.java +++ b/jdbc/src/test/java/ResultSetTest.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + import cn.edu.tsinghua.iginx.constant.GlobalConstant; import java.sql.ResultSet; import java.sql.SQLException; diff --git a/jdbc/src/test/java/TestUtils.java b/jdbc/src/test/java/TestUtils.java index 499661cebd..74f29b727a 100644 --- a/jdbc/src/test/java/TestUtils.java +++ b/jdbc/src/test/java/TestUtils.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + import cn.edu.tsinghua.iginx.jdbc.IginXResultSet; import cn.edu.tsinghua.iginx.session.SessionExecuteSqlResult; import cn.edu.tsinghua.iginx.thrift.DataType; diff --git a/optimizer/.gitignore b/optimizer/.gitignore new file mode 100644 index 0000000000..5ff6309b71 --- /dev/null +++ b/optimizer/.gitignore @@ -0,0 +1,38 @@ +target/ +!.mvn/wrapper/maven-wrapper.jar +!**/src/main/**/target/ +!**/src/test/**/target/ + +### IntelliJ IDEA ### +.idea/modules.xml +.idea/jarRepositories.xml +.idea/compiler.xml +.idea/libraries/ +*.iws +*.iml +*.ipr + +### Eclipse ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +build/ +!**/src/main/**/build/ +!**/src/test/**/build/ + +### VS Code ### +.vscode/ + +### Mac OS ### +.DS_Store \ No newline at end of file diff --git a/optimizer/pom.xml b/optimizer/pom.xml new file mode 100644 index 0000000000..e604d642cf --- /dev/null +++ b/optimizer/pom.xml @@ -0,0 +1,112 @@ + + + + 4.0.0 + + cn.edu.tsinghua + iginx + ${revision} + ../pom.xml + + iginx-optimizer + IGinX Optimizer + + + true + + + + + cn.edu.tsinghua + iginx-core + provided + + + com.google.auto.service + auto-service-annotations + 1.1.1 + + + + + + + maven-compiler-plugin + 3.13.0 + + + + com.google.auto.service + auto-service + 1.1.1 + + + + + + org.apache.maven.plugins + maven-dependency-plugin + 2.10 + + + copy-dependencies + + copy-dependencies + + package + + ../core/target/iginx-core-${project.version}/lib/ + provided + + + + + + + org.apache.maven.plugins + maven-antrun-plugin + 1.7 + + + copy-native-libraries + + run + + package + + + + + + + + + + + + + + + + + + diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/FilterFragmentOptimizer.java b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/FilterFragmentOptimizer.java similarity index 88% rename from core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/FilterFragmentOptimizer.java rename to optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/FilterFragmentOptimizer.java index cdc222c317..4a06563683 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/FilterFragmentOptimizer.java +++ b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/FilterFragmentOptimizer.java @@ -1,7 +1,24 @@ -package cn.edu.tsinghua.iginx.engine.logical.optimizer; - -import static cn.edu.tsinghua.iginx.metadata.utils.FragmentUtils.keyFromColumnsIntervalToKeyInterval; - +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.logical.optimizer; + +import cn.edu.tsinghua.iginx.engine.logical.optimizer.Optimizer; import cn.edu.tsinghua.iginx.engine.logical.utils.LogicalFilterUtils; import cn.edu.tsinghua.iginx.engine.logical.utils.OperatorUtils; import cn.edu.tsinghua.iginx.engine.shared.Constants; @@ -18,6 +35,7 @@ import cn.edu.tsinghua.iginx.metadata.entity.ColumnsInterval; import cn.edu.tsinghua.iginx.metadata.entity.FragmentMeta; import cn.edu.tsinghua.iginx.metadata.entity.KeyInterval; +import cn.edu.tsinghua.iginx.metadata.utils.FragmentUtils; import cn.edu.tsinghua.iginx.utils.Pair; import java.util.ArrayList; import java.util.List; @@ -92,7 +110,7 @@ private void filterFragmentByTimeRange(Select selectOperator) { Map> fragmentsByColumnsInterval = metaManager.getFragmentMapByColumnsInterval(columnsInterval, true); Pair>, List> pair = - keyFromColumnsIntervalToKeyInterval(fragmentsByColumnsInterval); + FragmentUtils.keyFromColumnsIntervalToKeyInterval(fragmentsByColumnsInterval); Map> fragments = pair.k; List dummyFragments = pair.v; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/FilterPushDownOptimizer.java b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/FilterPushDownOptimizer.java similarity index 89% rename from core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/FilterPushDownOptimizer.java rename to optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/FilterPushDownOptimizer.java index de2680ebc9..a87823fc87 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/FilterPushDownOptimizer.java +++ b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/FilterPushDownOptimizer.java @@ -1,5 +1,24 @@ -package cn.edu.tsinghua.iginx.engine.logical.optimizer; - +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.logical.optimizer; + +import cn.edu.tsinghua.iginx.engine.logical.optimizer.Optimizer; import cn.edu.tsinghua.iginx.engine.logical.utils.LogicalFilterUtils; import cn.edu.tsinghua.iginx.engine.logical.utils.OperatorUtils; import cn.edu.tsinghua.iginx.engine.shared.operator.*; @@ -89,7 +108,7 @@ private void pushDown(Select selectOperator) { getRenameOperator(selectOperator, project, renameList); Filter unrenamedFilter = filter.copy(); for (Operator rename : renameList) { - unrenamedFilter = replacePathByRenameMap(unrenamedFilter, ((Rename) rename).getAliasMap()); + unrenamedFilter = replacePathByRenameMap(unrenamedFilter, ((Rename) rename).getAliasList()); } // the same meta just call once. @@ -245,7 +264,7 @@ private boolean getRenameOperator( return isCorrectRoad; } - private Filter replacePathByRenameMap(Filter filter, Map renameMap) { + private Filter replacePathByRenameMap(Filter filter, List> renameMap) { switch (filter.getType()) { case Or: List orChildren = ((OrFilter) filter).getChildren(); @@ -263,8 +282,8 @@ private Filter replacePathByRenameMap(Filter filter, Map renameM break; case Value: String path = ((ValueFilter) filter).getPath(); - for (Map.Entry entry : renameMap.entrySet()) { - path = replacePathByRenameEntry(path, entry); + for (Pair pair : renameMap) { + path = replacePathByRenameEntry(path, pair); } return new ValueFilter( path, ((ValueFilter) filter).getOp(), ((ValueFilter) filter).getValue()); @@ -272,8 +291,8 @@ private Filter replacePathByRenameMap(Filter filter, Map renameM String pathA = ((PathFilter) filter).getPathA(); String pathB = ((PathFilter) filter).getPathB(); - for (Map.Entry entry : renameMap.entrySet()) { - pathA = replacePathByRenameEntry(pathA, entry); + for (Pair pair : renameMap) { + pathA = replacePathByRenameEntry(pathA, pair); } return new PathFilter(pathA, ((PathFilter) filter).getOp(), pathB); @@ -283,9 +302,9 @@ private Filter replacePathByRenameMap(Filter filter, Map renameM return filter; } - private String replacePathByRenameEntry(String path, Map.Entry entry) { - String nameBeforeRename = entry.getKey(); - String nameAfterRename = entry.getValue(); + private String replacePathByRenameEntry(String path, Pair pair) { + String nameBeforeRename = pair.k; + String nameAfterRename = pair.v; if (path.equals(nameAfterRename)) { return nameBeforeRename; } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/RemoveNotOptimizer.java b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/RemoveNotOptimizer.java similarity index 66% rename from core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/RemoveNotOptimizer.java rename to optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/RemoveNotOptimizer.java index 0456e3f239..5e536fc5db 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/RemoveNotOptimizer.java +++ b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/RemoveNotOptimizer.java @@ -1,5 +1,24 @@ -package cn.edu.tsinghua.iginx.engine.logical.optimizer; +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cn.edu.tsinghua.iginx.logical.optimizer; + +import cn.edu.tsinghua.iginx.engine.logical.optimizer.Optimizer; import cn.edu.tsinghua.iginx.engine.logical.utils.LogicalFilterUtils; import cn.edu.tsinghua.iginx.engine.logical.utils.OperatorUtils; import cn.edu.tsinghua.iginx.engine.shared.operator.*; diff --git a/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/core/Operand.java b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/core/Operand.java new file mode 100644 index 0000000000..90fdf6d8e8 --- /dev/null +++ b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/core/Operand.java @@ -0,0 +1,47 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.logical.optimizer.core; + +import cn.edu.tsinghua.iginx.engine.shared.operator.Operator; +import java.util.List; + +public class Operand { + + private final Class clazz; + + private final List children; + + public static final Operand ANY_OPERAND = new Operand(null, null); + + public Operand(Class clazz, List children) { + this.clazz = clazz; + this.children = children; + } + + public List getChildren() { + return children; + } + + public boolean matches(Operator operator) { + if (this == ANY_OPERAND) { + return true; + } + return clazz.isInstance(operator); + } +} diff --git a/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/core/Planner.java b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/core/Planner.java new file mode 100644 index 0000000000..7f17c1065d --- /dev/null +++ b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/core/Planner.java @@ -0,0 +1,48 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.logical.optimizer.core; + +import cn.edu.tsinghua.iginx.engine.shared.operator.Operator; +import cn.edu.tsinghua.iginx.logical.optimizer.core.iterator.MatchOrder; +import cn.edu.tsinghua.iginx.logical.optimizer.rules.Rule; +import java.util.List; + +public interface Planner { + + // unban a single rule + void unbanRule(Rule rule); + + // unban a set of rules,e.g.PPD + void unbanRuleCollection(List rules); + + // set up the unoptimized query tree and initialize the optimization context + void setRoot(Operator root); + + // set the maximum number of rule matches + void setMatchLimit(int matchLimit); + + // set the maximum time limit for rule matching, unit: ms + void setLimitTime(long limitTime); + + // set the Rule matching order for this query tree, depth-first, leveled etc + void setMatchOrder(MatchOrder order); + + // get the optimized query tree + Operator findBest(); +} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/core/RuleCall.java b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/core/RuleCall.java similarity index 72% rename from core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/core/RuleCall.java rename to optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/core/RuleCall.java index 5d6f300b20..bfc882038e 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/core/RuleCall.java +++ b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/core/RuleCall.java @@ -1,4 +1,22 @@ -package cn.edu.tsinghua.iginx.engine.logical.optimizer.core; +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.logical.optimizer.core; import cn.edu.tsinghua.iginx.engine.shared.operator.BinaryOperator; import cn.edu.tsinghua.iginx.engine.shared.operator.MultipleOperator; @@ -7,6 +25,7 @@ import cn.edu.tsinghua.iginx.engine.shared.operator.type.OperatorType; import cn.edu.tsinghua.iginx.engine.shared.source.OperatorSource; import cn.edu.tsinghua.iginx.engine.shared.source.Source; +import cn.edu.tsinghua.iginx.logical.optimizer.rbo.RuleBasedPlanner; import java.util.List; import java.util.Map; import org.slf4j.Logger; @@ -24,13 +43,17 @@ public abstract class RuleCall { private Object context; + private final RuleBasedPlanner planner; + public RuleCall( Operator matchedRoot, Map parentIndexMap, - Map> childrenIndex) { + Map> childrenIndex, + RuleBasedPlanner planner) { this.matchedRoot = matchedRoot; this.parentIndexMap = parentIndexMap; this.childrenIndex = childrenIndex; + this.planner = planner; } public Operator getMatchedRoot() { @@ -47,7 +70,10 @@ public Map getParentIndexMap() { public void transformTo(Operator newRoot) { Operator parent = parentIndexMap.get(matchedRoot); - assert parent != null; + if (parent == null) { + planner.setRoot(newRoot); + return; + } // replace topology OperatorType parentType = parent.getType(); diff --git a/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/core/iterator/DeepFirstIterator.java b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/core/iterator/DeepFirstIterator.java new file mode 100644 index 0000000000..bb955e4697 --- /dev/null +++ b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/core/iterator/DeepFirstIterator.java @@ -0,0 +1,44 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.logical.optimizer.core.iterator; + +import cn.edu.tsinghua.iginx.engine.shared.operator.Operator; +import cn.edu.tsinghua.iginx.engine.shared.operator.visitor.DeepFirstQueueVisitor; +import java.util.Queue; + +public class DeepFirstIterator implements TreeIterator { + + private final Queue queue; + + public DeepFirstIterator(Operator root) { + DeepFirstQueueVisitor visitor = new DeepFirstQueueVisitor(); + root.accept(visitor); + queue = visitor.getQueue(); + } + + @Override + public boolean hasNext() { + return !queue.isEmpty(); + } + + @Override + public Operator next() { + return queue.poll(); + } +} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/core/iterator/LeveledIterator.java b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/core/iterator/LeveledIterator.java similarity index 69% rename from core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/core/iterator/LeveledIterator.java rename to optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/core/iterator/LeveledIterator.java index 493cefe0c6..4e1cb6cd8f 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/core/iterator/LeveledIterator.java +++ b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/core/iterator/LeveledIterator.java @@ -1,4 +1,22 @@ -package cn.edu.tsinghua.iginx.engine.logical.optimizer.core.iterator; +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.logical.optimizer.core.iterator; import cn.edu.tsinghua.iginx.engine.shared.operator.BinaryOperator; import cn.edu.tsinghua.iginx.engine.shared.operator.MultipleOperator; diff --git a/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/core/iterator/MatchOrder.java b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/core/iterator/MatchOrder.java new file mode 100644 index 0000000000..2484b42354 --- /dev/null +++ b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/core/iterator/MatchOrder.java @@ -0,0 +1,29 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.logical.optimizer.core.iterator; + +public enum MatchOrder { + Leveled, + + ReverseLeveled, + + DeepFirst, + + ReverseDeepFirst +} diff --git a/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/core/iterator/ReverseDeepFirstIterator.java b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/core/iterator/ReverseDeepFirstIterator.java new file mode 100644 index 0000000000..f0d2a43855 --- /dev/null +++ b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/core/iterator/ReverseDeepFirstIterator.java @@ -0,0 +1,48 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.logical.optimizer.core.iterator; + +import cn.edu.tsinghua.iginx.engine.shared.operator.Operator; +import cn.edu.tsinghua.iginx.engine.shared.operator.visitor.DeepFirstQueueVisitor; +import java.util.Deque; +import java.util.LinkedList; +import java.util.Queue; + +public class ReverseDeepFirstIterator implements TreeIterator { + private final Deque stack = new LinkedList<>(); + + public ReverseDeepFirstIterator(Operator root) { + DeepFirstQueueVisitor visitor = new DeepFirstQueueVisitor(); + root.accept(visitor); + Queue forward = visitor.getQueue(); + while (!forward.isEmpty()) { + stack.push(forward.poll()); + } + } + + @Override + public boolean hasNext() { + return !stack.isEmpty(); + } + + @Override + public Operator next() { + return stack.poll(); + } +} diff --git a/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/core/iterator/ReverseLeveledIterator.java b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/core/iterator/ReverseLeveledIterator.java new file mode 100644 index 0000000000..fa107bdab5 --- /dev/null +++ b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/core/iterator/ReverseLeveledIterator.java @@ -0,0 +1,45 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.logical.optimizer.core.iterator; + +import cn.edu.tsinghua.iginx.engine.shared.operator.Operator; +import java.util.Deque; +import java.util.LinkedList; + +public class ReverseLeveledIterator implements TreeIterator { + + private final Deque stack = new LinkedList<>(); + + public ReverseLeveledIterator(Operator root) { + LeveledIterator it = new LeveledIterator(root); + while (it.hasNext()) { + stack.push(it.next()); + } + } + + @Override + public boolean hasNext() { + return !stack.isEmpty(); + } + + @Override + public Operator next() { + return stack.poll(); + } +} diff --git a/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/core/iterator/TreeIterator.java b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/core/iterator/TreeIterator.java new file mode 100644 index 0000000000..ce43a192b0 --- /dev/null +++ b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/core/iterator/TreeIterator.java @@ -0,0 +1,29 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.logical.optimizer.core.iterator; + +import cn.edu.tsinghua.iginx.engine.shared.operator.Operator; +import java.util.Iterator; + +public interface TreeIterator extends Iterator { + + boolean hasNext(); + + Operator next(); +} diff --git a/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rbo/RBORuleCall.java b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rbo/RBORuleCall.java new file mode 100644 index 0000000000..89727fa48a --- /dev/null +++ b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rbo/RBORuleCall.java @@ -0,0 +1,35 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.logical.optimizer.rbo; + +import cn.edu.tsinghua.iginx.engine.shared.operator.Operator; +import cn.edu.tsinghua.iginx.logical.optimizer.core.RuleCall; +import java.util.List; +import java.util.Map; + +public class RBORuleCall extends RuleCall { + + public RBORuleCall( + Operator subRoot, + Map parentIndexMap, + Map> childrenIndex, + RuleBasedPlanner planner) { + super(subRoot, parentIndexMap, childrenIndex, planner); + } +} diff --git a/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rbo/RuleBasedOptimizer.java b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rbo/RuleBasedOptimizer.java new file mode 100644 index 0000000000..c22f303922 --- /dev/null +++ b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rbo/RuleBasedOptimizer.java @@ -0,0 +1,41 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.logical.optimizer.rbo; + +import cn.edu.tsinghua.iginx.engine.logical.optimizer.Optimizer; +import cn.edu.tsinghua.iginx.engine.shared.operator.Operator; +import cn.edu.tsinghua.iginx.logical.optimizer.core.Planner; + +public class RuleBasedOptimizer implements Optimizer { + + private static final class InstanceHolder { + static final RuleBasedOptimizer instance = new RuleBasedOptimizer(); + } + + public static RuleBasedOptimizer getInstance() { + return InstanceHolder.instance; + } + + @Override + public Operator optimize(Operator root) { + Planner planner = new RuleBasedPlanner(); + planner.setRoot(root); + return planner.findBest(); + } +} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rbo/RuleBasedPlanner.java b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rbo/RuleBasedPlanner.java similarity index 69% rename from core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rbo/RuleBasedPlanner.java rename to optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rbo/RuleBasedPlanner.java index 5f90c53d6d..ac700b3753 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rbo/RuleBasedPlanner.java +++ b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rbo/RuleBasedPlanner.java @@ -1,26 +1,44 @@ -package cn.edu.tsinghua.iginx.engine.logical.optimizer.rbo; - -import cn.edu.tsinghua.iginx.engine.logical.optimizer.core.Operand; -import cn.edu.tsinghua.iginx.engine.logical.optimizer.core.Planner; -import cn.edu.tsinghua.iginx.engine.logical.optimizer.core.RuleCall; -import cn.edu.tsinghua.iginx.engine.logical.optimizer.core.iterator.DeepFirstIterator; -import cn.edu.tsinghua.iginx.engine.logical.optimizer.core.iterator.LeveledIterator; -import cn.edu.tsinghua.iginx.engine.logical.optimizer.core.iterator.MatchOrder; -import cn.edu.tsinghua.iginx.engine.logical.optimizer.core.iterator.ReverseDeepFirstIterator; -import cn.edu.tsinghua.iginx.engine.logical.optimizer.core.iterator.ReverseLeveledIterator; -import cn.edu.tsinghua.iginx.engine.logical.optimizer.core.iterator.TreeIterator; -import cn.edu.tsinghua.iginx.engine.logical.optimizer.rules.Rule; -import cn.edu.tsinghua.iginx.engine.logical.optimizer.rules.RuleCollection; -import cn.edu.tsinghua.iginx.engine.logical.optimizer.rules.RuleStrategy; +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.logical.optimizer.rbo; + import cn.edu.tsinghua.iginx.engine.shared.operator.Operator; import cn.edu.tsinghua.iginx.engine.shared.operator.visitor.IndexVisitor; +import cn.edu.tsinghua.iginx.logical.optimizer.core.Operand; +import cn.edu.tsinghua.iginx.logical.optimizer.core.Planner; +import cn.edu.tsinghua.iginx.logical.optimizer.core.RuleCall; +import cn.edu.tsinghua.iginx.logical.optimizer.core.iterator.DeepFirstIterator; +import cn.edu.tsinghua.iginx.logical.optimizer.core.iterator.LeveledIterator; +import cn.edu.tsinghua.iginx.logical.optimizer.core.iterator.MatchOrder; +import cn.edu.tsinghua.iginx.logical.optimizer.core.iterator.ReverseDeepFirstIterator; +import cn.edu.tsinghua.iginx.logical.optimizer.core.iterator.ReverseLeveledIterator; +import cn.edu.tsinghua.iginx.logical.optimizer.core.iterator.TreeIterator; +import cn.edu.tsinghua.iginx.logical.optimizer.rules.Rule; +import cn.edu.tsinghua.iginx.logical.optimizer.rules.RuleCollection; +import cn.edu.tsinghua.iginx.logical.optimizer.rules.RuleStrategy; import java.util.*; public class RuleBasedPlanner implements Planner { private Operator root; - private final RuleCollection ruleCollection = RuleCollection.getInstance(); + private final RuleCollection ruleCollection = RuleCollection.INSTANCE; private Map parentIndex; private Map> childrenIndex; @@ -91,7 +109,7 @@ public Operator findBest() { continue; } - RuleCall ruleCall = new RBORuleCall(op, parentIndex, childrenIndex); + RuleCall ruleCall = new RBORuleCall(op, parentIndex, childrenIndex, this); if (!rule.matches(ruleCall)) { continue; } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/ColumnPruningRule.java b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/ColumnPruningRule.java similarity index 85% rename from core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/ColumnPruningRule.java rename to optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/ColumnPruningRule.java index fab9fa0b41..5497103508 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/ColumnPruningRule.java +++ b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/ColumnPruningRule.java @@ -1,10 +1,25 @@ -package cn.edu.tsinghua.iginx.engine.logical.optimizer.rules; +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.logical.optimizer.rules; -import static cn.edu.tsinghua.iginx.engine.logical.utils.OperatorUtils.covers; -import static cn.edu.tsinghua.iginx.engine.logical.utils.PathUtils.*; - -import cn.edu.tsinghua.iginx.engine.logical.optimizer.core.RuleCall; import cn.edu.tsinghua.iginx.engine.logical.utils.OperatorUtils; +import cn.edu.tsinghua.iginx.engine.logical.utils.PathUtils; import cn.edu.tsinghua.iginx.engine.physical.memory.execute.utils.ExprUtils; import cn.edu.tsinghua.iginx.engine.physical.memory.execute.utils.FilterUtils; import cn.edu.tsinghua.iginx.engine.shared.Constants; @@ -20,25 +35,21 @@ import cn.edu.tsinghua.iginx.engine.shared.operator.type.OperatorType; import cn.edu.tsinghua.iginx.engine.shared.source.FragmentSource; import cn.edu.tsinghua.iginx.engine.shared.source.OperatorSource; +import cn.edu.tsinghua.iginx.logical.optimizer.core.RuleCall; import cn.edu.tsinghua.iginx.utils.Pair; import cn.edu.tsinghua.iginx.utils.StringUtils; +import com.google.auto.service.AutoService; import java.util.*; +import java.util.regex.Pattern; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@AutoService(Rule.class) public class ColumnPruningRule extends Rule { private static final Logger LOGGER = LoggerFactory.getLogger(ColumnPruningRule.class); - private static final class InstanceHolder { - static final ColumnPruningRule INSTANCE = new ColumnPruningRule(); - } - - public static ColumnPruningRule getInstance() { - return InstanceHolder.INSTANCE; - } - - protected ColumnPruningRule() { + public ColumnPruningRule() { /* * we want to match the topology like: * Any @@ -78,7 +89,7 @@ private void collectColumns( } else if (operator.getType() == OperatorType.Reorder) { Reorder reorder = (Reorder) operator; if (columns.isEmpty() - || (hasWildCard(columns) && checkCoverage(columns, reorder.getPatterns()))) { + || (hasWildCard(columns) && PathUtils.checkCoverage(columns, reorder.getPatterns()))) { columns.clear(); columns.addAll(reorder.getPatterns()); } else { @@ -89,7 +100,7 @@ private void collectColumns( } else if (operator.getType() == OperatorType.Project) { Project project = (Project) operator; if (columns.isEmpty() - || hasWildCard(columns) && checkCoverage(columns, project.getPatterns())) { + || hasWildCard(columns) && PathUtils.checkCoverage(columns, project.getPatterns())) { columns.clear(); columns.addAll(project.getPatterns()); tagFilter = project.getTagFilter(); @@ -105,8 +116,9 @@ private void collectColumns( } } else if (operator.getType() == OperatorType.Rename) { Rename rename = (Rename) operator; - Map aliasMap = rename.getAliasMap(); - columns = new HashSet<>(recoverRenamedPatterns(aliasMap, new ArrayList<>(columns))); + List> aliasList = rename.getAliasList(); + columns = + new HashSet<>(PathUtils.recoverRenamedPatterns(aliasList, new ArrayList<>(columns))); } else if (operator.getType() == OperatorType.GroupBy) { GroupBy groupBy = (GroupBy) operator; @@ -250,8 +262,10 @@ private void collectColumns( if (PrefixA != null && PrefixB != null) { for (String column : columns) { if (column.contains("*") - && (!covers(PrefixA + ".*", column) && covers(column, PrefixA + ".*")) - && (!covers(PrefixB + ".*", column) && covers(column, PrefixB + ".*"))) { + && (!OperatorUtils.covers(PrefixA + ".*", column) + && OperatorUtils.covers(column, PrefixA + ".*")) + && (!OperatorUtils.covers(PrefixB + ".*", column) + && OperatorUtils.covers(column, PrefixB + ".*"))) { /* 如果现有的列中包含通配符*,这个通配符会受到当前Join算子的限制,不再能下推,再推下去会导致取的列过多 例如最顶上Project的test.*,但是JOIN算子左右两侧是test.a.*和test.b.*,这是的test.*实际上取的列是test.a.*和test.b.*的并集 @@ -270,9 +284,11 @@ private void collectColumns( break; } - if (covers(PrefixA + ".*", column) || covers(column, PrefixA + ".*")) { + if (OperatorUtils.covers(PrefixA + ".*", column) + || OperatorUtils.covers(column, PrefixA + ".*")) { leftColumns.add(column); - } else if (covers(PrefixB + ".*", column) || covers(column, PrefixB + ".*")) { + } else if (OperatorUtils.covers(PrefixB + ".*", column) + || OperatorUtils.covers(column, PrefixB + ".*")) { rightColumns.add(column); } else { // 如果没有匹配的话,可能是连续JOIN的情况,即SELECT t1.*, t2.*, t3.* FROM t1 JOIN t2 JOIN t3;(这里省略了ON条件) @@ -290,13 +306,10 @@ private void collectColumns( } else if (OperatorType.isSetOperator(operator.getType())) { Pair, List> orderPair = getSetOperatorOrder(operator); List leftOrder = orderPair.getK(), rightOrder = orderPair.getV(); - if (hasPatternOperator(visitedOperators)) { + // 如果左侧将要替换的columns和右侧order有*的话,我们无法确定具体*对应的列数,因此不裁剪,继续往下递归 + if (hasPatternOperator(visitedOperators) + && !(columns.isEmpty() || hasWildCard(columns) || hasWildCard(rightOrder))) { // SetOperator要求左右两侧的列数相同,我们只对左侧进行列裁剪,然后右侧相应减少对应的列(有顺序要求) - // 如果左侧将要替换的columns和右侧order有*的话,我们无法确定具体*对应的列数,不能进行列裁剪,直接return - if (hasWildCard(columns) || hasWildCard(rightOrder)) { - return; - } - List newLeftOrder = getOrderListFromColumns(columns, leftOrder); List newRightOrder = new ArrayList<>(); // 去除右侧,索引超出左侧的列 @@ -310,6 +323,29 @@ private void collectColumns( leftColumns.addAll(leftOrder); rightColumns.addAll(rightOrder); } + } else if (operator.getType().equals(OperatorType.Join)) { + List leftPatterns = + OperatorUtils.getPatternFromOperatorChildren( + ((OperatorSource) ((BinaryOperator) operator).getSourceA()).getOperator(), + new ArrayList<>()); + List rightPatterns = + OperatorUtils.getPatternFromOperatorChildren( + ((OperatorSource) ((BinaryOperator) operator).getSourceB()).getOperator(), + new ArrayList<>()); + for (String column : columns) { + for (String leftPattern : leftPatterns) { + if (Pattern.matches(StringUtils.reformatPath(leftPattern), column)) { + leftColumns.add(column); + break; + } + } + for (String rightPattern : rightPatterns) { + if (Pattern.matches(StringUtils.reformatPath(rightPattern), column)) { + rightColumns.add(column); + break; + } + } + } } else { leftColumns.addAll(columns); rightColumns.addAll(columns); @@ -499,7 +535,7 @@ private static Set getNewColumns( for (String newColumn : newColumnList) { boolean covered = false; for (String c : columns) { - if (covers(c, newColumn)) { + if (OperatorUtils.covers(c, newColumn)) { covered = true; break; } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/ConstantPropagationRule.java b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/ConstantPropagationRule.java similarity index 92% rename from core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/ConstantPropagationRule.java rename to optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/ConstantPropagationRule.java index b2c42866df..26a2afb2b9 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/ConstantPropagationRule.java +++ b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/ConstantPropagationRule.java @@ -1,6 +1,23 @@ -package cn.edu.tsinghua.iginx.engine.logical.optimizer.rules; +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.logical.optimizer.rules; -import cn.edu.tsinghua.iginx.engine.logical.optimizer.core.RuleCall; import cn.edu.tsinghua.iginx.engine.logical.utils.LogicalFilterUtils; import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; import cn.edu.tsinghua.iginx.engine.physical.memory.execute.utils.ExprUtils; @@ -10,26 +27,21 @@ import cn.edu.tsinghua.iginx.engine.shared.expr.*; import cn.edu.tsinghua.iginx.engine.shared.operator.Select; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.*; +import cn.edu.tsinghua.iginx.logical.optimizer.core.RuleCall; import cn.edu.tsinghua.iginx.utils.Pair; +import com.google.auto.service.AutoService; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.logging.Logger; +@AutoService(Rule.class) public class ConstantPropagationRule extends Rule { private static final Logger LOGGER = Logger.getLogger(ConstantPropagationRule.class.getName()); - private static final class InstanceHolder { - static final ConstantPropagationRule INSTANCE = new ConstantPropagationRule(); - } - - public static ConstantPropagationRule getInstance() { - return InstanceHolder.INSTANCE; - } - - protected ConstantPropagationRule() { + public ConstantPropagationRule() { /* * we want to match the topology like: * SELECT diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FilterConstantFoldingRule.java b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/FilterConstantFoldingRule.java similarity index 70% rename from core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FilterConstantFoldingRule.java rename to optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/FilterConstantFoldingRule.java index fd625e7415..9de5f1f110 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FilterConstantFoldingRule.java +++ b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/FilterConstantFoldingRule.java @@ -1,27 +1,37 @@ -package cn.edu.tsinghua.iginx.engine.logical.optimizer.rules; +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.logical.optimizer.rules; -import cn.edu.tsinghua.iginx.engine.logical.optimizer.core.RuleCall; import cn.edu.tsinghua.iginx.engine.logical.utils.LogicalFilterUtils; import cn.edu.tsinghua.iginx.engine.physical.memory.execute.utils.ExprUtils; import cn.edu.tsinghua.iginx.engine.shared.expr.*; import cn.edu.tsinghua.iginx.engine.shared.operator.Select; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.ExprFilter; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; +import cn.edu.tsinghua.iginx.logical.optimizer.core.RuleCall; +import com.google.auto.service.AutoService; import java.util.List; -import java.util.logging.Logger; +@AutoService(Rule.class) public class FilterConstantFoldingRule extends Rule { - private static final Logger LOGGER = Logger.getLogger(FilterConstantFoldingRule.class.getName()); - - private static final class InstanceHolder { - static final FilterConstantFoldingRule INSTANCE = new FilterConstantFoldingRule(); - } - - public static FilterConstantFoldingRule getInstance() { - return InstanceHolder.INSTANCE; - } - protected FilterConstantFoldingRule() { + public FilterConstantFoldingRule() { /* * we want to match the topology like: * SELECT diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FilterJoinTransposeRule.java b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/FilterJoinTransposeRule.java similarity index 63% rename from core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FilterJoinTransposeRule.java rename to optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/FilterJoinTransposeRule.java index 79c815cb03..b6fde4e1ee 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FilterJoinTransposeRule.java +++ b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/FilterJoinTransposeRule.java @@ -1,24 +1,35 @@ -package cn.edu.tsinghua.iginx.engine.logical.optimizer.rules; +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.logical.optimizer.rules; -import cn.edu.tsinghua.iginx.engine.logical.optimizer.core.RuleCall; import cn.edu.tsinghua.iginx.engine.shared.operator.AbstractJoin; import cn.edu.tsinghua.iginx.engine.shared.operator.MarkJoin; import cn.edu.tsinghua.iginx.engine.shared.operator.Select; import cn.edu.tsinghua.iginx.engine.shared.operator.SingleJoin; import cn.edu.tsinghua.iginx.engine.shared.source.OperatorSource; import cn.edu.tsinghua.iginx.engine.shared.source.Source; +import cn.edu.tsinghua.iginx.logical.optimizer.core.RuleCall; +// @AutoService(Rule.class) public class FilterJoinTransposeRule extends Rule { - private static final class InstanceHolder { - static final FilterJoinTransposeRule INSTANCE = new FilterJoinTransposeRule(); - } - - public static FilterJoinTransposeRule getInstance() { - return InstanceHolder.INSTANCE; - } - - protected FilterJoinTransposeRule() { + public FilterJoinTransposeRule() { /* * we want to match the topology like: * Select diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FilterPushDownAddSchemaPrefixRule.java b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/FilterPushDownAddSchemaPrefixRule.java similarity index 78% rename from core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FilterPushDownAddSchemaPrefixRule.java rename to optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/FilterPushDownAddSchemaPrefixRule.java index b8f96b5d58..1b772aeb11 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FilterPushDownAddSchemaPrefixRule.java +++ b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/FilterPushDownAddSchemaPrefixRule.java @@ -1,24 +1,35 @@ -package cn.edu.tsinghua.iginx.engine.logical.optimizer.rules; +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.logical.optimizer.rules; -import cn.edu.tsinghua.iginx.engine.logical.optimizer.core.RuleCall; import cn.edu.tsinghua.iginx.engine.shared.expr.*; import cn.edu.tsinghua.iginx.engine.shared.operator.AddSchemaPrefix; import cn.edu.tsinghua.iginx.engine.shared.operator.Select; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.*; import cn.edu.tsinghua.iginx.engine.shared.source.OperatorSource; +import cn.edu.tsinghua.iginx.logical.optimizer.core.RuleCall; +import com.google.auto.service.AutoService; +@AutoService(Rule.class) public class FilterPushDownAddSchemaPrefixRule extends Rule { - private static class InstanceHolder { - private static final FilterPushDownAddSchemaPrefixRule INSTANCE = - new FilterPushDownAddSchemaPrefixRule(); - } - - public static FilterPushDownAddSchemaPrefixRule getInstance() { - return InstanceHolder.INSTANCE; - } - - protected FilterPushDownAddSchemaPrefixRule() { + public FilterPushDownAddSchemaPrefixRule() { /* * we want to match the topology like: * Select diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FilterPushDownGroupByRule.java b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/FilterPushDownGroupByRule.java similarity index 76% rename from core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FilterPushDownGroupByRule.java rename to optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/FilterPushDownGroupByRule.java index 727ecb29f0..f30df3158b 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FilterPushDownGroupByRule.java +++ b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/FilterPushDownGroupByRule.java @@ -1,30 +1,42 @@ -package cn.edu.tsinghua.iginx.engine.logical.optimizer.rules; +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.logical.optimizer.rules; -import cn.edu.tsinghua.iginx.engine.logical.optimizer.core.RuleCall; import cn.edu.tsinghua.iginx.engine.logical.utils.LogicalFilterUtils; import cn.edu.tsinghua.iginx.engine.shared.operator.*; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.AndFilter; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; import cn.edu.tsinghua.iginx.engine.shared.operator.type.OperatorType; import cn.edu.tsinghua.iginx.engine.shared.source.OperatorSource; +import cn.edu.tsinghua.iginx.logical.optimizer.core.RuleCall; +import com.google.auto.service.AutoService; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; import java.util.List; +@AutoService(Rule.class) public class FilterPushDownGroupByRule extends Rule { private static List validOps = Arrays.asList(OperatorType.GroupBy, OperatorType.Distinct); - private static class InstanceHolder { - private static final FilterPushDownGroupByRule INSTANCE = new FilterPushDownGroupByRule(); - } - - public static FilterPushDownGroupByRule getInstance() { - return InstanceHolder.INSTANCE; - } - - protected FilterPushDownGroupByRule() { + public FilterPushDownGroupByRule() { /* * we want to match the topology like: * Select diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FilterPushDownPathUnionJoinRule.java b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/FilterPushDownPathUnionJoinRule.java similarity index 91% rename from core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FilterPushDownPathUnionJoinRule.java rename to optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/FilterPushDownPathUnionJoinRule.java index b8a4732330..63b2fe4577 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FilterPushDownPathUnionJoinRule.java +++ b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/FilterPushDownPathUnionJoinRule.java @@ -1,6 +1,23 @@ -package cn.edu.tsinghua.iginx.engine.logical.optimizer.rules; +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.logical.optimizer.rules; -import cn.edu.tsinghua.iginx.engine.logical.optimizer.core.RuleCall; import cn.edu.tsinghua.iginx.engine.logical.utils.LogicalFilterUtils; import cn.edu.tsinghua.iginx.engine.logical.utils.OperatorUtils; import cn.edu.tsinghua.iginx.engine.physical.memory.execute.utils.ExprUtils; @@ -10,10 +27,13 @@ import cn.edu.tsinghua.iginx.engine.shared.operator.type.OuterJoinType; import cn.edu.tsinghua.iginx.engine.shared.source.FragmentSource; import cn.edu.tsinghua.iginx.engine.shared.source.OperatorSource; +import cn.edu.tsinghua.iginx.logical.optimizer.core.RuleCall; import cn.edu.tsinghua.iginx.metadata.entity.ColumnsInterval; +import com.google.auto.service.AutoService; import java.util.*; import java.util.stream.Collectors; +@AutoService(Rule.class) public class FilterPushDownPathUnionJoinRule extends Rule { private static final Set validOps = new HashSet<>( @@ -29,16 +49,7 @@ public class FilterPushDownPathUnionJoinRule extends Rule { private static final Map selectMap = new HashMap<>(); - private static class InstanceHolder { - private static final FilterPushDownPathUnionJoinRule instance = - new FilterPushDownPathUnionJoinRule(); - } - - public static FilterPushDownPathUnionJoinRule getInstance() { - return InstanceHolder.instance; - } - - protected FilterPushDownPathUnionJoinRule() { + public FilterPushDownPathUnionJoinRule() { /* * we want to match the topology like: * Select diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FilterPushDownProjectReorderSortRule.java b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/FilterPushDownProjectReorderSortRule.java similarity index 60% rename from core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FilterPushDownProjectReorderSortRule.java rename to optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/FilterPushDownProjectReorderSortRule.java index 4ff8029199..10d8ab473e 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FilterPushDownProjectReorderSortRule.java +++ b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/FilterPushDownProjectReorderSortRule.java @@ -1,28 +1,39 @@ -package cn.edu.tsinghua.iginx.engine.logical.optimizer.rules; +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.logical.optimizer.rules; -import cn.edu.tsinghua.iginx.engine.logical.optimizer.core.RuleCall; import cn.edu.tsinghua.iginx.engine.shared.operator.*; import cn.edu.tsinghua.iginx.engine.shared.source.OperatorSource; import cn.edu.tsinghua.iginx.engine.shared.source.SourceType; +import cn.edu.tsinghua.iginx.logical.optimizer.core.RuleCall; +import com.google.auto.service.AutoService; import java.util.Arrays; import java.util.HashSet; import java.util.Set; +@AutoService(Rule.class) public class FilterPushDownProjectReorderSortRule extends Rule { private static final Set validOps = new HashSet<>(Arrays.asList(Project.class, Reorder.class, Sort.class)); - private static final class InstanceHolder { - static final FilterPushDownProjectReorderSortRule INSTANCE = - new FilterPushDownProjectReorderSortRule(); - } - - public static FilterPushDownProjectReorderSortRule getInstance() { - return InstanceHolder.INSTANCE; - } - - protected FilterPushDownProjectReorderSortRule() { + public FilterPushDownProjectReorderSortRule() { /* * we want to match the topology like: * Select diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FilterPushDownRenameRule.java b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/FilterPushDownRenameRule.java similarity index 72% rename from core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FilterPushDownRenameRule.java rename to optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/FilterPushDownRenameRule.java index 979923feb7..52a8849129 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FilterPushDownRenameRule.java +++ b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/FilterPushDownRenameRule.java @@ -1,24 +1,38 @@ -package cn.edu.tsinghua.iginx.engine.logical.optimizer.rules; +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.logical.optimizer.rules; -import cn.edu.tsinghua.iginx.engine.logical.optimizer.core.RuleCall; import cn.edu.tsinghua.iginx.engine.logical.utils.PathUtils; import cn.edu.tsinghua.iginx.engine.shared.expr.*; import cn.edu.tsinghua.iginx.engine.shared.operator.Rename; import cn.edu.tsinghua.iginx.engine.shared.operator.Select; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.*; import cn.edu.tsinghua.iginx.engine.shared.source.OperatorSource; -import java.util.Map; +import cn.edu.tsinghua.iginx.logical.optimizer.core.RuleCall; +import cn.edu.tsinghua.iginx.utils.Pair; +import com.google.auto.service.AutoService; +import java.util.List; +@AutoService(Rule.class) public class FilterPushDownRenameRule extends Rule { - private static class InstanceHolder { - private static final FilterPushDownRenameRule INSTANCE = new FilterPushDownRenameRule(); - } - - public static FilterPushDownRenameRule getInstance() { - return InstanceHolder.INSTANCE; - } - protected FilterPushDownRenameRule() { + public FilterPushDownRenameRule() { /* * we want to match the topology like: * Select @@ -37,13 +51,13 @@ public boolean matches(RuleCall call) { public void onMatch(RuleCall call) { Select select = (Select) call.getMatchedRoot(); Rename rename = (Rename) ((OperatorSource) select.getSource()).getOperator(); - select.setFilter(replacePathByRenameMap(select.getFilter(), rename.getAliasMap())); + select.setFilter(replacePathByRenameMap(select.getFilter(), rename.getAliasList())); select.setSource(rename.getSource()); rename.setSource(new OperatorSource(select)); call.transformTo(rename); } - private Filter replacePathByRenameMap(Filter filter, Map renameMap) { + private Filter replacePathByRenameMap(Filter filter, List> renameMap) { Filter newFilter = filter.copy(); newFilter.accept( new FilterVisitor() { @@ -83,7 +97,8 @@ public void visit(ExprFilter filter) { return newFilter; } - private void replaceExpressionByRenameMap(Expression expression, Map renameMap) { + private void replaceExpressionByRenameMap( + Expression expression, List> renameMap) { expression.accept( new ExpressionVisitor() { @Override diff --git a/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/FilterPushDownSelectRule.java b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/FilterPushDownSelectRule.java new file mode 100644 index 0000000000..8c59575ee2 --- /dev/null +++ b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/FilterPushDownSelectRule.java @@ -0,0 +1,54 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.logical.optimizer.rules; + +import cn.edu.tsinghua.iginx.engine.logical.utils.LogicalFilterUtils; +import cn.edu.tsinghua.iginx.engine.shared.operator.Select; +import cn.edu.tsinghua.iginx.logical.optimizer.core.RuleCall; +import com.google.auto.service.AutoService; + +@AutoService(Rule.class) +public class FilterPushDownSelectRule extends Rule { + + public FilterPushDownSelectRule() { + /* + * we want to match the topology like: + * Select + * | + * Select + */ + super("FilterPushDownSelectRule", operand(Select.class, operand(Select.class, any()))); + } + + @Override + public boolean matches(RuleCall call) { + return super.matches(call); + } + + @Override + public void onMatch(RuleCall call) { + // 应当把两个Select合并,用and连接 + Select select = (Select) call.getMatchedRoot(); + Select childSelect = (Select) call.getChildrenIndex().get(select).get(0); + + childSelect.setFilter( + LogicalFilterUtils.mergeFilter(select.getFilter(), childSelect.getFilter())); + call.transformTo(childSelect); + } +} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FilterPushDownSetOpRule.java b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/FilterPushDownSetOpRule.java similarity index 85% rename from core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FilterPushDownSetOpRule.java rename to optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/FilterPushDownSetOpRule.java index 396860f51d..80f0e62abb 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FilterPushDownSetOpRule.java +++ b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/FilterPushDownSetOpRule.java @@ -1,14 +1,34 @@ -package cn.edu.tsinghua.iginx.engine.logical.optimizer.rules; +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.logical.optimizer.rules; -import cn.edu.tsinghua.iginx.engine.logical.optimizer.core.RuleCall; import cn.edu.tsinghua.iginx.engine.shared.Constants; import cn.edu.tsinghua.iginx.engine.shared.operator.*; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.*; import cn.edu.tsinghua.iginx.engine.shared.source.OperatorSource; +import cn.edu.tsinghua.iginx.logical.optimizer.core.RuleCall; import cn.edu.tsinghua.iginx.utils.Pair; +import com.google.auto.service.AutoService; import java.util.*; import java.util.stream.IntStream; +@AutoService(Rule.class) public class FilterPushDownSetOpRule extends Rule { static final Set validSetOps = @@ -17,15 +37,7 @@ public class FilterPushDownSetOpRule extends Rule { // 部分情况下Select只能超集下推,这时候需要记录该Select有没有超集下推过,如果有,下次就不能再下推了 static final Map> selectMap = new HashMap<>(); - private static class InstanceHolder { - private static final FilterPushDownSetOpRule INSTANCE = new FilterPushDownSetOpRule(); - } - - public static FilterPushDownSetOpRule getInstance() { - return InstanceHolder.INSTANCE; - } - - protected FilterPushDownSetOpRule() { + public FilterPushDownSetOpRule() { /* * we want to match the topology like: * Select diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FilterPushDownTransformRule.java b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/FilterPushDownTransformRule.java similarity index 86% rename from core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FilterPushDownTransformRule.java rename to optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/FilterPushDownTransformRule.java index 4b7d6bf362..c761586e44 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FilterPushDownTransformRule.java +++ b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/FilterPushDownTransformRule.java @@ -1,29 +1,41 @@ -package cn.edu.tsinghua.iginx.engine.logical.optimizer.rules; +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.logical.optimizer.rules; -import cn.edu.tsinghua.iginx.engine.logical.optimizer.core.RuleCall; import cn.edu.tsinghua.iginx.engine.logical.utils.LogicalFilterUtils; import cn.edu.tsinghua.iginx.engine.shared.expr.*; import cn.edu.tsinghua.iginx.engine.shared.function.FunctionCall; import cn.edu.tsinghua.iginx.engine.shared.operator.*; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.*; import cn.edu.tsinghua.iginx.engine.shared.source.OperatorSource; +import cn.edu.tsinghua.iginx.logical.optimizer.core.RuleCall; import cn.edu.tsinghua.iginx.utils.StringUtils; +import com.google.auto.service.AutoService; import java.util.*; +@AutoService(Rule.class) public class FilterPushDownTransformRule extends Rule { static final Set validTransforms = new HashSet<>(Arrays.asList(MappingTransform.class, RowTransform.class, SetTransform.class)); - private static class InstanceHolder { - private static final FilterPushDownTransformRule INSTANCE = new FilterPushDownTransformRule(); - } - - public static FilterPushDownTransformRule getInstance() { - return InstanceHolder.INSTANCE; - } - - protected FilterPushDownTransformRule() { + public FilterPushDownTransformRule() { /* * we want to match the topology like: * Select diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FilterPushIntoJoinConditionRule.java b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/FilterPushIntoJoinConditionRule.java similarity index 67% rename from core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FilterPushIntoJoinConditionRule.java rename to optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/FilterPushIntoJoinConditionRule.java index 916116b379..910d243b21 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FilterPushIntoJoinConditionRule.java +++ b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/FilterPushIntoJoinConditionRule.java @@ -1,25 +1,36 @@ -package cn.edu.tsinghua.iginx.engine.logical.optimizer.rules; +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.logical.optimizer.rules; -import cn.edu.tsinghua.iginx.engine.logical.optimizer.core.RuleCall; import cn.edu.tsinghua.iginx.engine.physical.memory.execute.utils.FilterUtils; import cn.edu.tsinghua.iginx.engine.shared.operator.*; import cn.edu.tsinghua.iginx.engine.shared.operator.type.OperatorType; +import cn.edu.tsinghua.iginx.logical.optimizer.core.RuleCall; +import com.google.auto.service.AutoService; import java.util.*; +@AutoService(Rule.class) public class FilterPushIntoJoinConditionRule extends Rule { private static final Set validOps = new HashSet<>(Arrays.asList(OperatorType.InnerJoin, OperatorType.CrossJoin)); - private static class FilterPushIntoJoinConditionRuleInstance { - private static final FilterPushIntoJoinConditionRule INSTANCE = - new FilterPushIntoJoinConditionRule(); - } - - public static FilterPushIntoJoinConditionRule getInstance() { - return FilterPushIntoJoinConditionRuleInstance.INSTANCE; - } - - protected FilterPushIntoJoinConditionRule() { + public FilterPushIntoJoinConditionRule() { /* * we want to match the topology like: * Select diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FilterPushOutJoinConditionRule.java b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/FilterPushOutJoinConditionRule.java similarity index 88% rename from core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FilterPushOutJoinConditionRule.java rename to optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/FilterPushOutJoinConditionRule.java index 1dc1905636..f5b62f79c5 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FilterPushOutJoinConditionRule.java +++ b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/FilterPushOutJoinConditionRule.java @@ -1,6 +1,23 @@ -package cn.edu.tsinghua.iginx.engine.logical.optimizer.rules; +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.logical.optimizer.rules; -import cn.edu.tsinghua.iginx.engine.logical.optimizer.core.RuleCall; import cn.edu.tsinghua.iginx.engine.logical.utils.LogicalFilterUtils; import cn.edu.tsinghua.iginx.engine.logical.utils.OperatorUtils; import cn.edu.tsinghua.iginx.engine.shared.operator.*; @@ -12,8 +29,11 @@ import cn.edu.tsinghua.iginx.engine.shared.operator.type.OperatorType; import cn.edu.tsinghua.iginx.engine.shared.operator.type.OuterJoinType; import cn.edu.tsinghua.iginx.engine.shared.source.OperatorSource; +import cn.edu.tsinghua.iginx.logical.optimizer.core.RuleCall; +import com.google.auto.service.AutoService; import java.util.*; +@AutoService(Rule.class) public class FilterPushOutJoinConditionRule extends Rule { private static final Set validOps = new HashSet<>( @@ -23,16 +43,7 @@ public class FilterPushOutJoinConditionRule extends Rule { OperatorType.MarkJoin, OperatorType.SingleJoin)); - private static class FilterPushOutJoinConditionRuleInstance { - private static final FilterPushOutJoinConditionRule INSTANCE = - new FilterPushOutJoinConditionRule(); - } - - public static FilterPushOutJoinConditionRule getInstance() { - return FilterPushOutJoinConditionRuleInstance.INSTANCE; - } - - protected FilterPushOutJoinConditionRule() { + public FilterPushOutJoinConditionRule() { /* * we want to match the topology like: * Inner/Outer/Mark/Single Join diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FragmentPruningByFilterRule.java b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/FragmentPruningByFilterRule.java similarity index 88% rename from core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FragmentPruningByFilterRule.java rename to optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/FragmentPruningByFilterRule.java index 63634c6612..b56681fa38 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FragmentPruningByFilterRule.java +++ b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/FragmentPruningByFilterRule.java @@ -1,8 +1,23 @@ -package cn.edu.tsinghua.iginx.engine.logical.optimizer.rules; +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.logical.optimizer.rules; -import static cn.edu.tsinghua.iginx.metadata.utils.FragmentUtils.keyFromColumnsIntervalToKeyInterval; - -import cn.edu.tsinghua.iginx.engine.logical.optimizer.core.RuleCall; import cn.edu.tsinghua.iginx.engine.logical.utils.LogicalFilterUtils; import cn.edu.tsinghua.iginx.engine.logical.utils.OperatorUtils; import cn.edu.tsinghua.iginx.engine.shared.Constants; @@ -13,12 +28,15 @@ import cn.edu.tsinghua.iginx.engine.shared.operator.visitor.OperatorVisitor; import cn.edu.tsinghua.iginx.engine.shared.source.FragmentSource; import cn.edu.tsinghua.iginx.engine.shared.source.OperatorSource; +import cn.edu.tsinghua.iginx.logical.optimizer.core.RuleCall; import cn.edu.tsinghua.iginx.metadata.IMetaManager; import cn.edu.tsinghua.iginx.metadata.MetaManagerWrapper; import cn.edu.tsinghua.iginx.metadata.entity.ColumnsInterval; import cn.edu.tsinghua.iginx.metadata.entity.FragmentMeta; import cn.edu.tsinghua.iginx.metadata.entity.KeyInterval; +import cn.edu.tsinghua.iginx.metadata.utils.FragmentUtils; import cn.edu.tsinghua.iginx.utils.Pair; +import com.google.auto.service.AutoService; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -26,19 +44,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@AutoService(Rule.class) public class FragmentPruningByFilterRule extends Rule { private static final Logger LOGGER = LoggerFactory.getLogger(FragmentPruningByFilterRule.class); - private static final class InstanceHolder { - static final FragmentPruningByFilterRule INSTANCE = new FragmentPruningByFilterRule(); - } - - public static FragmentPruningByFilterRule getInstance() { - return FragmentPruningByFilterRule.InstanceHolder.INSTANCE; - } - - protected FragmentPruningByFilterRule() { + public FragmentPruningByFilterRule() { /* * we want to match the topology like: * Select @@ -153,7 +164,7 @@ public void onMatch(RuleCall call) { Map> fragmentsByColumnsInterval = metaManager.getFragmentMapByColumnsInterval(columnsInterval, true); Pair>, List> pair = - keyFromColumnsIntervalToKeyInterval(fragmentsByColumnsInterval); + FragmentUtils.keyFromColumnsIntervalToKeyInterval(fragmentsByColumnsInterval); Map> fragments = pair.k; List dummyFragments = pair.v; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FragmentPruningByPatternRule.java b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/FragmentPruningByPatternRule.java similarity index 78% rename from core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FragmentPruningByPatternRule.java rename to optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/FragmentPruningByPatternRule.java index cb8040d8aa..ed428242ce 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FragmentPruningByPatternRule.java +++ b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/FragmentPruningByPatternRule.java @@ -1,31 +1,43 @@ -package cn.edu.tsinghua.iginx.engine.logical.optimizer.rules; +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.logical.optimizer.rules; -import cn.edu.tsinghua.iginx.engine.logical.optimizer.core.RuleCall; import cn.edu.tsinghua.iginx.engine.shared.operator.*; import cn.edu.tsinghua.iginx.engine.shared.operator.type.OperatorType; import cn.edu.tsinghua.iginx.engine.shared.source.FragmentSource; import cn.edu.tsinghua.iginx.engine.shared.source.OperatorSource; import cn.edu.tsinghua.iginx.engine.shared.source.Source; +import cn.edu.tsinghua.iginx.logical.optimizer.core.RuleCall; import cn.edu.tsinghua.iginx.metadata.entity.ColumnsInterval; +import com.google.auto.service.AutoService; import java.util.ArrayList; import java.util.List; import java.util.Map; +@AutoService(Rule.class) public class FragmentPruningByPatternRule extends Rule { /* 该规则是根据Project的Pattern来判断是否需要Fragment,与列裁剪规则有关, 列裁剪规则裁剪了Project-Fragment中不需要的列,可能导致该Fragment不再需要 */ - private static final class InstanceHolder { - private static final FragmentPruningByPatternRule instance = new FragmentPruningByPatternRule(); - } - - public static FragmentPruningByPatternRule getInstance() { - return InstanceHolder.instance; - } - - protected FragmentPruningByPatternRule() { + public FragmentPruningByPatternRule() { /* * we want to match the topology like: * Project diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FunctionDistinctEliminateRule.java b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/FunctionDistinctEliminateRule.java similarity index 73% rename from core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FunctionDistinctEliminateRule.java rename to optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/FunctionDistinctEliminateRule.java index fffe718ce9..a5de437d5d 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FunctionDistinctEliminateRule.java +++ b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/FunctionDistinctEliminateRule.java @@ -1,28 +1,41 @@ -package cn.edu.tsinghua.iginx.engine.logical.optimizer.rules; +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.logical.optimizer.rules; -import cn.edu.tsinghua.iginx.engine.logical.optimizer.core.RuleCall; import cn.edu.tsinghua.iginx.engine.shared.function.FunctionCall; import cn.edu.tsinghua.iginx.engine.shared.function.system.Max; import cn.edu.tsinghua.iginx.engine.shared.function.system.Min; import cn.edu.tsinghua.iginx.engine.shared.operator.*; import cn.edu.tsinghua.iginx.engine.shared.operator.type.OperatorType; import cn.edu.tsinghua.iginx.engine.shared.source.OperatorSource; +import cn.edu.tsinghua.iginx.logical.optimizer.core.RuleCall; +import cn.edu.tsinghua.iginx.utils.Pair; +import com.google.auto.service.AutoService; import java.util.*; /** 该类实现了GroupBy节点中函数中的Distinct的消除。该规则用于消除函数中的Distinct,比如在min(distinct a)中的distinct是不必要的。 */ +@AutoService(Rule.class) public class FunctionDistinctEliminateRule extends Rule { private static final Set functionSet = new HashSet<>(Arrays.asList(Min.MIN, Max.MAX)); - private static class InstanceHolder { - static final FunctionDistinctEliminateRule INSTANCE = new FunctionDistinctEliminateRule(); - } - - public static FunctionDistinctEliminateRule getInstance() { - return InstanceHolder.INSTANCE; - } - - protected FunctionDistinctEliminateRule() { + public FunctionDistinctEliminateRule() { /* * we want to match the topology like: * GroupBy/DownSample/Row/Mapping/SetTransform @@ -55,13 +68,13 @@ public boolean matches(RuleCall call) { public void onMatch(RuleCall call) { AbstractUnaryOperator unaryOperator = (AbstractUnaryOperator) call.getMatchedRoot(); List functionCallList = getFunctionCallList(unaryOperator); - Map renameMap = new HashMap<>(); + List> renameMap = new ArrayList<>(); // 将函数中的distinct去掉 for (FunctionCall functionCall : functionCallList) { String oldName = functionCall.getFunctionStr(); functionCall.getParams().setDistinct(false); - renameMap.put(functionCall.getFunctionStr(), oldName); + renameMap.add(new Pair<>(functionCall.getFunctionStr(), oldName)); } // 添加一个rename节点,将不带distinct的函数结果重命名为原来的函数名,否则显示与用户输入的SQL不一致 diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/InExistsDistinctEliminateRule.java b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/InExistsDistinctEliminateRule.java similarity index 67% rename from core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/InExistsDistinctEliminateRule.java rename to optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/InExistsDistinctEliminateRule.java index 4d616d58a3..bf01bb503b 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/InExistsDistinctEliminateRule.java +++ b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/InExistsDistinctEliminateRule.java @@ -1,25 +1,38 @@ -package cn.edu.tsinghua.iginx.engine.logical.optimizer.rules; +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.logical.optimizer.rules; -import cn.edu.tsinghua.iginx.engine.logical.optimizer.core.RuleCall; import cn.edu.tsinghua.iginx.engine.shared.operator.AbstractJoin; import cn.edu.tsinghua.iginx.engine.shared.operator.Distinct; import cn.edu.tsinghua.iginx.engine.shared.operator.Operator; import cn.edu.tsinghua.iginx.engine.shared.operator.type.OperatorType; +import cn.edu.tsinghua.iginx.logical.optimizer.core.RuleCall; +import com.google.auto.service.AutoService; /** * 该类实现了Distinct节点的消除。该规则用于消除IN/EXISTS子查询中的DISTINCT节点, 例如SELECT * FROM t1 WHERE EXISTS (SELECT * DISTINCT * FROM t2 WHERE t1.a = t2.a),这里面的DISTINCT节点其实是不必要的。 */ +@AutoService(Rule.class) public class InExistsDistinctEliminateRule extends Rule { - private static class InstanceHolder { - static final InExistsDistinctEliminateRule INSTANCE = new InExistsDistinctEliminateRule(); - } - - public static InExistsDistinctEliminateRule getInstance() { - return InstanceHolder.INSTANCE; - } - protected InExistsDistinctEliminateRule() { + public InExistsDistinctEliminateRule() { /* * we want to match the topology like: * MarkJoin diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/NotFilterRemoveRule.java b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/NotFilterRemoveRule.java similarity index 54% rename from core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/NotFilterRemoveRule.java rename to optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/NotFilterRemoveRule.java index 207ce87ff8..77033056ea 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/NotFilterRemoveRule.java +++ b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/NotFilterRemoveRule.java @@ -1,22 +1,33 @@ -package cn.edu.tsinghua.iginx.engine.logical.optimizer.rules; - -import static cn.edu.tsinghua.iginx.engine.logical.utils.LogicalFilterUtils.removeNot; - -import cn.edu.tsinghua.iginx.engine.logical.optimizer.core.RuleCall; +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.logical.optimizer.rules; + +import cn.edu.tsinghua.iginx.engine.logical.utils.LogicalFilterUtils; import cn.edu.tsinghua.iginx.engine.shared.operator.*; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.*; +import cn.edu.tsinghua.iginx.logical.optimizer.core.RuleCall; +import com.google.auto.service.AutoService; +@AutoService(Rule.class) public class NotFilterRemoveRule extends Rule { - private static final class InstanceHolder { - static final NotFilterRemoveRule INSTANCE = new NotFilterRemoveRule(); - } - - public static NotFilterRemoveRule getInstance() { - return NotFilterRemoveRule.InstanceHolder.INSTANCE; - } - - protected NotFilterRemoveRule() { + public NotFilterRemoveRule() { /* * we want to match the topology like: * Select @@ -66,6 +77,6 @@ public void visit(ExprFilter filter) {} @Override public void onMatch(RuleCall call) { Select select = (Select) call.getMatchedRoot(); - select.setFilter(removeNot(select.getFilter())); + select.setFilter(LogicalFilterUtils.removeNot(select.getFilter())); } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/RowTransformConstantFoldingRule.java b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/RowTransformConstantFoldingRule.java similarity index 66% rename from core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/RowTransformConstantFoldingRule.java rename to optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/RowTransformConstantFoldingRule.java index 4a5ce5dda5..06018b972e 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/RowTransformConstantFoldingRule.java +++ b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/RowTransformConstantFoldingRule.java @@ -1,27 +1,39 @@ -package cn.edu.tsinghua.iginx.engine.logical.optimizer.rules; +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.logical.optimizer.rules; -import cn.edu.tsinghua.iginx.engine.logical.optimizer.core.RuleCall; import cn.edu.tsinghua.iginx.engine.physical.memory.execute.utils.ExprUtils; import cn.edu.tsinghua.iginx.engine.shared.expr.Expression; import cn.edu.tsinghua.iginx.engine.shared.function.FunctionCall; import cn.edu.tsinghua.iginx.engine.shared.operator.Rename; import cn.edu.tsinghua.iginx.engine.shared.operator.RowTransform; import cn.edu.tsinghua.iginx.engine.shared.source.OperatorSource; -import java.util.HashMap; +import cn.edu.tsinghua.iginx.logical.optimizer.core.RuleCall; +import cn.edu.tsinghua.iginx.utils.Pair; +import com.google.auto.service.AutoService; +import java.util.ArrayList; import java.util.List; -import java.util.Map; +@AutoService(Rule.class) public class RowTransformConstantFoldingRule extends Rule { - private static final class InstanceHolder { - static final RowTransformConstantFoldingRule INSTANCE = new RowTransformConstantFoldingRule(); - } - - public static RowTransformConstantFoldingRule getInstance() { - return RowTransformConstantFoldingRule.InstanceHolder.INSTANCE; - } - - protected RowTransformConstantFoldingRule() { + public RowTransformConstantFoldingRule() { /* * we want to match the topology like: * RowTransform @@ -51,7 +63,7 @@ public boolean matches(RuleCall call) { public void onMatch(RuleCall call) { RowTransform rowTransform = (RowTransform) call.getMatchedRoot(); List functionCallList = rowTransform.getFunctionCallList(); - Map aliasMap = new HashMap<>(); + List> aliasList = new ArrayList<>(); for (FunctionCall functionCall : functionCallList) { Expression expr = functionCall.getParams().getExpr(); @@ -62,13 +74,13 @@ public void onMatch(RuleCall call) { Expression foldedExpression = ExprUtils.foldExpression(flattenedExpression); functionCall.getParams().setExpr(foldedExpression); String newName = foldedExpression.getColumnName(); - aliasMap.put(newName, oldName); + aliasList.add(new Pair<>(newName, oldName)); } } } // 改完之后要在上面加一层Rename,因为常量折叠改变了表达式输出的列名,但是我们依然要用原列名 - Rename rename = new Rename(new OperatorSource(rowTransform), aliasMap); + Rename rename = new Rename(new OperatorSource(rowTransform), aliasList); call.transformTo(rename); } } diff --git a/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/Rule.java b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/Rule.java new file mode 100644 index 0000000000..60537677a3 --- /dev/null +++ b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/Rule.java @@ -0,0 +1,103 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.logical.optimizer.rules; + +import cn.edu.tsinghua.iginx.engine.shared.operator.Operator; +import cn.edu.tsinghua.iginx.logical.optimizer.core.Operand; +import cn.edu.tsinghua.iginx.logical.optimizer.core.RuleCall; +import java.util.Arrays; + +public abstract class Rule { + + public static final long DEFAULT_PRIORITY = 0; + + public static final RuleStrategy DEFAULT_STRATEGY = RuleStrategy.FIXED_POINT; + + private final String ruleName; + + /** operand describes the local topology we want to match in this rule */ + private final Operand operand; + + private final RuleStrategy strategy; + + private final long priority; + + protected Rule(String ruleName, Operand operand) { + this(ruleName, operand, DEFAULT_PRIORITY, DEFAULT_STRATEGY); + } + + protected Rule(String ruleName, Operand operand, long priority) { + this(ruleName, operand, priority, DEFAULT_STRATEGY); + } + + protected Rule(String ruleName, Operand operand, RuleStrategy strategy) { + this(ruleName, operand, DEFAULT_PRIORITY, strategy); + } + + protected Rule(String ruleName, Operand operand, long priority, RuleStrategy strategy) { + this.ruleName = ruleName; + this.operand = operand; + this.priority = priority; + this.strategy = strategy; + } + + public String getRuleName() { + return ruleName; + } + + public Operand getOperand() { + return operand; + } + + public RuleStrategy getStrategy() { + return strategy; + } + + public long getPriority() { + return priority; + } + + /** + * Returns whether this rule could possibly match the given operands. + * + *

This method is an opportunity to apply side-conditions to a rule. The Planner calls this + * method after matching all operands of the rule, and before calling method onMatch. + * + *

The default implementation of this method returns true. + */ + public boolean matches(RuleCall call) { + return true; + } + + /** This method is used to modify the local topology after rule matching. */ + public abstract void onMatch(RuleCall call); + + public static Operand any() { + return Operand.ANY_OPERAND; + } + + public static Operand operand(Class clazz, Operand... children) { + return new Operand(clazz, Arrays.asList(children)); + } + + @Override + public String toString() { + return ruleName; + } +} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/RuleCollection.java b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/RuleCollection.java similarity index 69% rename from core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/RuleCollection.java rename to optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/RuleCollection.java index 13aa0f4e20..4db5894c94 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/RuleCollection.java +++ b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/RuleCollection.java @@ -1,53 +1,60 @@ -package cn.edu.tsinghua.iginx.engine.logical.optimizer.rules; +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.logical.optimizer.rules; import cn.edu.tsinghua.iginx.conf.Config; import cn.edu.tsinghua.iginx.conf.ConfigDescriptor; +import cn.edu.tsinghua.iginx.engine.logical.optimizer.IRuleCollection; import java.util.*; +import javax.annotation.Nonnull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class RuleCollection { +public class RuleCollection implements IRuleCollection, Iterable { private static final Logger LOGGER = LoggerFactory.getLogger(RuleCollection.class); + public static final RuleCollection INSTANCE = new RuleCollection(); + private final Map rules = new HashMap<>(); private final Map bannedRules = new HashMap<>(); - private ConfigDescriptor configDescriptor = ConfigDescriptor.getInstance(); - - private static final class InstanceHolder { - static final RuleCollection INSTANCE = new RuleCollection(); - } + private final ConfigDescriptor configDescriptor = ConfigDescriptor.getInstance(); - public static RuleCollection getInstance() { - return InstanceHolder.INSTANCE; + protected RuleCollection() { + addRulesBySPI(); + setRulesByConfig(); } - private RuleCollection() { - // add rules here - // 在这里添加规则 - addRule(NotFilterRemoveRule.getInstance()); - addRule(FragmentPruningByFilterRule.getInstance()); - addRule(ColumnPruningRule.getInstance()); - addRule(FragmentPruningByPatternRule.getInstance()); - addRule(ConstantPropagationRule.getInstance()); - addRule(FilterConstantFoldingRule.getInstance()); - addRule(RowTransformConstantFoldingRule.getInstance()); - addRule(FunctionDistinctEliminateRule.getInstance()); - addRule(InExistsDistinctEliminateRule.getInstance()); - addRule(FilterPushDownAddSchemaPrefixRule.getInstance()); - addRule(FilterPushDownGroupByRule.getInstance()); - addRule(FilterPushDownPathUnionJoinRule.getInstance()); - addRule(FilterPushDownProjectReorderSortRule.getInstance()); - addRule(FilterPushDownRenameRule.getInstance()); - addRule(FilterPushDownSelectRule.getInstance()); - addRule(FilterPushDownSetOpRule.getInstance()); - addRule(FilterPushDownTransformRule.getInstance()); - addRule(FilterPushIntoJoinConditionRule.getInstance()); - addRule(FilterPushOutJoinConditionRule.getInstance()); + private void addRulesBySPI() { + if (LOGGER.isDebugEnabled()) { + ClassLoader cl = Thread.currentThread().getContextClassLoader(); + LOGGER.debug("ClassLoader: {}", cl); + String path = System.getProperty("java.class.path"); + LOGGER.debug("ClassPath: {}", path); + } - setRulesByConfig(); + for (Rule rule : ServiceLoader.load(Rule.class)) { + LOGGER.debug("Add rule by SPI: {}", rule); + addRule(rule); + } } /** 根据配置文件设置rules,未在配置文件中出现的规则默认为off */ @@ -152,10 +159,17 @@ public Map getRulesInfo() { return rulesInfo; } + @Override + @Nonnull public Iterator iterator() { // ensure that this round of optimization will not be affected by rule set modifications // 确保这一轮优化不会受到规则集修改的影响 - return new RuleIterator(new ArrayList<>(rules.values()), new HashSet<>(bannedRules.values())); + List rules = new ArrayList<>(this.rules.values()); + Set bannedRules = new HashSet<>(this.bannedRules.values()); + + // sort rules by priority and rule name + rules.sort(Comparator.comparingLong(Rule::getPriority).thenComparing(Rule::getRuleName)); + return new RuleIterator(rules, bannedRules); } static class RuleIterator implements Iterator { diff --git a/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/RuleStrategy.java b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/RuleStrategy.java new file mode 100644 index 0000000000..91bc1cbbd0 --- /dev/null +++ b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/RuleStrategy.java @@ -0,0 +1,24 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.logical.optimizer.rules; + +public enum RuleStrategy { + FIXED_POINT, // 一直执行直到不再有匹配 + ONCE // 只执行一次 +} diff --git a/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/SetTransformPushDownPathUnionJoinRule.java b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/SetTransformPushDownPathUnionJoinRule.java new file mode 100644 index 0000000000..9ecd533477 --- /dev/null +++ b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/SetTransformPushDownPathUnionJoinRule.java @@ -0,0 +1,158 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cn.edu.tsinghua.iginx.logical.optimizer.rules; + +import cn.edu.tsinghua.iginx.engine.logical.utils.OperatorUtils; +import cn.edu.tsinghua.iginx.engine.shared.Constants; +import cn.edu.tsinghua.iginx.engine.shared.operator.*; +import cn.edu.tsinghua.iginx.engine.shared.operator.visitor.DeepFirstQueueVisitor; +import cn.edu.tsinghua.iginx.engine.shared.source.FragmentSource; +import cn.edu.tsinghua.iginx.engine.shared.source.OperatorSource; +import cn.edu.tsinghua.iginx.engine.shared.source.Source; +import cn.edu.tsinghua.iginx.engine.shared.source.SourceType; +import cn.edu.tsinghua.iginx.logical.optimizer.core.RuleCall; +import cn.edu.tsinghua.iginx.metadata.entity.ColumnsInterval; +import cn.edu.tsinghua.iginx.metadata.entity.FragmentMeta; +import com.google.auto.service.AutoService; +import com.google.common.collect.Range; +import com.google.common.collect.RangeSet; +import com.google.common.collect.TreeRangeSet; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +@AutoService(Rule.class) +public class SetTransformPushDownPathUnionJoinRule extends Rule { + + public SetTransformPushDownPathUnionJoinRule() { + /* + * we want to match the topology like: + * SetTransform + * | + * PathUnion/Join + * + */ + super( + SetTransformPushDownPathUnionJoinRule.class.getSimpleName(), + operand(SetTransform.class, operand(AbstractBinaryOperator.class, any(), any()))); + } + + public boolean matches(RuleCall call) { + SetTransform setTransform = (SetTransform) call.getMatchedRoot(); + + List projects = new ArrayList<>(); + OperatorUtils.findProjectOperators(projects, setTransform); + + List fragments = + projects.stream() + .map(AbstractUnaryOperator::getSource) + .filter(source -> source.getType() == SourceType.Fragment) + .map(source -> (FragmentSource) source) + .map(FragmentSource::getFragment) + .collect(Collectors.toList()); + + // Check if each column can only appear once + RangeSet columnRangeSet = TreeRangeSet.create(); + for (FragmentMeta fragment : fragments) { + // if there is a dummy fragment, we can't push down the set transform correctly + if (fragment.isDummyFragment()) { + return false; + } + ColumnsInterval columnsInterval = fragment.getColumnsInterval(); + // TODO: Would schema prefix affect the result? + if (columnsInterval.getSchemaPrefix() != null) { + return false; + } + String startColumn = columnsInterval.getStartColumn(); + String endColumn = columnsInterval.getEndColumn(); + if (startColumn == null || endColumn == null) {} + + Range columnRange = rangeOf(columnsInterval); + // if there is an intersection of fragment columnIntervals, maybe there are intersecting + // columns in Join + if (columnRangeSet.intersects(columnRange)) { + return false; + } + columnRangeSet.add(columnRange); + } + + Source source = setTransform.getSource(); + if (source.getType() != SourceType.Operator) { + return false; + } + + DeepFirstQueueVisitor deepFirstQueueVisitor = new DeepFirstQueueVisitor(); + ((OperatorSource) source).getOperator().accept(deepFirstQueueVisitor); + + for (Operator operator : deepFirstQueueVisitor.getQueue()) { + if (operator instanceof Project) { + Source projectSource = ((Project) operator).getSource(); + if (projectSource.getType() != SourceType.Fragment) { + return false; + } + } else if (operator instanceof Join) { + String joinBy = ((Join) operator).getJoinBy(); + switch (joinBy) { + case Constants.KEY: + case Constants.ORDINAL: + break; + default: + return false; + } + } else if (operator instanceof PathUnion) { + // do nothing + } else { + return false; + } + } + + return true; + } + + private static Range rangeOf(ColumnsInterval columnsInterval) { + String startColumn = columnsInterval.getStartColumn(); + String endColumn = columnsInterval.getEndColumn(); + if (startColumn == null && endColumn == null) { + return Range.all(); + } else if (startColumn == null) { + return Range.lessThan(endColumn); + } else if (endColumn == null) { + return Range.atLeast(startColumn); + } else { + return Range.closedOpen(startColumn, endColumn); + } + } + + @Override + public void onMatch(RuleCall call) { + SetTransform setTransform = (SetTransform) call.getMatchedRoot(); + List projectList = new ArrayList<>(); + OperatorUtils.findProjectOperators(projectList, setTransform); + + List setTransformList = + projectList.stream() + .map(Project::copy) + .map(OperatorSource::new) + .map(setTransform::copyWithSource) + .map(SetTransform.class::cast) + .collect(Collectors.toList()); + + Operator newRoot = OperatorUtils.joinOperators(setTransformList, Constants.ORDINAL); + call.transformTo(newRoot); + } +} diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/optimizer/naive/NaiveConstraintManager.java b/optimizer/src/main/java/cn/edu/tsinghua/iginx/physical/optimizer/naive/NaiveConstraintManager.java similarity index 77% rename from core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/optimizer/naive/NaiveConstraintManager.java rename to optimizer/src/main/java/cn/edu/tsinghua/iginx/physical/optimizer/naive/NaiveConstraintManager.java index 31b9a08741..23f3767608 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/optimizer/naive/NaiveConstraintManager.java +++ b/optimizer/src/main/java/cn/edu/tsinghua/iginx/physical/optimizer/naive/NaiveConstraintManager.java @@ -1,22 +1,21 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ -package cn.edu.tsinghua.iginx.engine.physical.optimizer.naive; +package cn.edu.tsinghua.iginx.physical.optimizer.naive; import cn.edu.tsinghua.iginx.engine.shared.constraint.ConstraintManager; import cn.edu.tsinghua.iginx.engine.shared.operator.*; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/optimizer/naive/NaivePhysicalOptimizer.java b/optimizer/src/main/java/cn/edu/tsinghua/iginx/physical/optimizer/naive/NaivePhysicalOptimizer.java similarity index 80% rename from core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/optimizer/naive/NaivePhysicalOptimizer.java rename to optimizer/src/main/java/cn/edu/tsinghua/iginx/physical/optimizer/naive/NaivePhysicalOptimizer.java index 3a65425d29..67960fe419 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/optimizer/naive/NaivePhysicalOptimizer.java +++ b/optimizer/src/main/java/cn/edu/tsinghua/iginx/physical/optimizer/naive/NaivePhysicalOptimizer.java @@ -1,27 +1,25 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ -package cn.edu.tsinghua.iginx.engine.physical.optimizer.naive; +package cn.edu.tsinghua.iginx.physical.optimizer.naive; import cn.edu.tsinghua.iginx.conf.ConfigDescriptor; import cn.edu.tsinghua.iginx.engine.physical.optimizer.PhysicalOptimizer; import cn.edu.tsinghua.iginx.engine.physical.optimizer.ReplicaDispatcher; -import cn.edu.tsinghua.iginx.engine.physical.optimizer.rule.Rule; import cn.edu.tsinghua.iginx.engine.physical.task.*; import cn.edu.tsinghua.iginx.engine.shared.RequestContext; import cn.edu.tsinghua.iginx.engine.shared.constraint.ConstraintManager; @@ -30,6 +28,7 @@ import cn.edu.tsinghua.iginx.engine.shared.source.OperatorSource; import cn.edu.tsinghua.iginx.engine.shared.source.Source; import cn.edu.tsinghua.iginx.engine.shared.source.SourceType; +import cn.edu.tsinghua.iginx.physical.optimizer.rule.Rule; import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -58,7 +57,6 @@ public ReplicaDispatcher getReplicaDispatcher() { return NaiveReplicaDispatcher.getInstance(); } - @Override public void setRules(Collection rules) {} private PhysicalTask constructTask(Operator operator, RequestContext context) { @@ -77,15 +75,23 @@ private PhysicalTask constructTask(Operator operator, RequestContext context) { OperatorSource operatorSource = (OperatorSource) source; Operator sourceOperator = operatorSource.getOperator(); PhysicalTask sourceTask = constructTask(operatorSource.getOperator(), context); + // push down Select operator if (ConfigDescriptor.getInstance().getConfig().isEnablePushDown() && sourceTask instanceof StoragePhysicalTask && sourceOperator.getType() == OperatorType.Project && ((Project) sourceOperator).getTagFilter() == null - && ((UnaryOperator) sourceOperator).getSource().getType() == SourceType.Fragment - && operator.getType() == OperatorType.Select - && ((Select) operator).getTagFilter() == null) { - sourceTask.getOperators().add(operator); - return sourceTask; + && ((UnaryOperator) sourceOperator).getSource().getType() == SourceType.Fragment) { + switch (operator.getType()) { + case Select: + if (((Select) operator).getTagFilter() == null) { + sourceTask.getOperators().add(operator); + return sourceTask; + } + break; + case SetTransform: + sourceTask.getOperators().add(operator); + return sourceTask; + } } List operators = new ArrayList<>(); operators.add(operator); diff --git a/optimizer/src/main/java/cn/edu/tsinghua/iginx/physical/optimizer/naive/NaiveReplicaDispatcher.java b/optimizer/src/main/java/cn/edu/tsinghua/iginx/physical/optimizer/naive/NaiveReplicaDispatcher.java new file mode 100644 index 0000000000..3f0ffa1fe7 --- /dev/null +++ b/optimizer/src/main/java/cn/edu/tsinghua/iginx/physical/optimizer/naive/NaiveReplicaDispatcher.java @@ -0,0 +1,40 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cn.edu.tsinghua.iginx.physical.optimizer.naive; + +import cn.edu.tsinghua.iginx.engine.physical.optimizer.ReplicaDispatcher; +import cn.edu.tsinghua.iginx.engine.physical.task.StoragePhysicalTask; + +public class NaiveReplicaDispatcher implements ReplicaDispatcher { + + private static final NaiveReplicaDispatcher INSTANCE = new NaiveReplicaDispatcher(); + + private NaiveReplicaDispatcher() {} + + @Override + public String chooseReplica(StoragePhysicalTask task) { + if (task == null) { + return null; + } + return task.getTargetFragment().getMasterStorageUnitId(); + } + + public static NaiveReplicaDispatcher getInstance() { + return INSTANCE; + } +} diff --git a/optimizer/src/main/java/cn/edu/tsinghua/iginx/physical/optimizer/rule/Rule.java b/optimizer/src/main/java/cn/edu/tsinghua/iginx/physical/optimizer/rule/Rule.java new file mode 100644 index 0000000000..e06a64edc1 --- /dev/null +++ b/optimizer/src/main/java/cn/edu/tsinghua/iginx/physical/optimizer/rule/Rule.java @@ -0,0 +1,24 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cn.edu.tsinghua.iginx.physical.optimizer.rule; + +public interface Rule { + + // TODO: how to define rule + +} diff --git a/core/src/test/java/cn/edu/tsinghua/iginx/optimizer/IteratorTest.java b/optimizer/src/test/java/cn/edu/tsinghua/iginx/optimizer/IteratorTest.java similarity index 85% rename from core/src/test/java/cn/edu/tsinghua/iginx/optimizer/IteratorTest.java rename to optimizer/src/test/java/cn/edu/tsinghua/iginx/optimizer/IteratorTest.java index 50ee56edee..04e85b60af 100644 --- a/core/src/test/java/cn/edu/tsinghua/iginx/optimizer/IteratorTest.java +++ b/optimizer/src/test/java/cn/edu/tsinghua/iginx/optimizer/IteratorTest.java @@ -1,16 +1,30 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.optimizer; -import cn.edu.tsinghua.iginx.engine.logical.optimizer.core.iterator.DeepFirstIterator; -import cn.edu.tsinghua.iginx.engine.logical.optimizer.core.iterator.LeveledIterator; -import cn.edu.tsinghua.iginx.engine.logical.optimizer.core.iterator.ReverseDeepFirstIterator; -import cn.edu.tsinghua.iginx.engine.logical.optimizer.core.iterator.ReverseLeveledIterator; -import cn.edu.tsinghua.iginx.engine.logical.optimizer.core.iterator.TreeIterator; import cn.edu.tsinghua.iginx.engine.shared.operator.InnerJoin; import cn.edu.tsinghua.iginx.engine.shared.operator.Operator; import cn.edu.tsinghua.iginx.engine.shared.operator.OuterJoin; import cn.edu.tsinghua.iginx.engine.shared.operator.Project; import cn.edu.tsinghua.iginx.engine.shared.operator.Reorder; import cn.edu.tsinghua.iginx.engine.shared.operator.Select; +import cn.edu.tsinghua.iginx.logical.optimizer.core.iterator.*; import java.util.Arrays; import java.util.List; import org.junit.Assert; diff --git a/core/src/test/java/cn/edu/tsinghua/iginx/optimizer/OperandTest.java b/optimizer/src/test/java/cn/edu/tsinghua/iginx/optimizer/OperandTest.java similarity index 66% rename from core/src/test/java/cn/edu/tsinghua/iginx/optimizer/OperandTest.java rename to optimizer/src/test/java/cn/edu/tsinghua/iginx/optimizer/OperandTest.java index 91c606176d..ab4f4bbecb 100644 --- a/core/src/test/java/cn/edu/tsinghua/iginx/optimizer/OperandTest.java +++ b/optimizer/src/test/java/cn/edu/tsinghua/iginx/optimizer/OperandTest.java @@ -1,15 +1,33 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.optimizer; -import static cn.edu.tsinghua.iginx.engine.logical.optimizer.rules.Rule.any; -import static cn.edu.tsinghua.iginx.engine.logical.optimizer.rules.Rule.operand; +import static cn.edu.tsinghua.iginx.logical.optimizer.rules.Rule.any; +import static cn.edu.tsinghua.iginx.logical.optimizer.rules.Rule.operand; -import cn.edu.tsinghua.iginx.engine.logical.optimizer.core.Operand; import cn.edu.tsinghua.iginx.engine.shared.operator.AbstractJoin; import cn.edu.tsinghua.iginx.engine.shared.operator.Operator; import cn.edu.tsinghua.iginx.engine.shared.operator.Project; import cn.edu.tsinghua.iginx.engine.shared.operator.Reorder; import cn.edu.tsinghua.iginx.engine.shared.operator.Select; import cn.edu.tsinghua.iginx.engine.shared.source.EmptySource; +import cn.edu.tsinghua.iginx.logical.optimizer.core.Operand; import java.util.Collections; import org.junit.Assert; import org.junit.Test; diff --git a/core/src/test/java/cn/edu/tsinghua/iginx/optimizer/RBOTest.java b/optimizer/src/test/java/cn/edu/tsinghua/iginx/optimizer/RBOTest.java similarity index 81% rename from core/src/test/java/cn/edu/tsinghua/iginx/optimizer/RBOTest.java rename to optimizer/src/test/java/cn/edu/tsinghua/iginx/optimizer/RBOTest.java index 68eafbbaf4..b082cfb121 100644 --- a/core/src/test/java/cn/edu/tsinghua/iginx/optimizer/RBOTest.java +++ b/optimizer/src/test/java/cn/edu/tsinghua/iginx/optimizer/RBOTest.java @@ -1,8 +1,26 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.optimizer; -import cn.edu.tsinghua.iginx.engine.logical.optimizer.rbo.RuleBasedOptimizer; -import cn.edu.tsinghua.iginx.engine.logical.optimizer.rules.RuleCollection; import cn.edu.tsinghua.iginx.engine.shared.operator.Operator; +import cn.edu.tsinghua.iginx.logical.optimizer.rbo.RuleBasedOptimizer; +import cn.edu.tsinghua.iginx.logical.optimizer.rules.RuleCollection; import java.util.List; import org.junit.Assert; import org.junit.Test; @@ -10,7 +28,7 @@ public class RBOTest { private final RuleBasedOptimizer rbo = new RuleBasedOptimizer(); - private final RuleCollection ruleCollection = RuleCollection.getInstance(); + private final RuleCollection ruleCollection = RuleCollection.INSTANCE; @Test public void testNotFilterRemoveRule() { diff --git a/core/src/test/java/cn/edu/tsinghua/iginx/optimizer/RBOTestUtils.java b/optimizer/src/test/java/cn/edu/tsinghua/iginx/optimizer/RBOTestUtils.java similarity index 60% rename from core/src/test/java/cn/edu/tsinghua/iginx/optimizer/RBOTestUtils.java rename to optimizer/src/test/java/cn/edu/tsinghua/iginx/optimizer/RBOTestUtils.java index 04801f7384..0fceff9f87 100644 --- a/core/src/test/java/cn/edu/tsinghua/iginx/optimizer/RBOTestUtils.java +++ b/optimizer/src/test/java/cn/edu/tsinghua/iginx/optimizer/RBOTestUtils.java @@ -1,13 +1,31 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.optimizer; -import cn.edu.tsinghua.iginx.engine.logical.optimizer.rules.RuleCollection; +import cn.edu.tsinghua.iginx.logical.optimizer.rules.RuleCollection; import java.util.ArrayList; import java.util.List; import java.util.Map; public class RBOTestUtils { - private static final RuleCollection ruleCollection = RuleCollection.getInstance(); + private static final RuleCollection ruleCollection = RuleCollection.INSTANCE; /** * 禁止给定规则之外的规则 diff --git a/core/src/test/java/cn/edu/tsinghua/iginx/optimizer/TreeBuilder.java b/optimizer/src/test/java/cn/edu/tsinghua/iginx/optimizer/TreeBuilder.java similarity index 92% rename from core/src/test/java/cn/edu/tsinghua/iginx/optimizer/TreeBuilder.java rename to optimizer/src/test/java/cn/edu/tsinghua/iginx/optimizer/TreeBuilder.java index f7547e6c25..31a444bb16 100644 --- a/core/src/test/java/cn/edu/tsinghua/iginx/optimizer/TreeBuilder.java +++ b/optimizer/src/test/java/cn/edu/tsinghua/iginx/optimizer/TreeBuilder.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.optimizer; import cn.edu.tsinghua.iginx.engine.shared.Constants; diff --git a/optimizer/src/test/java/cn/edu/tsinghua/iginx/optimizer/TreePrinter.java b/optimizer/src/test/java/cn/edu/tsinghua/iginx/optimizer/TreePrinter.java new file mode 100644 index 0000000000..65e0ba5487 --- /dev/null +++ b/optimizer/src/test/java/cn/edu/tsinghua/iginx/optimizer/TreePrinter.java @@ -0,0 +1,31 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.optimizer; + +import cn.edu.tsinghua.iginx.engine.shared.operator.Operator; +import cn.edu.tsinghua.iginx.engine.shared.operator.visitor.TreeInfoVisitor; + +public class TreePrinter { + + public static String getTreeInfo(Operator root) { + TreeInfoVisitor visitor = new TreeInfoVisitor(); + root.accept(visitor); + return visitor.getTreeInfo(); + } +} diff --git a/pom.xml b/pom.xml index 034c86e16c..de3cf9f36e 100644 --- a/pom.xml +++ b/pom.xml @@ -1,4 +1,23 @@ + 4.0.0 @@ -10,16 +29,20 @@ IGinX - core - thrift + + dependency shared + thrift + session + jdbc client antlr - jdbc - session - dataSources + core + optimizer + dataSource example test + assembly @@ -39,9 +62,11 @@ 0.8.6 ${project.basedir} + 8 + 8 UTF-8 UTF-8 - 0.6.0-SNAPSHOT + 0.8.0-SNAPSHOT @@ -71,6 +96,11 @@ iginx-shared ${project.version} + + cn.edu.tsinghua + iginx-optimizer + ${project.version} + cn.edu.tsinghua iotdb12 @@ -104,7 +134,7 @@ cn.edu.tsinghua relational - 0.6.0-SNAPSHOT + ${project.version} cn.edu.tsinghua @@ -127,15 +157,24 @@ 2.2 - com.fasterxml.jackson.core - jackson-databind - 2.13.4.2 + com.fasterxml.jackson + jackson-bom + 2.17.2 + pom + import cn.hutool hutool-all 5.8.26 + + org.apache.arrow + arrow-bom + 17.0.0 + pom + import + org.apache.commons commons-lang3 @@ -171,6 +210,13 @@ commons-email 1.6.0 + + io.netty + netty-bom + 4.1.110.Final + pom + import + org.apache.zookeeper zookeeper @@ -189,7 +235,7 @@ cn.edu.tsinghua.iginx parquet-file - 0.0.2 + 0.1.0 redis.clients @@ -204,18 +250,15 @@ io.etcd jetcd-core - 0.5.7 - true - - - org.slf4j - slf4j-log4j12 - - - log4j - log4j - - + 0.5.11 + + + + io.grpc + grpc-bom + 1.57.2 + pom + import com.influxdb @@ -244,6 +287,13 @@ lombok 1.18.20 + + com.google.guava + guava-bom + 33.2.1-jre + pom + import + @@ -270,7 +320,29 @@ org.apache.maven.plugins maven-assembly-plugin - 3.6.0 + 3.7.1 + + + com.mycila + license-maven-plugin + 4.5 + + true + true + + +

LICENSE_HEADER
+ + **/test/resources/** + **/requirements.txt + + + + + SLASHSTAR_STYLE + SLASHSTAR_STYLE + + @@ -402,6 +474,18 @@ + + com.mycila + license-maven-plugin + + + + format + + initialize + + + diff --git a/session/pom.xml b/session/pom.xml index ff9f9b3bb1..34bac0393b 100644 --- a/session/pom.xml +++ b/session/pom.xml @@ -1,4 +1,23 @@ + 4.0.0 @@ -11,11 +30,6 @@ iginx-session IGinX Session - - 8 - 8 - - cn.edu.tsinghua diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/exception/SessionException.java b/session/src/main/java/cn/edu/tsinghua/iginx/exception/SessionException.java index a46e454a13..8f13d10bc0 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/exception/SessionException.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/exception/SessionException.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.exception; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/pool/IginxInfo.java b/session/src/main/java/cn/edu/tsinghua/iginx/pool/IginxInfo.java index 11829d6a2f..d5b004c191 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/pool/IginxInfo.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/pool/IginxInfo.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.pool; public class IginxInfo { diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/pool/SessionPool.java b/session/src/main/java/cn/edu/tsinghua/iginx/pool/SessionPool.java index 2510d33b0d..ce2023a4db 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/pool/SessionPool.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/pool/SessionPool.java @@ -1,5 +1,25 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.pool; +import static cn.edu.tsinghua.iginx.constant.GlobalConstant.KEY_MAX_VAL; +import static cn.edu.tsinghua.iginx.constant.GlobalConstant.KEY_MIN_VAL; import static java.lang.Math.max; import cn.edu.tsinghua.iginx.exception.SessionException; @@ -473,7 +493,7 @@ public void removeHistoryDataSource(List removedStorag return; } catch (SessionException e) { // TException means the connection is broken, remove it and get a new one. - LOGGER.warn("remove history data source failed", e); + LOGGER.warn("removeHistoryDataSource failed", e); cleanSessionAndMayThrowConnectionException(session, i, e); } catch (RuntimeException e) { putBack(session); @@ -505,7 +525,7 @@ public List showColumns() throws SessionException { return ret; } catch (SessionException e) { // TException means the connection is broken, remove it and get a new one. - LOGGER.warn("insertTablet failed", e); + LOGGER.warn("showColumns failed", e); cleanSessionAndMayThrowConnectionException(session, i, e); } catch (RuntimeException e) { putBack(session); @@ -572,12 +592,12 @@ public void deleteColumns( } public void insertColumnRecords( - List paths, long[] timestamps, Object[] valuesList, List dataTypeList) + List paths, long[] keys, Object[] valuesList, List dataTypeList) throws SessionException { for (int i = 0; i < RETRY; i++) { Session session = getSession(); try { - session.insertColumnRecords(paths, timestamps, valuesList, dataTypeList); + session.insertColumnRecords(paths, keys, valuesList, dataTypeList); putBack(session); return; } catch (SessionException e) { @@ -593,7 +613,7 @@ public void insertColumnRecords( public void insertColumnRecords( List paths, - long[] timestamps, + long[] keys, Object[] valuesList, List dataTypeList, List> tagsList) @@ -601,7 +621,7 @@ public void insertColumnRecords( for (int i = 0; i < RETRY; i++) { Session session = getSession(); try { - session.insertColumnRecords(paths, timestamps, valuesList, dataTypeList, tagsList); + session.insertColumnRecords(paths, keys, valuesList, dataTypeList, tagsList); putBack(session); return; } catch (SessionException e) { @@ -617,7 +637,7 @@ public void insertColumnRecords( public void insertColumnRecords( List paths, - long[] timestamps, + long[] keys, Object[] valuesList, List dataTypeList, List> tagsList, @@ -626,8 +646,7 @@ public void insertColumnRecords( for (int i = 0; i < RETRY; i++) { Session session = getSession(); try { - session.insertColumnRecords( - paths, timestamps, valuesList, dataTypeList, tagsList, precision); + session.insertColumnRecords(paths, keys, valuesList, dataTypeList, tagsList, precision); putBack(session); return; } catch (SessionException e) { @@ -642,12 +661,12 @@ public void insertColumnRecords( } public void insertNonAlignedColumnRecords( - List paths, long[] timestamps, Object[] valuesList, List dataTypeList) + List paths, long[] keys, Object[] valuesList, List dataTypeList) throws SessionException { for (int i = 0; i < RETRY; i++) { Session session = getSession(); try { - session.insertNonAlignedColumnRecords(paths, timestamps, valuesList, dataTypeList); + session.insertNonAlignedColumnRecords(paths, keys, valuesList, dataTypeList); putBack(session); return; } catch (SessionException e) { @@ -663,7 +682,7 @@ public void insertNonAlignedColumnRecords( public void insertNonAlignedColumnRecords( List paths, - long[] timestamps, + long[] keys, Object[] valuesList, List dataTypeList, List> tagsList) @@ -671,8 +690,7 @@ public void insertNonAlignedColumnRecords( for (int i = 0; i < RETRY; i++) { Session session = getSession(); try { - session.insertNonAlignedColumnRecords( - paths, timestamps, valuesList, dataTypeList, tagsList); + session.insertNonAlignedColumnRecords(paths, keys, valuesList, dataTypeList, tagsList); putBack(session); return; } catch (SessionException e) { @@ -688,7 +706,7 @@ public void insertNonAlignedColumnRecords( public void insertNonAlignedColumnRecords( List paths, - long[] timestamps, + long[] keys, Object[] valuesList, List dataTypeList, List> tagsList, @@ -698,7 +716,7 @@ public void insertNonAlignedColumnRecords( Session session = getSession(); try { session.insertNonAlignedColumnRecords( - paths, timestamps, valuesList, dataTypeList, tagsList, precision); + paths, keys, valuesList, dataTypeList, tagsList, precision); putBack(session); return; } catch (SessionException e) { @@ -714,7 +732,7 @@ public void insertNonAlignedColumnRecords( public void insertRowRecords( List paths, - long[] timestamps, + long[] keys, Object[] valuesList, List dataTypeList, List> tagsList) @@ -722,7 +740,7 @@ public void insertRowRecords( for (int i = 0; i < RETRY; i++) { Session session = getSession(); try { - session.insertRowRecords(paths, timestamps, valuesList, dataTypeList, tagsList); + session.insertRowRecords(paths, keys, valuesList, dataTypeList, tagsList); putBack(session); return; } catch (SessionException e) { @@ -738,7 +756,7 @@ public void insertRowRecords( public void insertRowRecords( List paths, - long[] timestamps, + long[] keys, Object[] valuesList, List dataTypeList, List> tagsList, @@ -747,7 +765,7 @@ public void insertRowRecords( for (int i = 0; i < RETRY; i++) { Session session = getSession(); try { - session.insertRowRecords(paths, timestamps, valuesList, dataTypeList, tagsList, precison); + session.insertRowRecords(paths, keys, valuesList, dataTypeList, tagsList, precison); putBack(session); return; } catch (SessionException e) { @@ -762,12 +780,12 @@ public void insertRowRecords( } public void insertNonAlignedRowRecords( - List paths, long[] timestamps, Object[] valuesList, List dataTypeList) + List paths, long[] keys, Object[] valuesList, List dataTypeList) throws SessionException { for (int i = 0; i < RETRY; i++) { Session session = getSession(); try { - session.insertNonAlignedRowRecords(paths, timestamps, valuesList, dataTypeList); + session.insertNonAlignedRowRecords(paths, keys, valuesList, dataTypeList); putBack(session); return; } catch (SessionException e) { @@ -783,7 +801,7 @@ public void insertNonAlignedRowRecords( public void insertNonAlignedRowRecords( List paths, - long[] timestamps, + long[] keys, Object[] valuesList, List dataTypeList, List> tagsList) @@ -791,7 +809,7 @@ public void insertNonAlignedRowRecords( for (int i = 0; i < RETRY; i++) { Session session = getSession(); try { - session.insertNonAlignedRowRecords(paths, timestamps, valuesList, dataTypeList, tagsList); + session.insertNonAlignedRowRecords(paths, keys, valuesList, dataTypeList, tagsList); putBack(session); return; } catch (SessionException e) { @@ -807,7 +825,7 @@ public void insertNonAlignedRowRecords( public void insertNonAlignedRowRecords( List paths, - long[] timestamps, + long[] keys, Object[] valuesList, List dataTypeList, List> tagsList, @@ -817,7 +835,7 @@ public void insertNonAlignedRowRecords( Session session = getSession(); try { session.insertNonAlignedRowRecords( - paths, timestamps, valuesList, dataTypeList, tagsList, precision); + paths, keys, valuesList, dataTypeList, tagsList, precision); putBack(session); return; } catch (SessionException e) { @@ -985,6 +1003,13 @@ public SessionAggregateQueryDataSet aggregateQuery( return sessionAggregateQueryDataSet; } + // downsample query without time interval + public SessionQueryDataSet downsampleQuery( + List paths, AggregateType aggregateType, long precision) throws SessionException { + return downsampleQuery(paths, KEY_MIN_VAL, KEY_MAX_VAL, aggregateType, precision); + } + + // downsample query with key interval public SessionQueryDataSet downsampleQuery( List paths, long startKey, long endKey, AggregateType aggregateType, long precision) throws SessionException { diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session/ClusterInfo.java b/session/src/main/java/cn/edu/tsinghua/iginx/session/ClusterInfo.java index fecfdfe5f3..55916b1149 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session/ClusterInfo.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session/ClusterInfo.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session/Column.java b/session/src/main/java/cn/edu/tsinghua/iginx/session/Column.java index e1db46dbf1..c1aab542ab 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session/Column.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session/Column.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session/CurveMatchResult.java b/session/src/main/java/cn/edu/tsinghua/iginx/session/CurveMatchResult.java index 762331a1f0..077a880b42 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session/CurveMatchResult.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session/CurveMatchResult.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.session; public class CurveMatchResult { diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session/Point.java b/session/src/main/java/cn/edu/tsinghua/iginx/session/Point.java index 40628ea06f..4e849c7fc0 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session/Point.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session/Point.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session/QueryDataSet.java b/session/src/main/java/cn/edu/tsinghua/iginx/session/QueryDataSet.java index 948248f9b2..7cd0d6b22c 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session/QueryDataSet.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session/QueryDataSet.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session/Session.java b/session/src/main/java/cn/edu/tsinghua/iginx/session/Session.java index c7215fc5fd..b2f94cde63 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session/Session.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session/Session.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session; @@ -309,33 +308,30 @@ public void deleteColumns( } public void insertColumnRecords( - List paths, long[] timestamps, Object[] valuesList, List dataTypeList) + List paths, long[] keys, Object[] valuesList, List dataTypeList) throws SessionException { - insertColumnRecords(paths, timestamps, valuesList, dataTypeList, null, timeUnit); + insertColumnRecords(paths, keys, valuesList, dataTypeList, null, timeUnit); } public void insertColumnRecords( List paths, - long[] timestamps, + long[] keys, Object[] valuesList, List dataTypeList, List> tagsList) throws SessionException { - insertColumnRecords(paths, timestamps, valuesList, dataTypeList, tagsList, timeUnit); + insertColumnRecords(paths, keys, valuesList, dataTypeList, tagsList, timeUnit); } public void insertColumnRecords( List paths, - long[] timestamps, + long[] keys, Object[] valuesList, List dataTypeList, List> tagsList, TimePrecision precision) throws SessionException { - if (paths.isEmpty() - || timestamps.length == 0 - || valuesList.length == 0 - || dataTypeList.isEmpty()) { + if (paths.isEmpty() || keys.length == 0 || valuesList.length == 0 || dataTypeList.isEmpty()) { LOGGER.error("Invalid insert request!"); return; } @@ -348,14 +344,14 @@ public void insertColumnRecords( return; } - long[] sortedTimestamps = Arrays.copyOf(timestamps, timestamps.length); - Integer[] index = new Integer[sortedTimestamps.length]; - for (int i = 0; i < sortedTimestamps.length; i++) { + long[] sortedKeys = Arrays.copyOf(keys, keys.length); + Integer[] index = new Integer[sortedKeys.length]; + for (int i = 0; i < sortedKeys.length; i++) { index[i] = i; } Arrays.sort( - index, Comparator.comparingLong(Arrays.asList(ArrayUtils.toObject(sortedTimestamps))::get)); - Arrays.sort(sortedTimestamps); + index, Comparator.comparingLong(Arrays.asList(ArrayUtils.toObject(sortedKeys))::get)); + Arrays.sort(sortedKeys); for (int i = 0; i < valuesList.length; i++) { Object[] values = new Object[index.length]; for (int j = 0; j < index.length; j++) { @@ -388,13 +384,13 @@ public void insertColumnRecords( List bitmapBufferList = new ArrayList<>(); for (int i = 0; i < sortedValuesList.length; i++) { Object[] values = (Object[]) sortedValuesList[i]; - if (values.length != sortedTimestamps.length) { - LOGGER.error("The sizes of timestamps and the element of valuesList should be equal."); + if (values.length != sortedKeys.length) { + LOGGER.error("The sizes of keys and the element of valuesList should be equal."); return; } valueBufferList.add(ByteUtils.getColumnByteBuffer(values, sortedDataTypeList.get(i))); - Bitmap bitmap = new Bitmap(sortedTimestamps.length); - for (int j = 0; j < sortedTimestamps.length; j++) { + Bitmap bitmap = new Bitmap(sortedKeys.length); + for (int j = 0; j < sortedKeys.length; j++) { if (values[j] != null) { bitmap.mark(j); } @@ -405,7 +401,7 @@ public void insertColumnRecords( InsertColumnRecordsReq req = new InsertColumnRecordsReq(); req.setSessionId(sessionId); req.setPaths(sortedPaths); - req.setKeys(getByteArrayFromLongArray(sortedTimestamps)); + req.setKeys(getByteArrayFromLongArray(sortedKeys)); req.setValuesList(valueBufferList); req.setBitmapList(bitmapBufferList); req.setDataTypeList(sortedDataTypeList); @@ -416,33 +412,30 @@ public void insertColumnRecords( } public void insertNonAlignedColumnRecords( - List paths, long[] timestamps, Object[] valuesList, List dataTypeList) + List paths, long[] keys, Object[] valuesList, List dataTypeList) throws SessionException { - insertNonAlignedColumnRecords(paths, timestamps, valuesList, dataTypeList, null, timeUnit); + insertNonAlignedColumnRecords(paths, keys, valuesList, dataTypeList, null, timeUnit); } public void insertNonAlignedColumnRecords( List paths, - long[] timestamps, + long[] keys, Object[] valuesList, List dataTypeList, List> tagsList) throws SessionException { - insertNonAlignedColumnRecords(paths, timestamps, valuesList, dataTypeList, tagsList, timeUnit); + insertNonAlignedColumnRecords(paths, keys, valuesList, dataTypeList, tagsList, timeUnit); } public void insertNonAlignedColumnRecords( List paths, - long[] timestamps, + long[] keys, Object[] valuesList, List dataTypeList, List> tagsList, TimePrecision precision) throws SessionException { - if (paths.isEmpty() - || timestamps.length == 0 - || valuesList.length == 0 - || dataTypeList.isEmpty()) { + if (paths.isEmpty() || keys.length == 0 || valuesList.length == 0 || dataTypeList.isEmpty()) { LOGGER.error("Invalid insert request!"); return; } @@ -455,14 +448,14 @@ public void insertNonAlignedColumnRecords( return; } - long[] sortedTimestamps = Arrays.copyOf(timestamps, timestamps.length); - Integer[] index = new Integer[sortedTimestamps.length]; - for (int i = 0; i < sortedTimestamps.length; i++) { + long[] sortedKeys = Arrays.copyOf(keys, keys.length); + Integer[] index = new Integer[sortedKeys.length]; + for (int i = 0; i < sortedKeys.length; i++) { index[i] = i; } Arrays.sort( - index, Comparator.comparingLong(Arrays.asList(ArrayUtils.toObject(sortedTimestamps))::get)); - Arrays.sort(sortedTimestamps); + index, Comparator.comparingLong(Arrays.asList(ArrayUtils.toObject(sortedKeys))::get)); + Arrays.sort(sortedKeys); for (int i = 0; i < valuesList.length; i++) { Object[] values = new Object[index.length]; for (int j = 0; j < index.length; j++) { @@ -495,13 +488,13 @@ public void insertNonAlignedColumnRecords( List bitmapBufferList = new ArrayList<>(); for (int i = 0; i < sortedValuesList.length; i++) { Object[] values = (Object[]) sortedValuesList[i]; - if (values.length != sortedTimestamps.length) { - LOGGER.error("The sizes of timestamps and the element of valuesList should be equal."); + if (values.length != sortedKeys.length) { + LOGGER.error("The sizes of keys and the element of valuesList should be equal."); return; } valueBufferList.add(ByteUtils.getColumnByteBuffer(values, sortedDataTypeList.get(i))); - Bitmap bitmap = new Bitmap(sortedTimestamps.length); - for (int j = 0; j < sortedTimestamps.length; j++) { + Bitmap bitmap = new Bitmap(sortedKeys.length); + for (int j = 0; j < sortedKeys.length; j++) { if (values[j] != null) { bitmap.mark(j); } @@ -512,7 +505,7 @@ public void insertNonAlignedColumnRecords( InsertNonAlignedColumnRecordsReq req = new InsertNonAlignedColumnRecordsReq(); req.setSessionId(sessionId); req.setPaths(sortedPaths); - req.setKeys(getByteArrayFromLongArray(sortedTimestamps)); + req.setKeys(getByteArrayFromLongArray(sortedKeys)); req.setValuesList(valueBufferList); req.setBitmapList(bitmapBufferList); req.setDataTypeList(sortedDataTypeList); @@ -524,26 +517,23 @@ public void insertNonAlignedColumnRecords( public void insertRowRecords( List paths, - long[] timestamps, + long[] keys, Object[] valuesList, List dataTypeList, List> tagsList) throws SessionException { - insertRowRecords(paths, timestamps, valuesList, dataTypeList, tagsList, timeUnit); + insertRowRecords(paths, keys, valuesList, dataTypeList, tagsList, timeUnit); } public void insertRowRecords( List paths, - long[] timestamps, + long[] keys, Object[] valuesList, List dataTypeList, List> tagsList, TimePrecision precision) throws SessionException { - if (paths.isEmpty() - || timestamps.length == 0 - || valuesList.length == 0 - || dataTypeList.isEmpty()) { + if (paths.isEmpty() || keys.length == 0 || valuesList.length == 0 || dataTypeList.isEmpty()) { LOGGER.error("Invalid insert request!"); return; } @@ -551,8 +541,8 @@ public void insertRowRecords( LOGGER.error("The sizes of paths and dataTypeList should be equal."); return; } - if (timestamps.length != valuesList.length) { - LOGGER.error("The sizes of timestamps and valuesList should be equal."); + if (keys.length != valuesList.length) { + LOGGER.error("The sizes of keys and valuesList should be equal."); return; } if (tagsList != null && !tagsList.isEmpty() && paths.size() != tagsList.size()) { @@ -560,14 +550,14 @@ public void insertRowRecords( return; } - long[] sortedTimestamps = Arrays.copyOf(timestamps, timestamps.length); - Integer[] index = new Integer[sortedTimestamps.length]; - for (int i = 0; i < sortedTimestamps.length; i++) { + long[] sortedKeys = Arrays.copyOf(keys, keys.length); + Integer[] index = new Integer[sortedKeys.length]; + for (int i = 0; i < sortedKeys.length; i++) { index[i] = i; } Arrays.sort( - index, Comparator.comparingLong(Arrays.asList(ArrayUtils.toObject(sortedTimestamps))::get)); - Arrays.sort(sortedTimestamps); + index, Comparator.comparingLong(Arrays.asList(ArrayUtils.toObject(sortedKeys))::get)); + Arrays.sort(sortedKeys); Object[] sortedValuesList = new Object[valuesList.length]; for (int i = 0; i < valuesList.length; i++) { sortedValuesList[i] = valuesList[index[i]]; @@ -600,7 +590,7 @@ public void insertRowRecords( List valueBufferList = new ArrayList<>(); List bitmapBufferList = new ArrayList<>(); - for (int i = 0; i < sortedTimestamps.length; i++) { + for (int i = 0; i < sortedKeys.length; i++) { Object[] values = (Object[]) sortedValuesList[i]; if (values.length != sortedPaths.size()) { LOGGER.error("The sizes of paths and the element of valuesList should be equal."); @@ -619,7 +609,7 @@ public void insertRowRecords( InsertRowRecordsReq req = new InsertRowRecordsReq(); req.setSessionId(sessionId); req.setPaths(sortedPaths); - req.setKeys(getByteArrayFromLongArray(sortedTimestamps)); + req.setKeys(getByteArrayFromLongArray(sortedKeys)); req.setValuesList(valueBufferList); req.setBitmapList(bitmapBufferList); req.setDataTypeList(sortedDataTypeList); @@ -630,33 +620,30 @@ public void insertRowRecords( } public void insertNonAlignedRowRecords( - List paths, long[] timestamps, Object[] valuesList, List dataTypeList) + List paths, long[] keys, Object[] valuesList, List dataTypeList) throws SessionException { - insertNonAlignedRowRecords(paths, timestamps, valuesList, dataTypeList, null, timeUnit); + insertNonAlignedRowRecords(paths, keys, valuesList, dataTypeList, null, timeUnit); } public void insertNonAlignedRowRecords( List paths, - long[] timestamps, + long[] keys, Object[] valuesList, List dataTypeList, List> tagsList) throws SessionException { - insertNonAlignedRowRecords(paths, timestamps, valuesList, dataTypeList, tagsList, timeUnit); + insertNonAlignedRowRecords(paths, keys, valuesList, dataTypeList, tagsList, timeUnit); } public void insertNonAlignedRowRecords( List paths, - long[] timestamps, + long[] keys, Object[] valuesList, List dataTypeList, List> tagsList, TimePrecision precision) throws SessionException { - if (paths.isEmpty() - || timestamps.length == 0 - || valuesList.length == 0 - || dataTypeList.isEmpty()) { + if (paths.isEmpty() || keys.length == 0 || valuesList.length == 0 || dataTypeList.isEmpty()) { LOGGER.error("Invalid insert request!"); return; } @@ -664,8 +651,8 @@ public void insertNonAlignedRowRecords( LOGGER.error("The sizes of paths and dataTypeList should be equal."); return; } - if (timestamps.length != valuesList.length) { - LOGGER.error("The sizes of timestamps and valuesList should be equal."); + if (keys.length != valuesList.length) { + LOGGER.error("The sizes of keys and valuesList should be equal."); return; } if (tagsList != null && !tagsList.isEmpty() && paths.size() != tagsList.size()) { @@ -673,14 +660,14 @@ public void insertNonAlignedRowRecords( return; } - long[] sortedTimestamps = Arrays.copyOf(timestamps, timestamps.length); - Integer[] index = new Integer[sortedTimestamps.length]; - for (int i = 0; i < sortedTimestamps.length; i++) { + long[] sortedKeys = Arrays.copyOf(keys, keys.length); + Integer[] index = new Integer[sortedKeys.length]; + for (int i = 0; i < sortedKeys.length; i++) { index[i] = i; } Arrays.sort( - index, Comparator.comparingLong(Arrays.asList(ArrayUtils.toObject(sortedTimestamps))::get)); - Arrays.sort(sortedTimestamps); + index, Comparator.comparingLong(Arrays.asList(ArrayUtils.toObject(sortedKeys))::get)); + Arrays.sort(sortedKeys); Object[] sortedValuesList = new Object[valuesList.length]; for (int i = 0; i < valuesList.length; i++) { sortedValuesList[i] = valuesList[index[i]]; @@ -713,7 +700,7 @@ public void insertNonAlignedRowRecords( List valueBufferList = new ArrayList<>(); List bitmapBufferList = new ArrayList<>(); - for (int i = 0; i < sortedTimestamps.length; i++) { + for (int i = 0; i < sortedKeys.length; i++) { Object[] values = (Object[]) sortedValuesList[i]; if (values.length != sortedPaths.size()) { LOGGER.error("The sizes of paths and the element of valuesList should be equal."); @@ -732,7 +719,7 @@ public void insertNonAlignedRowRecords( InsertNonAlignedRowRecordsReq req = new InsertNonAlignedRowRecordsReq(); req.setSessionId(sessionId); req.setPaths(sortedPaths); - req.setKeys(getByteArrayFromLongArray(sortedTimestamps)); + req.setKeys(getByteArrayFromLongArray(sortedKeys)); req.setValuesList(valueBufferList); req.setBitmapList(bitmapBufferList); req.setDataTypeList(sortedDataTypeList); @@ -1134,11 +1121,21 @@ void closeQuery(long queryId) throws SessionException { public long commitTransformJob( List taskInfoList, ExportType exportType, String fileName) throws SessionException { + return commitTransformJob(taskInfoList, exportType, fileName, null); + } + + public long commitTransformJob( + List taskInfoList, ExportType exportType, String fileName, String schedule) + throws SessionException { CommitTransformJobReq req = new CommitTransformJobReq(sessionId, taskInfoList, exportType); if (fileName != null) { req.setFileName(fileName); } + if (schedule != null) { + req.setSchedule(schedule); + } + Reference ref = new Reference<>(); executeWithCheck(() -> (ref.resp = client.commitTransformJob(req)).status); return ref.resp.getJobId(); diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session/SessionAggregateQueryDataSet.java b/session/src/main/java/cn/edu/tsinghua/iginx/session/SessionAggregateQueryDataSet.java index 456df5f747..809403ca61 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session/SessionAggregateQueryDataSet.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session/SessionAggregateQueryDataSet.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.session; import static cn.edu.tsinghua.iginx.utils.ByteUtils.getLongArrayFromByteBuffer; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session/SessionExecuteSqlResult.java b/session/src/main/java/cn/edu/tsinghua/iginx/session/SessionExecuteSqlResult.java index 453a5de818..c3a02c3ae3 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session/SessionExecuteSqlResult.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session/SessionExecuteSqlResult.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session/SessionQueryDataSet.java b/session/src/main/java/cn/edu/tsinghua/iginx/session/SessionQueryDataSet.java index cc3fa2653d..ee2f281913 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session/SessionQueryDataSet.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session/SessionQueryDataSet.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/Arguments.java b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/Arguments.java index b74524ad5b..28fe266314 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/Arguments.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/Arguments.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session_v2; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/AsyncWriteClient.java b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/AsyncWriteClient.java index aa6a7ddf7d..91642be2fc 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/AsyncWriteClient.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/AsyncWriteClient.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session_v2; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/ClusterClient.java b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/ClusterClient.java index 2ba4dd4acf..ddc094902a 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/ClusterClient.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/ClusterClient.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session_v2; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/DeleteClient.java b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/DeleteClient.java index 632f9546d1..84aa16a53b 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/DeleteClient.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/DeleteClient.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session_v2; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/IginXClient.java b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/IginXClient.java index e9197623c9..d65b479892 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/IginXClient.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/IginXClient.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session_v2; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/IginXClientFactory.java b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/IginXClientFactory.java index b5e4080836..f759985a17 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/IginXClientFactory.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/IginXClientFactory.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session_v2; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/IginXClientOptions.java b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/IginXClientOptions.java index f4cafe2137..ab83f4b5f0 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/IginXClientOptions.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/IginXClientOptions.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session_v2; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/QueryClient.java b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/QueryClient.java index 946932de1c..6130983192 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/QueryClient.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/QueryClient.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session_v2; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/TransformClient.java b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/TransformClient.java index 5c2de90c11..982b5dc81c 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/TransformClient.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/TransformClient.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.session_v2; import cn.edu.tsinghua.iginx.session_v2.domain.Transform; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/UsersClient.java b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/UsersClient.java index a746e11b97..9544f25da3 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/UsersClient.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/UsersClient.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session_v2; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/WriteClient.java b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/WriteClient.java index b3e594ba47..b89c443206 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/WriteClient.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/WriteClient.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session_v2; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/annotations/Field.java b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/annotations/Field.java index b48b595330..1d1d6a429f 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/annotations/Field.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/annotations/Field.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session_v2.annotations; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/annotations/Measurement.java b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/annotations/Measurement.java index 75e823eb5c..bce8e68988 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/annotations/Measurement.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/annotations/Measurement.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session_v2.annotations; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/domain/ClusterInfo.java b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/domain/ClusterInfo.java index cb873c3864..a83f31cca0 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/domain/ClusterInfo.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/domain/ClusterInfo.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session_v2.domain; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/domain/Storage.java b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/domain/Storage.java index c367e5de8c..4334cb5785 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/domain/Storage.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/domain/Storage.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session_v2.domain; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/domain/Task.java b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/domain/Task.java index 91d5154ba0..c21f8726d2 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/domain/Task.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/domain/Task.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.session_v2.domain; import cn.edu.tsinghua.iginx.session_v2.Arguments; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/domain/Transform.java b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/domain/Transform.java index 7d876c18ff..662fd8ee49 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/domain/Transform.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/domain/Transform.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.session_v2.domain; import cn.edu.tsinghua.iginx.session_v2.Arguments; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/domain/User.java b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/domain/User.java index 799f2eae28..7615e62b28 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/domain/User.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/domain/User.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session_v2.domain; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/exception/IginXException.java b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/exception/IginXException.java index 63eeaa1e48..324c2a7495 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/exception/IginXException.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/exception/IginXException.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session_v2.exception; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/internal/AbstractFunctionClient.java b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/internal/AbstractFunctionClient.java index 3a97cdd2ab..941dc977f9 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/internal/AbstractFunctionClient.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/internal/AbstractFunctionClient.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session_v2.internal; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/internal/AsyncWriteClientImpl.java b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/internal/AsyncWriteClientImpl.java index 25cd74ca29..9e564d6bc2 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/internal/AsyncWriteClientImpl.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/internal/AsyncWriteClientImpl.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session_v2.internal; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/internal/ClusterClientImpl.java b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/internal/ClusterClientImpl.java index b012af02c1..ee31303489 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/internal/ClusterClientImpl.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/internal/ClusterClientImpl.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session_v2.internal; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/internal/DeleteClientImpl.java b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/internal/DeleteClientImpl.java index 20dc720e60..f74574018c 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/internal/DeleteClientImpl.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/internal/DeleteClientImpl.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session_v2.internal; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/internal/IginXClientImpl.java b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/internal/IginXClientImpl.java index 4b1e05e96f..b18259a01d 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/internal/IginXClientImpl.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/internal/IginXClientImpl.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session_v2.internal; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/internal/MeasurementMapper.java b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/internal/MeasurementMapper.java index c28e9a35fd..ee466670a4 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/internal/MeasurementMapper.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/internal/MeasurementMapper.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session_v2.internal; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/internal/MeasurementUtils.java b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/internal/MeasurementUtils.java index 4e44dfe832..c50fb98cb6 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/internal/MeasurementUtils.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/internal/MeasurementUtils.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.session_v2.internal; import java.util.ArrayList; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/internal/QueryClientImpl.java b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/internal/QueryClientImpl.java index ab3c037779..284558e342 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/internal/QueryClientImpl.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/internal/QueryClientImpl.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session_v2.internal; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/internal/ResultMapper.java b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/internal/ResultMapper.java index 2e9a311435..08323ca044 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/internal/ResultMapper.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/internal/ResultMapper.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session_v2.internal; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/internal/TransformClientImpl.java b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/internal/TransformClientImpl.java index 419c770d09..892e96928e 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/internal/TransformClientImpl.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/internal/TransformClientImpl.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.session_v2.internal; import cn.edu.tsinghua.iginx.exception.SessionException; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/internal/UsersClientImpl.java b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/internal/UsersClientImpl.java index dcad9c0dc2..eed5b062dd 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/internal/UsersClientImpl.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/internal/UsersClientImpl.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session_v2.internal; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/internal/WriteClientImpl.java b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/internal/WriteClientImpl.java index 1c09a8a6df..fc068a0ebf 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/internal/WriteClientImpl.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/internal/WriteClientImpl.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session_v2.internal; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/query/AggregateQuery.java b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/query/AggregateQuery.java index 0ead2e9c85..d87b497f3f 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/query/AggregateQuery.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/query/AggregateQuery.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session_v2.query; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/query/DownsampleQuery.java b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/query/DownsampleQuery.java index ddfad4fdbf..073d86dbdb 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/query/DownsampleQuery.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/query/DownsampleQuery.java @@ -1,23 +1,25 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session_v2.query; +import static cn.edu.tsinghua.iginx.constant.GlobalConstant.KEY_MAX_VAL; +import static cn.edu.tsinghua.iginx.constant.GlobalConstant.KEY_MIN_VAL; + import cn.edu.tsinghua.iginx.session_v2.Arguments; import cn.edu.tsinghua.iginx.thrift.AggregateType; import java.util.*; @@ -108,8 +110,8 @@ public static class Builder { private Builder() { this.measurements = new HashSet<>(); this.tagsList = new ArrayList<>(); - this.startKey = 0L; - this.endKey = Long.MAX_VALUE; + this.startKey = KEY_MIN_VAL; + this.endKey = KEY_MAX_VAL; this.precision = 0L; this.timePrecision = null; } @@ -150,7 +152,7 @@ public DownsampleQuery.Builder startKey(long startKey) { public DownsampleQuery.Builder endKey(long endKey) { if (endKey < 0) { - throw new IllegalArgumentException("endKey mush greater than zero."); + throw new IllegalArgumentException("endKey must greater than zero."); } if (endKey <= startKey) { throw new IllegalArgumentException("endKey must greater than startKey."); diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/query/IginXColumn.java b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/query/IginXColumn.java index bed65ab57e..86d9e72e3b 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/query/IginXColumn.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/query/IginXColumn.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session_v2.query; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/query/IginXHeader.java b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/query/IginXHeader.java index 4f025fdf15..a1d1a7669f 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/query/IginXHeader.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/query/IginXHeader.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session_v2.query; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/query/IginXRecord.java b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/query/IginXRecord.java index 5fde058f0e..8c5cc8c31f 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/query/IginXRecord.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/query/IginXRecord.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session_v2.query; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/query/IginXTable.java b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/query/IginXTable.java index e2c9cdc34d..93106d35a5 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/query/IginXTable.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/query/IginXTable.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session_v2.query; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/query/LastQuery.java b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/query/LastQuery.java index dd51cc8490..c8d795f377 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/query/LastQuery.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/query/LastQuery.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session_v2.query; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/query/Query.java b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/query/Query.java index 3900923c40..73daae5208 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/query/Query.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/query/Query.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session_v2.query; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/query/SimpleQuery.java b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/query/SimpleQuery.java index 5a231771d6..465e8967ed 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/query/SimpleQuery.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/query/SimpleQuery.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session_v2.query; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/write/Point.java b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/write/Point.java index 6d91df640b..d31b5df320 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/write/Point.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/write/Point.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session_v2.write; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/write/Record.java b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/write/Record.java index aef2a80aeb..a274877f27 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/write/Record.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/write/Record.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session_v2.write; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/write/Table.java b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/write/Table.java index 24475a0bed..9993ae0eb9 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/write/Table.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session_v2/write/Table.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.session_v2.write; diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/utils/StatusUtils.java b/session/src/main/java/cn/edu/tsinghua/iginx/utils/StatusUtils.java index 62e1926a2f..788e9c8893 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/utils/StatusUtils.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/utils/StatusUtils.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.utils; diff --git a/session/src/test/java/ResultFormatTest.java b/session/src/test/java/ResultFormatTest.java index fcf7bd8285..218b0b5a62 100644 --- a/session/src/test/java/ResultFormatTest.java +++ b/session/src/test/java/ResultFormatTest.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + import static org.junit.Assert.assertEquals; import cn.edu.tsinghua.iginx.session.SessionExecuteSqlResult; diff --git a/session_py/example.py b/session_py/example.py index 1a207cf9ce..01ca135ecf 100644 --- a/session_py/example.py +++ b/session_py/example.py @@ -1,20 +1,21 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. # +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + import pandas as pd from iginx_pyclient.session import Session diff --git a/session_py/file_example.py b/session_py/file_example.py index 6735ef1365..b7d6c9ae07 100644 --- a/session_py/file_example.py +++ b/session_py/file_example.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + """ A simple example on how to manipulate file with IGinX """ diff --git a/session_py/iginx/iginx_pyclient/__init__.py b/session_py/iginx/iginx_pyclient/__init__.py index 2a1e720805..537ea8cde0 100644 --- a/session_py/iginx/iginx_pyclient/__init__.py +++ b/session_py/iginx/iginx_pyclient/__init__.py @@ -1,17 +1,19 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + # diff --git a/session_py/iginx/iginx_pyclient/cluster_info.py b/session_py/iginx/iginx_pyclient/cluster_info.py index 11c47e70d3..608769b762 100644 --- a/session_py/iginx/iginx_pyclient/cluster_info.py +++ b/session_py/iginx/iginx_pyclient/cluster_info.py @@ -1,19 +1,19 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . # from .thrift.rpc.ttypes import StorageEngineType diff --git a/session_py/iginx/iginx_pyclient/dataset.py b/session_py/iginx/iginx_pyclient/dataset.py index d6e08cfdf9..8b4b949350 100644 --- a/session_py/iginx/iginx_pyclient/dataset.py +++ b/session_py/iginx/iginx_pyclient/dataset.py @@ -1,20 +1,21 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. # +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + from enum import Enum import pandas as pd @@ -22,6 +23,84 @@ from .thrift.rpc.ttypes import SqlType, AggregateType, ExecuteSqlResp from .utils.bitmap import Bitmap from .utils.byte_utils import get_long_array, get_values_by_data_type, BytesParser +from .thrift.rpc.ttypes import DataType + +def map_dtype(dtype): + if pd.api.types.is_bool_dtype(dtype): + return DataType.BOOLEAN + elif pd.api.types.is_integer_dtype(dtype): + if dtype == 'int32': + return DataType.INTEGER + else: + return DataType.LONG + elif pd.api.types.is_float_dtype(dtype): + if dtype == 'float32': + return DataType.FLOAT + else: + return DataType.DOUBLE + elif pd.api.types.is_numeric_dtype(dtype): + return DataType.DOUBLE + else: + return DataType.BINARY # all other types are treated as BINARY + +# TODO: process NA values +def column_dataset_from_df(df: pd.DataFrame, prefix: str = ""): + # if no prefix is provided, the column names must contain at least one '.' except key. + column_list = df.columns.tolist() + if prefix == "": + for col in column_list: + if '.' not in col and col != 'key': + raise RuntimeError(f"The paths in data must contain '.' or prefix must be set") + else: + prefix = prefix + '.' + + if 'key' in column_list: + # examine key type + key_dtype = df['key'].dtype + valid_key = False + if key_dtype == 'int64': + valid_key = True + elif pd.api.types.is_integer_dtype(key_dtype): + df['key'] = df['key'].astype('int64') + valid_key = True + + if not valid_key: + raise RuntimeError(f"Invalid key type is provided. Required long/int but was %s.", df['key'].dtype) + key_list = df['key'].tolist() + df = df.drop(columns=['key']) + # if there is no key column in data, creat one + else: + key_list = list(range(len(df))) + + mapped_types = [map_dtype(dtype) for col, dtype in df.dtypes.items()] + values_list = [df[col].tolist() for col in df.columns] + column_list = [prefix + col for col in df.columns.tolist()] + return ColumnDataSet(column_list, mapped_types, key_list, values_list) + + +class ColumnDataSet(object): + def __init__(self, paths, types, keys, values_list): + self.__paths = paths + self.__types = types + self.__keys = keys + self.__values_list = values_list + + def get_insert_args(self): + return self.__paths, self.__keys, self.__values_list, self.__types + + def __str__(self): + column_names = ['key'] + self.__paths + num_rows = len(self.__keys) + columns = [self.__keys] + self.__values_list + output = [] + + output.append('\t'.join(column_names)) + + for i in range(num_rows): + row = [str(columns[j][i]) for j in range(len(columns))] + output.append('\t'.join(row)) + + return '\n'.join(output) class Point(object): @@ -99,15 +178,20 @@ def __str__(self): return value def to_df(self): - columns = ["key"] + has_key = self.__timestamps != [] + columns = ["key"] if has_key else [] for column in self.__paths: columns.append(str(column)) value_matrix = [] - for i in range(len(self.__timestamps)): - value = [self.__timestamps[i]] - value.extend(self.__values[i]) - value_matrix.append(value) + if has_key: + for i in range(len(self.__timestamps)): + value = [self.__timestamps[i]] + value.extend(self.__values[i]) + value_matrix.append(value) + else: + for i in range(len(self.__values)): + value_matrix.append(self.__values[i]) return pd.DataFrame(value_matrix, columns=columns) diff --git a/session_py/iginx/iginx_pyclient/session.py b/session_py/iginx/iginx_pyclient/session.py index 25068a73d8..0b76a9b622 100644 --- a/session_py/iginx/iginx_pyclient/session.py +++ b/session_py/iginx/iginx_pyclient/session.py @@ -1,31 +1,33 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. # +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + import csv import logging import os.path from datetime import datetime +import pandas as pd from thrift.protocol import TBinaryProtocol from thrift.transport import TSocket, TTransport from pathlib import Path from .cluster_info import ClusterInfo -from .dataset import QueryDataSet, AggregateQueryDataSet, StatementExecuteDataSet +from .dataset import column_dataset_from_df, QueryDataSet, AggregateQueryDataSet, StatementExecuteDataSet from .thrift.rpc.IService import Client from .thrift.rpc.ttypes import ( OpenSessionReq, @@ -62,6 +64,11 @@ logger = logging.getLogger("IginX") +# key value boundary for IGinX(defined in shared.GlobalConstant) +MAX_KEY = 9223372036854775807 +MIN_KEY = -9223372036854775807 + + def isPyReg(statement:str): statement = statement.strip().lower() return statement.startswith("create") and ("function" in statement) @@ -400,6 +407,17 @@ def insert_non_aligned_column_records(self, paths, timestamps, values_list, data status = self.__client.insertNonAlignedColumnRecords(req) Session.verify_status(status) + def insert_df(self, df: pd.DataFrame, prefix: str = ""): + """ + insert dataframe data into IGinX + :param df: dataframe that contains data + :param prefix: (optional) path names in IGinX + must contain '.'. If columns in dataframe does not meet the requirement, a prefix can be used + """ + dataset = column_dataset_from_df(df, prefix) + paths, keys, value_list, type_list = dataset.get_insert_args() + self.insert_column_records(paths, keys, value_list, type_list) + def query(self, paths, start_time, end_time, timePrecision=None): req = QueryDataReq(sessionId=self.__session_id, paths=Session.merge_and_sort_paths(paths), startKey=start_time, endKey=end_time, timePrecision=timePrecision) @@ -424,6 +442,10 @@ def last_query(self, paths, start_time=0, timePrecision=None): return QueryDataSet(paths, data_types, raw_data_set.keys, raw_data_set.valuesList, raw_data_set.bitmapList) + def downsample_query_no_interval(self, paths, type, precision, timePrecision=None): + return self.downsample_query(paths, MIN_KEY, MAX_KEY, type, precision, timePrecision) + + def downsample_query(self, paths, start_time, end_time, type, precision, timePrecision=None): req = DownsampleQueryReq(sessionId=self.__session_id, paths=paths, startKey=start_time, endKey=end_time, aggregateType=type, diff --git a/session_py/iginx/iginx_pyclient/thrift/__init__.py b/session_py/iginx/iginx_pyclient/thrift/__init__.py index e69de29bb2..02225d5e15 100644 --- a/session_py/iginx/iginx_pyclient/thrift/__init__.py +++ b/session_py/iginx/iginx_pyclient/thrift/__init__.py @@ -0,0 +1,18 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + diff --git a/session_py/iginx/iginx_pyclient/thrift/rpc/IService-remote b/session_py/iginx/iginx_pyclient/thrift/rpc/IService-remote index d59dd67032..19c3f274b3 100644 --- a/session_py/iginx/iginx_pyclient/thrift/rpc/IService-remote +++ b/session_py/iginx/iginx_pyclient/thrift/rpc/IService-remote @@ -49,6 +49,7 @@ if len(sys.argv) <= 1 or sys.argv[1] == '--help': print(' ExecuteStatementResp executeStatement(ExecuteStatementReq req)') print(' FetchResultsResp fetchResults(FetchResultsReq req)') print(' LoadCSVResp loadCSV(LoadCSVReq req)') + print(' LoadUDFResp loadUDF(LoadUDFReq req)') print(' Status closeStatement(CloseStatementReq req)') print(' CommitTransformJobResp commitTransformJob(CommitTransformJobReq req)') print(' QueryTransformJobStatusResp queryTransformJobStatus(QueryTransformJobStatusReq req)') @@ -60,6 +61,8 @@ if len(sys.argv) <= 1 or sys.argv[1] == '--help': print(' CurveMatchResp curveMatch(CurveMatchReq req)') print(' DebugInfoResp debugInfo(DebugInfoReq req)') print(' ShowSessionIDResp showSessionID(ShowSessionIDReq req)') + print(' ShowRulesResp showRules(ShowRulesReq req)') + print(' Status setRules(SetRulesReq req)') print('') sys.exit(0) @@ -289,6 +292,12 @@ elif cmd == 'loadCSV': sys.exit(1) pp.pprint(client.loadCSV(eval(args[0]),)) +elif cmd == 'loadUDF': + if len(args) != 1: + print('loadUDF requires 1 args') + sys.exit(1) + pp.pprint(client.loadUDF(eval(args[0]),)) + elif cmd == 'closeStatement': if len(args) != 1: print('closeStatement requires 1 args') @@ -355,6 +364,18 @@ elif cmd == 'showSessionID': sys.exit(1) pp.pprint(client.showSessionID(eval(args[0]),)) +elif cmd == 'showRules': + if len(args) != 1: + print('showRules requires 1 args') + sys.exit(1) + pp.pprint(client.showRules(eval(args[0]),)) + +elif cmd == 'setRules': + if len(args) != 1: + print('setRules requires 1 args') + sys.exit(1) + pp.pprint(client.setRules(eval(args[0]),)) + else: print('Unrecognized method %s' % cmd) sys.exit(1) diff --git a/session_py/iginx/iginx_pyclient/thrift/rpc/IService.py b/session_py/iginx/iginx_pyclient/thrift/rpc/IService.py index 7af0b1502d..358925678d 100644 --- a/session_py/iginx/iginx_pyclient/thrift/rpc/IService.py +++ b/session_py/iginx/iginx_pyclient/thrift/rpc/IService.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + # # Autogenerated by Thrift Compiler (0.16.0) # @@ -219,6 +237,14 @@ def loadCSV(self, req): """ pass + def loadUDF(self, req): + """ + Parameters: + - req + + """ + pass + def closeStatement(self, req): """ Parameters: @@ -307,6 +333,22 @@ def showSessionID(self, req): """ pass + def showRules(self, req): + """ + Parameters: + - req + + """ + pass + + def setRules(self, req): + """ + Parameters: + - req + + """ + pass + class Client(Iface): def __init__(self, iprot, oprot=None): @@ -1115,6 +1157,38 @@ def recv_loadCSV(self): return result.success raise TApplicationException(TApplicationException.MISSING_RESULT, "loadCSV failed: unknown result") + def loadUDF(self, req): + """ + Parameters: + - req + + """ + self.send_loadUDF(req) + return self.recv_loadUDF() + + def send_loadUDF(self, req): + self._oprot.writeMessageBegin('loadUDF', TMessageType.CALL, self._seqid) + args = loadUDF_args() + args.req = req + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_loadUDF(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = loadUDF_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "loadUDF failed: unknown result") + def closeStatement(self, req): """ Parameters: @@ -1467,6 +1541,70 @@ def recv_showSessionID(self): return result.success raise TApplicationException(TApplicationException.MISSING_RESULT, "showSessionID failed: unknown result") + def showRules(self, req): + """ + Parameters: + - req + + """ + self.send_showRules(req) + return self.recv_showRules() + + def send_showRules(self, req): + self._oprot.writeMessageBegin('showRules', TMessageType.CALL, self._seqid) + args = showRules_args() + args.req = req + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_showRules(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = showRules_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "showRules failed: unknown result") + + def setRules(self, req): + """ + Parameters: + - req + + """ + self.send_setRules(req) + return self.recv_setRules() + + def send_setRules(self, req): + self._oprot.writeMessageBegin('setRules', TMessageType.CALL, self._seqid) + args = setRules_args() + args.req = req + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_setRules(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = setRules_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "setRules failed: unknown result") + class Processor(Iface, TProcessor): def __init__(self, handler): @@ -1497,6 +1635,7 @@ def __init__(self, handler): self._processMap["executeStatement"] = Processor.process_executeStatement self._processMap["fetchResults"] = Processor.process_fetchResults self._processMap["loadCSV"] = Processor.process_loadCSV + self._processMap["loadUDF"] = Processor.process_loadUDF self._processMap["closeStatement"] = Processor.process_closeStatement self._processMap["commitTransformJob"] = Processor.process_commitTransformJob self._processMap["queryTransformJobStatus"] = Processor.process_queryTransformJobStatus @@ -1508,6 +1647,8 @@ def __init__(self, handler): self._processMap["curveMatch"] = Processor.process_curveMatch self._processMap["debugInfo"] = Processor.process_debugInfo self._processMap["showSessionID"] = Processor.process_showSessionID + self._processMap["showRules"] = Processor.process_showRules + self._processMap["setRules"] = Processor.process_setRules self._on_message_begin = None def on_message_begin(self, func): @@ -2105,6 +2246,29 @@ def process_loadCSV(self, seqid, iprot, oprot): oprot.writeMessageEnd() oprot.trans.flush() + def process_loadUDF(self, seqid, iprot, oprot): + args = loadUDF_args() + args.read(iprot) + iprot.readMessageEnd() + result = loadUDF_result() + try: + result.success = self._handler.loadUDF(args.req) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception('TApplication exception in handler') + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception('Unexpected exception in handler') + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') + oprot.writeMessageBegin("loadUDF", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + def process_closeStatement(self, seqid, iprot, oprot): args = closeStatement_args() args.read(iprot) @@ -2358,6 +2522,52 @@ def process_showSessionID(self, seqid, iprot, oprot): oprot.writeMessageEnd() oprot.trans.flush() + def process_showRules(self, seqid, iprot, oprot): + args = showRules_args() + args.read(iprot) + iprot.readMessageEnd() + result = showRules_result() + try: + result.success = self._handler.showRules(args.req) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception('TApplication exception in handler') + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception('Unexpected exception in handler') + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') + oprot.writeMessageBegin("showRules", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_setRules(self, seqid, iprot, oprot): + args = setRules_args() + args.read(iprot) + iprot.readMessageEnd() + result = setRules_result() + try: + result.success = self._handler.setRules(args.req) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception('TApplication exception in handler') + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception('Unexpected exception in handler') + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') + oprot.writeMessageBegin("setRules", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + # HELPER FUNCTIONS AND STRUCTURES @@ -5486,6 +5696,131 @@ def __ne__(self, other): ) +class loadUDF_args(object): + """ + Attributes: + - req + + """ + + + def __init__(self, req=None,): + self.req = req + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.req = LoadUDFReq() + self.req.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('loadUDF_args') + if self.req is not None: + oprot.writeFieldBegin('req', TType.STRUCT, 1) + self.req.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) +all_structs.append(loadUDF_args) +loadUDF_args.thrift_spec = ( + None, # 0 + (1, TType.STRUCT, 'req', [LoadUDFReq, None], None, ), # 1 +) + + +class loadUDF_result(object): + """ + Attributes: + - success + + """ + + + def __init__(self, success=None,): + self.success = success + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = LoadUDFResp() + self.success.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('loadUDF_result') + if self.success is not None: + oprot.writeFieldBegin('success', TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) +all_structs.append(loadUDF_result) +loadUDF_result.thrift_spec = ( + (0, TType.STRUCT, 'success', [LoadUDFResp, None], None, ), # 0 +) + + class closeStatement_args(object): """ Attributes: @@ -6859,5 +7194,255 @@ def __ne__(self, other): showSessionID_result.thrift_spec = ( (0, TType.STRUCT, 'success', [ShowSessionIDResp, None], None, ), # 0 ) + + +class showRules_args(object): + """ + Attributes: + - req + + """ + + + def __init__(self, req=None,): + self.req = req + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.req = ShowRulesReq() + self.req.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('showRules_args') + if self.req is not None: + oprot.writeFieldBegin('req', TType.STRUCT, 1) + self.req.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) +all_structs.append(showRules_args) +showRules_args.thrift_spec = ( + None, # 0 + (1, TType.STRUCT, 'req', [ShowRulesReq, None], None, ), # 1 +) + + +class showRules_result(object): + """ + Attributes: + - success + + """ + + + def __init__(self, success=None,): + self.success = success + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = ShowRulesResp() + self.success.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('showRules_result') + if self.success is not None: + oprot.writeFieldBegin('success', TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) +all_structs.append(showRules_result) +showRules_result.thrift_spec = ( + (0, TType.STRUCT, 'success', [ShowRulesResp, None], None, ), # 0 +) + + +class setRules_args(object): + """ + Attributes: + - req + + """ + + + def __init__(self, req=None,): + self.req = req + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.req = SetRulesReq() + self.req.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('setRules_args') + if self.req is not None: + oprot.writeFieldBegin('req', TType.STRUCT, 1) + self.req.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) +all_structs.append(setRules_args) +setRules_args.thrift_spec = ( + None, # 0 + (1, TType.STRUCT, 'req', [SetRulesReq, None], None, ), # 1 +) + + +class setRules_result(object): + """ + Attributes: + - success + + """ + + + def __init__(self, success=None,): + self.success = success + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = Status() + self.success.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('setRules_result') + if self.success is not None: + oprot.writeFieldBegin('success', TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) +all_structs.append(setRules_result) +setRules_result.thrift_spec = ( + (0, TType.STRUCT, 'success', [Status, None], None, ), # 0 +) fix_spec(all_structs) del all_structs diff --git a/session_py/iginx/iginx_pyclient/thrift/rpc/__init__.py b/session_py/iginx/iginx_pyclient/thrift/rpc/__init__.py index cc24d07ca3..9ed5ba1cbb 100644 --- a/session_py/iginx/iginx_pyclient/thrift/rpc/__init__.py +++ b/session_py/iginx/iginx_pyclient/thrift/rpc/__init__.py @@ -1 +1,19 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + __all__ = ['ttypes', 'constants', 'IService'] diff --git a/session_py/iginx/iginx_pyclient/thrift/rpc/constants.py b/session_py/iginx/iginx_pyclient/thrift/rpc/constants.py index 82c674e052..a4d9713781 100644 --- a/session_py/iginx/iginx_pyclient/thrift/rpc/constants.py +++ b/session_py/iginx/iginx_pyclient/thrift/rpc/constants.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + # # Autogenerated by Thrift Compiler (0.16.0) # diff --git a/session_py/iginx/iginx_pyclient/thrift/rpc/ttypes.py b/session_py/iginx/iginx_pyclient/thrift/rpc/ttypes.py index ab63b9837f..0498f488bf 100644 --- a/session_py/iginx/iginx_pyclient/thrift/rpc/ttypes.py +++ b/session_py/iginx/iginx_pyclient/thrift/rpc/ttypes.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + # # Autogenerated by Thrift Compiler (0.16.0) # @@ -47,7 +65,7 @@ class StorageEngineType(object): iotdb12 = 0 influxdb = 1 parquet = 2 - postgresql = 3 + relational = 3 mongodb = 4 redis = 5 filesystem = 6 @@ -59,7 +77,7 @@ class StorageEngineType(object): 0: "iotdb12", 1: "influxdb", 2: "parquet", - 3: "postgresql", + 3: "relational", 4: "mongodb", 5: "redis", 6: "filesystem", @@ -72,7 +90,7 @@ class StorageEngineType(object): "iotdb12": 0, "influxdb": 1, "parquet": 2, - "postgresql": 3, + "relational": 3, "mongodb": 4, "redis": 5, "filesystem": 6, @@ -145,6 +163,8 @@ class SqlType(object): ExportStream = 23 LoadCsv = 24 ShowSessionID = 25 + ShowRules = 26 + SetRules = 27 _VALUES_TO_NAMES = { 0: "Unknown", @@ -173,6 +193,8 @@ class SqlType(object): 23: "ExportStream", 24: "LoadCsv", 25: "ShowSessionID", + 26: "ShowRules", + 27: "SetRules", } _NAMES_TO_VALUES = { @@ -202,6 +224,8 @@ class SqlType(object): "ExportStream": 23, "LoadCsv": 24, "ShowSessionID": 25, + "ShowRules": 26, + "SetRules": 27, } @@ -511,6 +535,78 @@ def __ne__(self, other): return not (self == other) +class UDFClassPair(object): + """ + Attributes: + - name + - classPath + + """ + + + def __init__(self, name=None, classPath=None,): + self.name = name + self.classPath = classPath + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.name = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.classPath = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('UDFClassPair') + if self.name is not None: + oprot.writeFieldBegin('name', TType.STRING, 1) + oprot.writeString(self.name.encode('utf-8') if sys.version_info[0] == 2 else self.name) + oprot.writeFieldEnd() + if self.classPath is not None: + oprot.writeFieldBegin('classPath', TType.STRING, 2) + oprot.writeString(self.classPath.encode('utf-8') if sys.version_info[0] == 2 else self.classPath) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.name is None: + raise TProtocolException(message='Required field name is unset!') + if self.classPath is None: + raise TProtocolException(message='Required field classPath is unset!') + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + class OpenSessionReq(object): """ Attributes: @@ -3743,14 +3839,16 @@ class ExecuteSqlResp(object): - jobId - jobState - jobIdList - - configValue + - configs - loadCsvPath - sessionIDList + - rules + - UDFModulePath """ - def __init__(self, status=None, type=None, paths=None, tagsList=None, dataTypeList=None, queryDataSet=None, keys=None, valuesList=None, replicaNum=None, pointsNum=None, aggregateType=None, parseErrorMsg=None, limit=None, offset=None, orderByPath=None, ascending=None, iginxInfos=None, storageEngineInfos=None, metaStorageInfos=None, localMetaStorageInfo=None, registerTaskInfos=None, jobId=None, jobState=None, jobIdList=None, configValue=None, loadCsvPath=None, sessionIDList=None,): + def __init__(self, status=None, type=None, paths=None, tagsList=None, dataTypeList=None, queryDataSet=None, keys=None, valuesList=None, replicaNum=None, pointsNum=None, aggregateType=None, parseErrorMsg=None, limit=None, offset=None, orderByPath=None, ascending=None, iginxInfos=None, storageEngineInfos=None, metaStorageInfos=None, localMetaStorageInfo=None, registerTaskInfos=None, jobId=None, jobState=None, jobIdList=None, configs=None, loadCsvPath=None, sessionIDList=None, rules=None, UDFModulePath=None,): self.status = status self.type = type self.paths = paths @@ -3775,9 +3873,11 @@ def __init__(self, status=None, type=None, paths=None, tagsList=None, dataTypeLi self.jobId = jobId self.jobState = jobState self.jobIdList = jobIdList - self.configValue = configValue + self.configs = configs self.loadCsvPath = loadCsvPath self.sessionIDList = sessionIDList + self.rules = rules + self.UDFModulePath = UDFModulePath def read(self, iprot): if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: @@ -3962,8 +4062,14 @@ def read(self, iprot): else: iprot.skip(ftype) elif fid == 25: - if ftype == TType.STRING: - self.configValue = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + if ftype == TType.MAP: + self.configs = {} + (_ktype599, _vtype600, _size598) = iprot.readMapBegin() + for _i602 in range(_size598): + _key603 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + _val604 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.configs[_key603] = _val604 + iprot.readMapEnd() else: iprot.skip(ftype) elif fid == 26: @@ -3974,13 +4080,29 @@ def read(self, iprot): elif fid == 27: if ftype == TType.LIST: self.sessionIDList = [] - (_etype601, _size598) = iprot.readListBegin() - for _i602 in range(_size598): - _elem603 = iprot.readI64() - self.sessionIDList.append(_elem603) + (_etype608, _size605) = iprot.readListBegin() + for _i609 in range(_size605): + _elem610 = iprot.readI64() + self.sessionIDList.append(_elem610) iprot.readListEnd() else: iprot.skip(ftype) + elif fid == 28: + if ftype == TType.MAP: + self.rules = {} + (_ktype612, _vtype613, _size611) = iprot.readMapBegin() + for _i615 in range(_size611): + _key616 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + _val617 = iprot.readBool() + self.rules[_key616] = _val617 + iprot.readMapEnd() + else: + iprot.skip(ftype) + elif fid == 29: + if ftype == TType.STRING: + self.UDFModulePath = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() @@ -4002,26 +4124,26 @@ def write(self, oprot): if self.paths is not None: oprot.writeFieldBegin('paths', TType.LIST, 3) oprot.writeListBegin(TType.STRING, len(self.paths)) - for iter604 in self.paths: - oprot.writeString(iter604.encode('utf-8') if sys.version_info[0] == 2 else iter604) + for iter618 in self.paths: + oprot.writeString(iter618.encode('utf-8') if sys.version_info[0] == 2 else iter618) oprot.writeListEnd() oprot.writeFieldEnd() if self.tagsList is not None: oprot.writeFieldBegin('tagsList', TType.LIST, 4) oprot.writeListBegin(TType.MAP, len(self.tagsList)) - for iter605 in self.tagsList: - oprot.writeMapBegin(TType.STRING, TType.STRING, len(iter605)) - for kiter606, viter607 in iter605.items(): - oprot.writeString(kiter606.encode('utf-8') if sys.version_info[0] == 2 else kiter606) - oprot.writeString(viter607.encode('utf-8') if sys.version_info[0] == 2 else viter607) + for iter619 in self.tagsList: + oprot.writeMapBegin(TType.STRING, TType.STRING, len(iter619)) + for kiter620, viter621 in iter619.items(): + oprot.writeString(kiter620.encode('utf-8') if sys.version_info[0] == 2 else kiter620) + oprot.writeString(viter621.encode('utf-8') if sys.version_info[0] == 2 else viter621) oprot.writeMapEnd() oprot.writeListEnd() oprot.writeFieldEnd() if self.dataTypeList is not None: oprot.writeFieldBegin('dataTypeList', TType.LIST, 5) oprot.writeListBegin(TType.I32, len(self.dataTypeList)) - for iter608 in self.dataTypeList: - oprot.writeI32(iter608) + for iter622 in self.dataTypeList: + oprot.writeI32(iter622) oprot.writeListEnd() oprot.writeFieldEnd() if self.queryDataSet is not None: @@ -4071,22 +4193,22 @@ def write(self, oprot): if self.iginxInfos is not None: oprot.writeFieldBegin('iginxInfos', TType.LIST, 17) oprot.writeListBegin(TType.STRUCT, len(self.iginxInfos)) - for iter609 in self.iginxInfos: - iter609.write(oprot) + for iter623 in self.iginxInfos: + iter623.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.storageEngineInfos is not None: oprot.writeFieldBegin('storageEngineInfos', TType.LIST, 18) oprot.writeListBegin(TType.STRUCT, len(self.storageEngineInfos)) - for iter610 in self.storageEngineInfos: - iter610.write(oprot) + for iter624 in self.storageEngineInfos: + iter624.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.metaStorageInfos is not None: oprot.writeFieldBegin('metaStorageInfos', TType.LIST, 19) oprot.writeListBegin(TType.STRUCT, len(self.metaStorageInfos)) - for iter611 in self.metaStorageInfos: - iter611.write(oprot) + for iter625 in self.metaStorageInfos: + iter625.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.localMetaStorageInfo is not None: @@ -4096,8 +4218,8 @@ def write(self, oprot): if self.registerTaskInfos is not None: oprot.writeFieldBegin('registerTaskInfos', TType.LIST, 21) oprot.writeListBegin(TType.STRUCT, len(self.registerTaskInfos)) - for iter612 in self.registerTaskInfos: - iter612.write(oprot) + for iter626 in self.registerTaskInfos: + iter626.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.jobId is not None: @@ -4111,13 +4233,17 @@ def write(self, oprot): if self.jobIdList is not None: oprot.writeFieldBegin('jobIdList', TType.LIST, 24) oprot.writeListBegin(TType.I64, len(self.jobIdList)) - for iter613 in self.jobIdList: - oprot.writeI64(iter613) + for iter627 in self.jobIdList: + oprot.writeI64(iter627) oprot.writeListEnd() oprot.writeFieldEnd() - if self.configValue is not None: - oprot.writeFieldBegin('configValue', TType.STRING, 25) - oprot.writeString(self.configValue.encode('utf-8') if sys.version_info[0] == 2 else self.configValue) + if self.configs is not None: + oprot.writeFieldBegin('configs', TType.MAP, 25) + oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.configs)) + for kiter628, viter629 in self.configs.items(): + oprot.writeString(kiter628.encode('utf-8') if sys.version_info[0] == 2 else kiter628) + oprot.writeString(viter629.encode('utf-8') if sys.version_info[0] == 2 else viter629) + oprot.writeMapEnd() oprot.writeFieldEnd() if self.loadCsvPath is not None: oprot.writeFieldBegin('loadCsvPath', TType.STRING, 26) @@ -4126,10 +4252,22 @@ def write(self, oprot): if self.sessionIDList is not None: oprot.writeFieldBegin('sessionIDList', TType.LIST, 27) oprot.writeListBegin(TType.I64, len(self.sessionIDList)) - for iter614 in self.sessionIDList: - oprot.writeI64(iter614) + for iter630 in self.sessionIDList: + oprot.writeI64(iter630) oprot.writeListEnd() oprot.writeFieldEnd() + if self.rules is not None: + oprot.writeFieldBegin('rules', TType.MAP, 28) + oprot.writeMapBegin(TType.STRING, TType.BOOL, len(self.rules)) + for kiter631, viter632 in self.rules.items(): + oprot.writeString(kiter631.encode('utf-8') if sys.version_info[0] == 2 else kiter631) + oprot.writeBool(viter632) + oprot.writeMapEnd() + oprot.writeFieldEnd() + if self.UDFModulePath is not None: + oprot.writeFieldBegin('UDFModulePath', TType.STRING, 29) + oprot.writeString(self.UDFModulePath.encode('utf-8') if sys.version_info[0] == 2 else self.UDFModulePath) + oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() @@ -4196,10 +4334,10 @@ def read(self, iprot): elif fid == 4: if ftype == TType.SET: self.auths = set() - (_etype618, _size615) = iprot.readSetBegin() - for _i619 in range(_size615): - _elem620 = iprot.readI32() - self.auths.add(_elem620) + (_etype636, _size633) = iprot.readSetBegin() + for _i637 in range(_size633): + _elem638 = iprot.readI32() + self.auths.add(_elem638) iprot.readSetEnd() else: iprot.skip(ftype) @@ -4228,8 +4366,8 @@ def write(self, oprot): if self.auths is not None: oprot.writeFieldBegin('auths', TType.SET, 4) oprot.writeSetBegin(TType.I32, len(self.auths)) - for iter621 in self.auths: - oprot.writeI32(iter621) + for iter639 in self.auths: + oprot.writeI32(iter639) oprot.writeSetEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -4298,10 +4436,10 @@ def read(self, iprot): elif fid == 4: if ftype == TType.SET: self.auths = set() - (_etype625, _size622) = iprot.readSetBegin() - for _i626 in range(_size622): - _elem627 = iprot.readI32() - self.auths.add(_elem627) + (_etype643, _size640) = iprot.readSetBegin() + for _i644 in range(_size640): + _elem645 = iprot.readI32() + self.auths.add(_elem645) iprot.readSetEnd() else: iprot.skip(ftype) @@ -4330,8 +4468,8 @@ def write(self, oprot): if self.auths is not None: oprot.writeFieldBegin('auths', TType.SET, 4) oprot.writeSetBegin(TType.I32, len(self.auths)) - for iter628 in self.auths: - oprot.writeI32(iter628) + for iter646 in self.auths: + oprot.writeI32(iter646) oprot.writeSetEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -4462,10 +4600,10 @@ def read(self, iprot): elif fid == 2: if ftype == TType.LIST: self.usernames = [] - (_etype632, _size629) = iprot.readListBegin() - for _i633 in range(_size629): - _elem634 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - self.usernames.append(_elem634) + (_etype650, _size647) = iprot.readListBegin() + for _i651 in range(_size647): + _elem652 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.usernames.append(_elem652) iprot.readListEnd() else: iprot.skip(ftype) @@ -4486,8 +4624,8 @@ def write(self, oprot): if self.usernames is not None: oprot.writeFieldBegin('usernames', TType.LIST, 2) oprot.writeListBegin(TType.STRING, len(self.usernames)) - for iter635 in self.usernames: - oprot.writeString(iter635.encode('utf-8') if sys.version_info[0] == 2 else iter635) + for iter653 in self.usernames: + oprot.writeString(iter653.encode('utf-8') if sys.version_info[0] == 2 else iter653) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -4545,35 +4683,35 @@ def read(self, iprot): elif fid == 2: if ftype == TType.LIST: self.usernames = [] - (_etype639, _size636) = iprot.readListBegin() - for _i640 in range(_size636): - _elem641 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - self.usernames.append(_elem641) + (_etype657, _size654) = iprot.readListBegin() + for _i658 in range(_size654): + _elem659 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.usernames.append(_elem659) iprot.readListEnd() else: iprot.skip(ftype) elif fid == 3: if ftype == TType.LIST: self.userTypes = [] - (_etype645, _size642) = iprot.readListBegin() - for _i646 in range(_size642): - _elem647 = iprot.readI32() - self.userTypes.append(_elem647) + (_etype663, _size660) = iprot.readListBegin() + for _i664 in range(_size660): + _elem665 = iprot.readI32() + self.userTypes.append(_elem665) iprot.readListEnd() else: iprot.skip(ftype) elif fid == 4: if ftype == TType.LIST: self.auths = [] - (_etype651, _size648) = iprot.readListBegin() - for _i652 in range(_size648): - _elem653 = set() - (_etype657, _size654) = iprot.readSetBegin() - for _i658 in range(_size654): - _elem659 = iprot.readI32() - _elem653.add(_elem659) + (_etype669, _size666) = iprot.readListBegin() + for _i670 in range(_size666): + _elem671 = set() + (_etype675, _size672) = iprot.readSetBegin() + for _i676 in range(_size672): + _elem677 = iprot.readI32() + _elem671.add(_elem677) iprot.readSetEnd() - self.auths.append(_elem653) + self.auths.append(_elem671) iprot.readListEnd() else: iprot.skip(ftype) @@ -4594,24 +4732,24 @@ def write(self, oprot): if self.usernames is not None: oprot.writeFieldBegin('usernames', TType.LIST, 2) oprot.writeListBegin(TType.STRING, len(self.usernames)) - for iter660 in self.usernames: - oprot.writeString(iter660.encode('utf-8') if sys.version_info[0] == 2 else iter660) + for iter678 in self.usernames: + oprot.writeString(iter678.encode('utf-8') if sys.version_info[0] == 2 else iter678) oprot.writeListEnd() oprot.writeFieldEnd() if self.userTypes is not None: oprot.writeFieldBegin('userTypes', TType.LIST, 3) oprot.writeListBegin(TType.I32, len(self.userTypes)) - for iter661 in self.userTypes: - oprot.writeI32(iter661) + for iter679 in self.userTypes: + oprot.writeI32(iter679) oprot.writeListEnd() oprot.writeFieldEnd() if self.auths is not None: oprot.writeFieldBegin('auths', TType.LIST, 4) oprot.writeListBegin(TType.SET, len(self.auths)) - for iter662 in self.auths: - oprot.writeSetBegin(TType.I32, len(iter662)) - for iter663 in iter662: - oprot.writeI32(iter663) + for iter680 in self.auths: + oprot.writeSetBegin(TType.I32, len(iter680)) + for iter681 in iter680: + oprot.writeI32(iter681) oprot.writeSetEnd() oprot.writeListEnd() oprot.writeFieldEnd() @@ -5080,33 +5218,33 @@ def read(self, iprot): elif fid == 2: if ftype == TType.LIST: self.iginxInfos = [] - (_etype667, _size664) = iprot.readListBegin() - for _i668 in range(_size664): - _elem669 = IginxInfo() - _elem669.read(iprot) - self.iginxInfos.append(_elem669) + (_etype685, _size682) = iprot.readListBegin() + for _i686 in range(_size682): + _elem687 = IginxInfo() + _elem687.read(iprot) + self.iginxInfos.append(_elem687) iprot.readListEnd() else: iprot.skip(ftype) elif fid == 3: if ftype == TType.LIST: self.storageEngineInfos = [] - (_etype673, _size670) = iprot.readListBegin() - for _i674 in range(_size670): - _elem675 = StorageEngineInfo() - _elem675.read(iprot) - self.storageEngineInfos.append(_elem675) + (_etype691, _size688) = iprot.readListBegin() + for _i692 in range(_size688): + _elem693 = StorageEngineInfo() + _elem693.read(iprot) + self.storageEngineInfos.append(_elem693) iprot.readListEnd() else: iprot.skip(ftype) elif fid == 4: if ftype == TType.LIST: self.metaStorageInfos = [] - (_etype679, _size676) = iprot.readListBegin() - for _i680 in range(_size676): - _elem681 = MetaStorageInfo() - _elem681.read(iprot) - self.metaStorageInfos.append(_elem681) + (_etype697, _size694) = iprot.readListBegin() + for _i698 in range(_size694): + _elem699 = MetaStorageInfo() + _elem699.read(iprot) + self.metaStorageInfos.append(_elem699) iprot.readListEnd() else: iprot.skip(ftype) @@ -5133,22 +5271,22 @@ def write(self, oprot): if self.iginxInfos is not None: oprot.writeFieldBegin('iginxInfos', TType.LIST, 2) oprot.writeListBegin(TType.STRUCT, len(self.iginxInfos)) - for iter682 in self.iginxInfos: - iter682.write(oprot) + for iter700 in self.iginxInfos: + iter700.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.storageEngineInfos is not None: oprot.writeFieldBegin('storageEngineInfos', TType.LIST, 3) oprot.writeListBegin(TType.STRUCT, len(self.storageEngineInfos)) - for iter683 in self.storageEngineInfos: - iter683.write(oprot) + for iter701 in self.storageEngineInfos: + iter701.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.metaStorageInfos is not None: oprot.writeFieldBegin('metaStorageInfos', TType.LIST, 4) oprot.writeListBegin(TType.STRUCT, len(self.metaStorageInfos)) - for iter684 in self.metaStorageInfos: - iter684.write(oprot) + for iter702 in self.metaStorageInfos: + iter702.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.localMetaStorageInfo is not None: @@ -5326,36 +5464,36 @@ def read(self, iprot): elif fid == 4: if ftype == TType.LIST: self.columns = [] - (_etype688, _size685) = iprot.readListBegin() - for _i689 in range(_size685): - _elem690 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - self.columns.append(_elem690) + (_etype706, _size703) = iprot.readListBegin() + for _i707 in range(_size703): + _elem708 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.columns.append(_elem708) iprot.readListEnd() else: iprot.skip(ftype) elif fid == 5: if ftype == TType.LIST: self.tagsList = [] - (_etype694, _size691) = iprot.readListBegin() - for _i695 in range(_size691): - _elem696 = {} - (_ktype698, _vtype699, _size697) = iprot.readMapBegin() - for _i701 in range(_size697): - _key702 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - _val703 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - _elem696[_key702] = _val703 + (_etype712, _size709) = iprot.readListBegin() + for _i713 in range(_size709): + _elem714 = {} + (_ktype716, _vtype717, _size715) = iprot.readMapBegin() + for _i719 in range(_size715): + _key720 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + _val721 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + _elem714[_key720] = _val721 iprot.readMapEnd() - self.tagsList.append(_elem696) + self.tagsList.append(_elem714) iprot.readListEnd() else: iprot.skip(ftype) elif fid == 6: if ftype == TType.LIST: self.dataTypeList = [] - (_etype707, _size704) = iprot.readListBegin() - for _i708 in range(_size704): - _elem709 = iprot.readI32() - self.dataTypeList.append(_elem709) + (_etype725, _size722) = iprot.readListBegin() + for _i726 in range(_size722): + _elem727 = iprot.readI32() + self.dataTypeList.append(_elem727) iprot.readListEnd() else: iprot.skip(ftype) @@ -5406,26 +5544,26 @@ def write(self, oprot): if self.columns is not None: oprot.writeFieldBegin('columns', TType.LIST, 4) oprot.writeListBegin(TType.STRING, len(self.columns)) - for iter710 in self.columns: - oprot.writeString(iter710.encode('utf-8') if sys.version_info[0] == 2 else iter710) + for iter728 in self.columns: + oprot.writeString(iter728.encode('utf-8') if sys.version_info[0] == 2 else iter728) oprot.writeListEnd() oprot.writeFieldEnd() if self.tagsList is not None: oprot.writeFieldBegin('tagsList', TType.LIST, 5) oprot.writeListBegin(TType.MAP, len(self.tagsList)) - for iter711 in self.tagsList: - oprot.writeMapBegin(TType.STRING, TType.STRING, len(iter711)) - for kiter712, viter713 in iter711.items(): - oprot.writeString(kiter712.encode('utf-8') if sys.version_info[0] == 2 else kiter712) - oprot.writeString(viter713.encode('utf-8') if sys.version_info[0] == 2 else viter713) + for iter729 in self.tagsList: + oprot.writeMapBegin(TType.STRING, TType.STRING, len(iter729)) + for kiter730, viter731 in iter729.items(): + oprot.writeString(kiter730.encode('utf-8') if sys.version_info[0] == 2 else kiter730) + oprot.writeString(viter731.encode('utf-8') if sys.version_info[0] == 2 else viter731) oprot.writeMapEnd() oprot.writeListEnd() oprot.writeFieldEnd() if self.dataTypeList is not None: oprot.writeFieldBegin('dataTypeList', TType.LIST, 6) oprot.writeListBegin(TType.I32, len(self.dataTypeList)) - for iter714 in self.dataTypeList: - oprot.writeI32(iter714) + for iter732 in self.dataTypeList: + oprot.writeI32(iter732) oprot.writeListEnd() oprot.writeFieldEnd() if self.queryDataSet is not None: @@ -5628,20 +5766,20 @@ def read(self, iprot): if fid == 1: if ftype == TType.LIST: self.valuesList = [] - (_etype718, _size715) = iprot.readListBegin() - for _i719 in range(_size715): - _elem720 = iprot.readBinary() - self.valuesList.append(_elem720) + (_etype736, _size733) = iprot.readListBegin() + for _i737 in range(_size733): + _elem738 = iprot.readBinary() + self.valuesList.append(_elem738) iprot.readListEnd() else: iprot.skip(ftype) elif fid == 2: if ftype == TType.LIST: self.bitmapList = [] - (_etype724, _size721) = iprot.readListBegin() - for _i725 in range(_size721): - _elem726 = iprot.readBinary() - self.bitmapList.append(_elem726) + (_etype742, _size739) = iprot.readListBegin() + for _i743 in range(_size739): + _elem744 = iprot.readBinary() + self.bitmapList.append(_elem744) iprot.readListEnd() else: iprot.skip(ftype) @@ -5658,15 +5796,15 @@ def write(self, oprot): if self.valuesList is not None: oprot.writeFieldBegin('valuesList', TType.LIST, 1) oprot.writeListBegin(TType.STRING, len(self.valuesList)) - for iter727 in self.valuesList: - oprot.writeBinary(iter727) + for iter745 in self.valuesList: + oprot.writeBinary(iter745) oprot.writeListEnd() oprot.writeFieldEnd() if self.bitmapList is not None: oprot.writeFieldBegin('bitmapList', TType.LIST, 2) oprot.writeListBegin(TType.STRING, len(self.bitmapList)) - for iter728 in self.bitmapList: - oprot.writeBinary(iter728) + for iter746 in self.bitmapList: + oprot.writeBinary(iter746) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -6062,10 +6200,10 @@ def read(self, iprot): elif fid == 2: if ftype == TType.LIST: self.columns = [] - (_etype732, _size729) = iprot.readListBegin() - for _i733 in range(_size729): - _elem734 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - self.columns.append(_elem734) + (_etype750, _size747) = iprot.readListBegin() + for _i751 in range(_size747): + _elem752 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.columns.append(_elem752) iprot.readListEnd() else: iprot.skip(ftype) @@ -6096,8 +6234,8 @@ def write(self, oprot): if self.columns is not None: oprot.writeFieldBegin('columns', TType.LIST, 2) oprot.writeListBegin(TType.STRING, len(self.columns)) - for iter735 in self.columns: - oprot.writeString(iter735.encode('utf-8') if sys.version_info[0] == 2 else iter735) + for iter753 in self.columns: + oprot.writeString(iter753.encode('utf-8') if sys.version_info[0] == 2 else iter753) oprot.writeListEnd() oprot.writeFieldEnd() if self.recordsNum is not None: @@ -6128,6 +6266,184 @@ def __ne__(self, other): return not (self == other) +class LoadUDFReq(object): + """ + Attributes: + - sessionId + - statement + - udfFile + - isRemote + + """ + + + def __init__(self, sessionId=None, statement=None, udfFile=None, isRemote=None,): + self.sessionId = sessionId + self.statement = statement + self.udfFile = udfFile + self.isRemote = isRemote + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I64: + self.sessionId = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.statement = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.udfFile = iprot.readBinary() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.BOOL: + self.isRemote = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('LoadUDFReq') + if self.sessionId is not None: + oprot.writeFieldBegin('sessionId', TType.I64, 1) + oprot.writeI64(self.sessionId) + oprot.writeFieldEnd() + if self.statement is not None: + oprot.writeFieldBegin('statement', TType.STRING, 2) + oprot.writeString(self.statement.encode('utf-8') if sys.version_info[0] == 2 else self.statement) + oprot.writeFieldEnd() + if self.udfFile is not None: + oprot.writeFieldBegin('udfFile', TType.STRING, 3) + oprot.writeBinary(self.udfFile) + oprot.writeFieldEnd() + if self.isRemote is not None: + oprot.writeFieldBegin('isRemote', TType.BOOL, 4) + oprot.writeBool(self.isRemote) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.sessionId is None: + raise TProtocolException(message='Required field sessionId is unset!') + if self.statement is None: + raise TProtocolException(message='Required field statement is unset!') + if self.isRemote is None: + raise TProtocolException(message='Required field isRemote is unset!') + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class LoadUDFResp(object): + """ + Attributes: + - status + - parseErrorMsg + - UDFModulePath + + """ + + + def __init__(self, status=None, parseErrorMsg=None, UDFModulePath=None,): + self.status = status + self.parseErrorMsg = parseErrorMsg + self.UDFModulePath = UDFModulePath + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.status = Status() + self.status.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.parseErrorMsg = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.UDFModulePath = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('LoadUDFResp') + if self.status is not None: + oprot.writeFieldBegin('status', TType.STRUCT, 1) + self.status.write(oprot) + oprot.writeFieldEnd() + if self.parseErrorMsg is not None: + oprot.writeFieldBegin('parseErrorMsg', TType.STRING, 2) + oprot.writeString(self.parseErrorMsg.encode('utf-8') if sys.version_info[0] == 2 else self.parseErrorMsg) + oprot.writeFieldEnd() + if self.UDFModulePath is not None: + oprot.writeFieldBegin('UDFModulePath', TType.STRING, 3) + oprot.writeString(self.UDFModulePath.encode('utf-8') if sys.version_info[0] == 2 else self.UDFModulePath) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.status is None: + raise TProtocolException(message='Required field status is unset!') + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + class TaskInfo(object): """ Attributes: @@ -6174,10 +6490,10 @@ def read(self, iprot): elif fid == 4: if ftype == TType.LIST: self.sqlList = [] - (_etype739, _size736) = iprot.readListBegin() - for _i740 in range(_size736): - _elem741 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - self.sqlList.append(_elem741) + (_etype757, _size754) = iprot.readListBegin() + for _i758 in range(_size754): + _elem759 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.sqlList.append(_elem759) iprot.readListEnd() else: iprot.skip(ftype) @@ -6211,8 +6527,8 @@ def write(self, oprot): if self.sqlList is not None: oprot.writeFieldBegin('sqlList', TType.LIST, 4) oprot.writeListBegin(TType.STRING, len(self.sqlList)) - for iter742 in self.sqlList: - oprot.writeString(iter742.encode('utf-8') if sys.version_info[0] == 2 else iter742) + for iter760 in self.sqlList: + oprot.writeString(iter760.encode('utf-8') if sys.version_info[0] == 2 else iter760) oprot.writeListEnd() oprot.writeFieldEnd() if self.pyTaskName is not None: @@ -6275,11 +6591,11 @@ def read(self, iprot): elif fid == 2: if ftype == TType.LIST: self.taskList = [] - (_etype746, _size743) = iprot.readListBegin() - for _i747 in range(_size743): - _elem748 = TaskInfo() - _elem748.read(iprot) - self.taskList.append(_elem748) + (_etype764, _size761) = iprot.readListBegin() + for _i765 in range(_size761): + _elem766 = TaskInfo() + _elem766.read(iprot) + self.taskList.append(_elem766) iprot.readListEnd() else: iprot.skip(ftype) @@ -6310,8 +6626,8 @@ def write(self, oprot): if self.taskList is not None: oprot.writeFieldBegin('taskList', TType.LIST, 2) oprot.writeListBegin(TType.STRUCT, len(self.taskList)) - for iter749 in self.taskList: - iter749.write(oprot) + for iter767 in self.taskList: + iter767.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.exportType is not None: @@ -6667,10 +6983,10 @@ def read(self, iprot): elif fid == 2: if ftype == TType.LIST: self.jobIdList = [] - (_etype753, _size750) = iprot.readListBegin() - for _i754 in range(_size750): - _elem755 = iprot.readI64() - self.jobIdList.append(_elem755) + (_etype771, _size768) = iprot.readListBegin() + for _i772 in range(_size768): + _elem773 = iprot.readI64() + self.jobIdList.append(_elem773) iprot.readListEnd() else: iprot.skip(ftype) @@ -6691,8 +7007,8 @@ def write(self, oprot): if self.jobIdList is not None: oprot.writeFieldBegin('jobIdList', TType.LIST, 2) oprot.writeListBegin(TType.I64, len(self.jobIdList)) - for iter756 in self.jobIdList: - oprot.writeI64(iter756) + for iter774 in self.jobIdList: + oprot.writeI64(iter774) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -6793,20 +7109,22 @@ class RegisterTaskReq(object): """ Attributes: - sessionId - - name - filePath - - className - - type + - UDFClassPairs + - types + - moduleFile + - isRemote """ - def __init__(self, sessionId=None, name=None, filePath=None, className=None, type=None,): + def __init__(self, sessionId=None, filePath=None, UDFClassPairs=None, types=None, moduleFile=None, isRemote=None,): self.sessionId = sessionId - self.name = name self.filePath = filePath - self.className = className - self.type = type + self.UDFClassPairs = UDFClassPairs + self.types = types + self.moduleFile = moduleFile + self.isRemote = isRemote def read(self, iprot): if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: @@ -6824,22 +7142,38 @@ def read(self, iprot): iprot.skip(ftype) elif fid == 2: if ftype == TType.STRING: - self.name = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.filePath = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() else: iprot.skip(ftype) elif fid == 3: - if ftype == TType.STRING: - self.filePath = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + if ftype == TType.LIST: + self.UDFClassPairs = [] + (_etype778, _size775) = iprot.readListBegin() + for _i779 in range(_size775): + _elem780 = UDFClassPair() + _elem780.read(iprot) + self.UDFClassPairs.append(_elem780) + iprot.readListEnd() else: iprot.skip(ftype) elif fid == 4: - if ftype == TType.STRING: - self.className = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + if ftype == TType.LIST: + self.types = [] + (_etype784, _size781) = iprot.readListBegin() + for _i785 in range(_size781): + _elem786 = iprot.readI32() + self.types.append(_elem786) + iprot.readListEnd() else: iprot.skip(ftype) elif fid == 5: - if ftype == TType.I32: - self.type = iprot.readI32() + if ftype == TType.STRING: + self.moduleFile = iprot.readBinary() + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.BOOL: + self.isRemote = iprot.readBool() else: iprot.skip(ftype) else: @@ -6856,21 +7190,31 @@ def write(self, oprot): oprot.writeFieldBegin('sessionId', TType.I64, 1) oprot.writeI64(self.sessionId) oprot.writeFieldEnd() - if self.name is not None: - oprot.writeFieldBegin('name', TType.STRING, 2) - oprot.writeString(self.name.encode('utf-8') if sys.version_info[0] == 2 else self.name) - oprot.writeFieldEnd() if self.filePath is not None: - oprot.writeFieldBegin('filePath', TType.STRING, 3) + oprot.writeFieldBegin('filePath', TType.STRING, 2) oprot.writeString(self.filePath.encode('utf-8') if sys.version_info[0] == 2 else self.filePath) oprot.writeFieldEnd() - if self.className is not None: - oprot.writeFieldBegin('className', TType.STRING, 4) - oprot.writeString(self.className.encode('utf-8') if sys.version_info[0] == 2 else self.className) + if self.UDFClassPairs is not None: + oprot.writeFieldBegin('UDFClassPairs', TType.LIST, 3) + oprot.writeListBegin(TType.STRUCT, len(self.UDFClassPairs)) + for iter787 in self.UDFClassPairs: + iter787.write(oprot) + oprot.writeListEnd() oprot.writeFieldEnd() - if self.type is not None: - oprot.writeFieldBegin('type', TType.I32, 5) - oprot.writeI32(self.type) + if self.types is not None: + oprot.writeFieldBegin('types', TType.LIST, 4) + oprot.writeListBegin(TType.I32, len(self.types)) + for iter788 in self.types: + oprot.writeI32(iter788) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.moduleFile is not None: + oprot.writeFieldBegin('moduleFile', TType.STRING, 5) + oprot.writeBinary(self.moduleFile) + oprot.writeFieldEnd() + if self.isRemote is not None: + oprot.writeFieldBegin('isRemote', TType.BOOL, 6) + oprot.writeBool(self.isRemote) oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() @@ -6878,14 +7222,16 @@ def write(self, oprot): def validate(self): if self.sessionId is None: raise TProtocolException(message='Required field sessionId is unset!') - if self.name is None: - raise TProtocolException(message='Required field name is unset!') if self.filePath is None: raise TProtocolException(message='Required field filePath is unset!') - if self.className is None: - raise TProtocolException(message='Required field className is unset!') - if self.type is None: - raise TProtocolException(message='Required field type is unset!') + if self.UDFClassPairs is None: + raise TProtocolException(message='Required field UDFClassPairs is unset!') + if self.types is None: + raise TProtocolException(message='Required field types is unset!') + if self.moduleFile is None: + raise TProtocolException(message='Required field moduleFile is unset!') + if self.isRemote is None: + raise TProtocolException(message='Required field isRemote is unset!') return def __repr__(self): @@ -7173,11 +7519,11 @@ def read(self, iprot): elif fid == 2: if ftype == TType.LIST: self.registerTaskInfoList = [] - (_etype760, _size757) = iprot.readListBegin() - for _i761 in range(_size757): - _elem762 = RegisterTaskInfo() - _elem762.read(iprot) - self.registerTaskInfoList.append(_elem762) + (_etype792, _size789) = iprot.readListBegin() + for _i793 in range(_size789): + _elem794 = RegisterTaskInfo() + _elem794.read(iprot) + self.registerTaskInfoList.append(_elem794) iprot.readListEnd() else: iprot.skip(ftype) @@ -7198,8 +7544,8 @@ def write(self, oprot): if self.registerTaskInfoList is not None: oprot.writeFieldBegin('registerTaskInfoList', TType.LIST, 2) oprot.writeListBegin(TType.STRUCT, len(self.registerTaskInfoList)) - for iter763 in self.registerTaskInfoList: - iter763.write(oprot) + for iter795 in self.registerTaskInfoList: + iter795.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -7260,10 +7606,10 @@ def read(self, iprot): elif fid == 2: if ftype == TType.LIST: self.paths = [] - (_etype767, _size764) = iprot.readListBegin() - for _i768 in range(_size764): - _elem769 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - self.paths.append(_elem769) + (_etype799, _size796) = iprot.readListBegin() + for _i800 in range(_size796): + _elem801 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.paths.append(_elem801) iprot.readListEnd() else: iprot.skip(ftype) @@ -7280,10 +7626,10 @@ def read(self, iprot): elif fid == 5: if ftype == TType.LIST: self.curveQuery = [] - (_etype773, _size770) = iprot.readListBegin() - for _i774 in range(_size770): - _elem775 = iprot.readDouble() - self.curveQuery.append(_elem775) + (_etype805, _size802) = iprot.readListBegin() + for _i806 in range(_size802): + _elem807 = iprot.readDouble() + self.curveQuery.append(_elem807) iprot.readListEnd() else: iprot.skip(ftype) @@ -7309,8 +7655,8 @@ def write(self, oprot): if self.paths is not None: oprot.writeFieldBegin('paths', TType.LIST, 2) oprot.writeListBegin(TType.STRING, len(self.paths)) - for iter776 in self.paths: - oprot.writeString(iter776.encode('utf-8') if sys.version_info[0] == 2 else iter776) + for iter808 in self.paths: + oprot.writeString(iter808.encode('utf-8') if sys.version_info[0] == 2 else iter808) oprot.writeListEnd() oprot.writeFieldEnd() if self.startKey is not None: @@ -7324,8 +7670,8 @@ def write(self, oprot): if self.curveQuery is not None: oprot.writeFieldBegin('curveQuery', TType.LIST, 5) oprot.writeListBegin(TType.DOUBLE, len(self.curveQuery)) - for iter777 in self.curveQuery: - oprot.writeDouble(iter777) + for iter809 in self.curveQuery: + oprot.writeDouble(iter809) oprot.writeListEnd() oprot.writeFieldEnd() if self.curveUnit is not None: @@ -7824,33 +8170,33 @@ def read(self, iprot): if fid == 1: if ftype == TType.LIST: self.fragments = [] - (_etype781, _size778) = iprot.readListBegin() - for _i782 in range(_size778): - _elem783 = Fragment() - _elem783.read(iprot) - self.fragments.append(_elem783) + (_etype813, _size810) = iprot.readListBegin() + for _i814 in range(_size810): + _elem815 = Fragment() + _elem815.read(iprot) + self.fragments.append(_elem815) iprot.readListEnd() else: iprot.skip(ftype) elif fid == 2: if ftype == TType.LIST: self.storages = [] - (_etype787, _size784) = iprot.readListBegin() - for _i788 in range(_size784): - _elem789 = Storage() - _elem789.read(iprot) - self.storages.append(_elem789) + (_etype819, _size816) = iprot.readListBegin() + for _i820 in range(_size816): + _elem821 = Storage() + _elem821.read(iprot) + self.storages.append(_elem821) iprot.readListEnd() else: iprot.skip(ftype) elif fid == 3: if ftype == TType.LIST: self.storageUnits = [] - (_etype793, _size790) = iprot.readListBegin() - for _i794 in range(_size790): - _elem795 = StorageUnit() - _elem795.read(iprot) - self.storageUnits.append(_elem795) + (_etype825, _size822) = iprot.readListBegin() + for _i826 in range(_size822): + _elem827 = StorageUnit() + _elem827.read(iprot) + self.storageUnits.append(_elem827) iprot.readListEnd() else: iprot.skip(ftype) @@ -7867,22 +8213,22 @@ def write(self, oprot): if self.fragments is not None: oprot.writeFieldBegin('fragments', TType.LIST, 1) oprot.writeListBegin(TType.STRUCT, len(self.fragments)) - for iter796 in self.fragments: - iter796.write(oprot) + for iter828 in self.fragments: + iter828.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.storages is not None: oprot.writeFieldBegin('storages', TType.LIST, 2) oprot.writeListBegin(TType.STRUCT, len(self.storages)) - for iter797 in self.storages: - iter797.write(oprot) + for iter829 in self.storages: + iter829.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.storageUnits is not None: oprot.writeFieldBegin('storageUnits', TType.LIST, 3) oprot.writeListBegin(TType.STRUCT, len(self.storageUnits)) - for iter798 in self.storageUnits: - iter798.write(oprot) + for iter830 in self.storageUnits: + iter830.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -8180,11 +8526,11 @@ def read(self, iprot): elif fid == 2: if ftype == TType.LIST: self.removedStorageEngineInfoList = [] - (_etype802, _size799) = iprot.readListBegin() - for _i803 in range(_size799): - _elem804 = RemovedStorageEngineInfo() - _elem804.read(iprot) - self.removedStorageEngineInfoList.append(_elem804) + (_etype834, _size831) = iprot.readListBegin() + for _i835 in range(_size831): + _elem836 = RemovedStorageEngineInfo() + _elem836.read(iprot) + self.removedStorageEngineInfoList.append(_elem836) iprot.readListEnd() else: iprot.skip(ftype) @@ -8205,8 +8551,8 @@ def write(self, oprot): if self.removedStorageEngineInfoList is not None: oprot.writeFieldBegin('removedStorageEngineInfoList', TType.LIST, 2) oprot.writeListBegin(TType.STRUCT, len(self.removedStorageEngineInfoList)) - for iter805 in self.removedStorageEngineInfoList: - iter805.write(oprot) + for iter837 in self.removedStorageEngineInfoList: + iter837.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -8321,10 +8667,10 @@ def read(self, iprot): elif fid == 2: if ftype == TType.LIST: self.sessionIDList = [] - (_etype809, _size806) = iprot.readListBegin() - for _i810 in range(_size806): - _elem811 = iprot.readI64() - self.sessionIDList.append(_elem811) + (_etype841, _size838) = iprot.readListBegin() + for _i842 in range(_size838): + _elem843 = iprot.readI64() + self.sessionIDList.append(_elem843) iprot.readListEnd() else: iprot.skip(ftype) @@ -8345,8 +8691,8 @@ def write(self, oprot): if self.sessionIDList is not None: oprot.writeFieldBegin('sessionIDList', TType.LIST, 2) oprot.writeListBegin(TType.I64, len(self.sessionIDList)) - for iter812 in self.sessionIDList: - oprot.writeI64(iter812) + for iter844 in self.sessionIDList: + oprot.writeI64(iter844) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -8369,6 +8715,230 @@ def __eq__(self, other): def __ne__(self, other): return not (self == other) + + +class ShowRulesReq(object): + """ + Attributes: + - sessionId + + """ + + + def __init__(self, sessionId=None,): + self.sessionId = sessionId + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I64: + self.sessionId = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('ShowRulesReq') + if self.sessionId is not None: + oprot.writeFieldBegin('sessionId', TType.I64, 1) + oprot.writeI64(self.sessionId) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.sessionId is None: + raise TProtocolException(message='Required field sessionId is unset!') + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class ShowRulesResp(object): + """ + Attributes: + - status + - rules + + """ + + + def __init__(self, status=None, rules=None,): + self.status = status + self.rules = rules + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.status = Status() + self.status.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.MAP: + self.rules = {} + (_ktype846, _vtype847, _size845) = iprot.readMapBegin() + for _i849 in range(_size845): + _key850 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + _val851 = iprot.readBool() + self.rules[_key850] = _val851 + iprot.readMapEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('ShowRulesResp') + if self.status is not None: + oprot.writeFieldBegin('status', TType.STRUCT, 1) + self.status.write(oprot) + oprot.writeFieldEnd() + if self.rules is not None: + oprot.writeFieldBegin('rules', TType.MAP, 2) + oprot.writeMapBegin(TType.STRING, TType.BOOL, len(self.rules)) + for kiter852, viter853 in self.rules.items(): + oprot.writeString(kiter852.encode('utf-8') if sys.version_info[0] == 2 else kiter852) + oprot.writeBool(viter853) + oprot.writeMapEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.status is None: + raise TProtocolException(message='Required field status is unset!') + if self.rules is None: + raise TProtocolException(message='Required field rules is unset!') + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class SetRulesReq(object): + """ + Attributes: + - sessionId + - rulesChange + + """ + + + def __init__(self, sessionId=None, rulesChange=None,): + self.sessionId = sessionId + self.rulesChange = rulesChange + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I64: + self.sessionId = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.MAP: + self.rulesChange = {} + (_ktype855, _vtype856, _size854) = iprot.readMapBegin() + for _i858 in range(_size854): + _key859 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + _val860 = iprot.readBool() + self.rulesChange[_key859] = _val860 + iprot.readMapEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('SetRulesReq') + if self.sessionId is not None: + oprot.writeFieldBegin('sessionId', TType.I64, 1) + oprot.writeI64(self.sessionId) + oprot.writeFieldEnd() + if self.rulesChange is not None: + oprot.writeFieldBegin('rulesChange', TType.MAP, 2) + oprot.writeMapBegin(TType.STRING, TType.BOOL, len(self.rulesChange)) + for kiter861, viter862 in self.rulesChange.items(): + oprot.writeString(kiter861.encode('utf-8') if sys.version_info[0] == 2 else kiter861) + oprot.writeBool(viter862) + oprot.writeMapEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.sessionId is None: + raise TProtocolException(message='Required field sessionId is unset!') + if self.rulesChange is None: + raise TProtocolException(message='Required field rulesChange is unset!') + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) all_structs.append(Status) Status.thrift_spec = ( None, # 0 @@ -8376,6 +8946,12 @@ def __ne__(self, other): (2, TType.STRING, 'message', 'UTF8', None, ), # 2 (3, TType.LIST, 'subStatus', (TType.STRUCT, [Status, None], False), None, ), # 3 ) +all_structs.append(UDFClassPair) +UDFClassPair.thrift_spec = ( + None, # 0 + (1, TType.STRING, 'name', 'UTF8', None, ), # 1 + (2, TType.STRING, 'classPath', 'UTF8', None, ), # 2 +) all_structs.append(OpenSessionReq) OpenSessionReq.thrift_spec = ( None, # 0 @@ -8621,9 +9197,11 @@ def __ne__(self, other): (22, TType.I64, 'jobId', None, None, ), # 22 (23, TType.I32, 'jobState', None, None, ), # 23 (24, TType.LIST, 'jobIdList', (TType.I64, None, False), None, ), # 24 - (25, TType.STRING, 'configValue', 'UTF8', None, ), # 25 + (25, TType.MAP, 'configs', (TType.STRING, 'UTF8', TType.STRING, 'UTF8', False), None, ), # 25 (26, TType.STRING, 'loadCsvPath', 'UTF8', None, ), # 26 (27, TType.LIST, 'sessionIDList', (TType.I64, None, False), None, ), # 27 + (28, TType.MAP, 'rules', (TType.STRING, 'UTF8', TType.BOOL, None, False), None, ), # 28 + (29, TType.STRING, 'UDFModulePath', 'UTF8', None, ), # 29 ) all_structs.append(UpdateUserReq) UpdateUserReq.thrift_spec = ( @@ -8779,6 +9357,21 @@ def __ne__(self, other): (3, TType.I64, 'recordsNum', None, None, ), # 3 (4, TType.STRING, 'parseErrorMsg', 'UTF8', None, ), # 4 ) +all_structs.append(LoadUDFReq) +LoadUDFReq.thrift_spec = ( + None, # 0 + (1, TType.I64, 'sessionId', None, None, ), # 1 + (2, TType.STRING, 'statement', 'UTF8', None, ), # 2 + (3, TType.STRING, 'udfFile', 'BINARY', None, ), # 3 + (4, TType.BOOL, 'isRemote', None, None, ), # 4 +) +all_structs.append(LoadUDFResp) +LoadUDFResp.thrift_spec = ( + None, # 0 + (1, TType.STRUCT, 'status', [Status, None], None, ), # 1 + (2, TType.STRING, 'parseErrorMsg', 'UTF8', None, ), # 2 + (3, TType.STRING, 'UDFModulePath', 'UTF8', None, ), # 3 +) all_structs.append(TaskInfo) TaskInfo.thrift_spec = ( None, # 0 @@ -8836,10 +9429,11 @@ def __ne__(self, other): RegisterTaskReq.thrift_spec = ( None, # 0 (1, TType.I64, 'sessionId', None, None, ), # 1 - (2, TType.STRING, 'name', 'UTF8', None, ), # 2 - (3, TType.STRING, 'filePath', 'UTF8', None, ), # 3 - (4, TType.STRING, 'className', 'UTF8', None, ), # 4 - (5, TType.I32, 'type', None, None, ), # 5 + (2, TType.STRING, 'filePath', 'UTF8', None, ), # 2 + (3, TType.LIST, 'UDFClassPairs', (TType.STRUCT, [UDFClassPair, None], False), None, ), # 3 + (4, TType.LIST, 'types', (TType.I32, None, False), None, ), # 4 + (5, TType.STRING, 'moduleFile', 'BINARY', None, ), # 5 + (6, TType.BOOL, 'isRemote', None, None, ), # 6 ) all_structs.append(DropTaskReq) DropTaskReq.thrift_spec = ( @@ -8957,5 +9551,22 @@ def __ne__(self, other): (1, TType.STRUCT, 'status', [Status, None], None, ), # 1 (2, TType.LIST, 'sessionIDList', (TType.I64, None, False), None, ), # 2 ) +all_structs.append(ShowRulesReq) +ShowRulesReq.thrift_spec = ( + None, # 0 + (1, TType.I64, 'sessionId', None, None, ), # 1 +) +all_structs.append(ShowRulesResp) +ShowRulesResp.thrift_spec = ( + None, # 0 + (1, TType.STRUCT, 'status', [Status, None], None, ), # 1 + (2, TType.MAP, 'rules', (TType.STRING, 'UTF8', TType.BOOL, None, False), None, ), # 2 +) +all_structs.append(SetRulesReq) +SetRulesReq.thrift_spec = ( + None, # 0 + (1, TType.I64, 'sessionId', None, None, ), # 1 + (2, TType.MAP, 'rulesChange', (TType.STRING, 'UTF8', TType.BOOL, None, False), None, ), # 2 +) fix_spec(all_structs) del all_structs diff --git a/session_py/iginx/iginx_pyclient/time_series.py b/session_py/iginx/iginx_pyclient/time_series.py index 1cf4cf9d64..c4d0da32e0 100644 --- a/session_py/iginx/iginx_pyclient/time_series.py +++ b/session_py/iginx/iginx_pyclient/time_series.py @@ -1,20 +1,21 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. # +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + from .thrift.rpc.ttypes import DataType class TimeSeries(object): diff --git a/session_py/iginx/iginx_pyclient/utils/__init__.py b/session_py/iginx/iginx_pyclient/utils/__init__.py index 4b8ee97fad..5740237d34 100644 --- a/session_py/iginx/iginx_pyclient/utils/__init__.py +++ b/session_py/iginx/iginx_pyclient/utils/__init__.py @@ -1,17 +1,19 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + # \ No newline at end of file diff --git a/session_py/iginx/iginx_pyclient/utils/bitmap.py b/session_py/iginx/iginx_pyclient/utils/bitmap.py index e27f4125df..c8d93106f2 100644 --- a/session_py/iginx/iginx_pyclient/utils/bitmap.py +++ b/session_py/iginx/iginx_pyclient/utils/bitmap.py @@ -1,19 +1,19 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . # class Bitmap(object): diff --git a/session_py/iginx/iginx_pyclient/utils/byte_utils.py b/session_py/iginx/iginx_pyclient/utils/byte_utils.py index 9010056693..9f1d79e55d 100644 --- a/session_py/iginx/iginx_pyclient/utils/byte_utils.py +++ b/session_py/iginx/iginx_pyclient/utils/byte_utils.py @@ -1,19 +1,19 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . # import struct @@ -190,7 +190,7 @@ def next(self, type): else: raise RuntimeError("unknown data type " + type) - def get_bytes_from_types(self, types, bitmap : Bitmap): + def get_bytes_from_types(self, types, bitmap: Bitmap): bytes_value = [] i = -1 for type in types: diff --git a/session_py/iginx/setup.py b/session_py/iginx/setup.py index 6aabea4ba8..4bf54b9a33 100644 --- a/session_py/iginx/setup.py +++ b/session_py/iginx/setup.py @@ -1,7 +1,26 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + import io import os import sys import warnings +import xml.etree.ElementTree as ET from setuptools import setup, find_packages @@ -12,6 +31,12 @@ RuntimeWarning) del fmt +tree = ET.parse('../../pom.xml') +root = tree.getroot() +namespaces = {'mvn': 'http://maven.apache.org/POM/4.0.0'} +properties = root.find('mvn:properties', namespaces) +revision = properties.find('mvn:revision', namespaces).text + with open('requirements.txt') as f: required = f.read().splitlines() @@ -21,7 +46,7 @@ setup( name='iginx-pyclient', - version='0.6.1', + version=revision, include_package_data=True, packages=find_packages(), author='THU IGinX', diff --git a/shared/pom.xml b/shared/pom.xml index ed94dcbeec..d8c589b5b3 100644 --- a/shared/pom.xml +++ b/shared/pom.xml @@ -1,4 +1,23 @@ + 4.0.0 @@ -11,11 +30,6 @@ iginx-shared IGinX Shared - - 8 - 8 - - org.apache.commons diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/constant/GlobalConstant.java b/shared/src/main/java/cn/edu/tsinghua/iginx/constant/GlobalConstant.java index c3e3e0a5f0..41b9327b6f 100644 --- a/shared/src/main/java/cn/edu/tsinghua/iginx/constant/GlobalConstant.java +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/constant/GlobalConstant.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.constant; public class GlobalConstant { @@ -8,4 +26,8 @@ public class GlobalConstant { public static final String CLEAR_DUMMY_DATA_CAUTION = "Unable to delete data from read-only nodes. The data of the writable nodes has been cleared."; + + public static final Long KEY_MIN_VAL = Long.MIN_VALUE + 1; + + public static final Long KEY_MAX_VAL = Long.MAX_VALUE; } diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/exception/IginxRuntimeException.java b/shared/src/main/java/cn/edu/tsinghua/iginx/exception/IginxRuntimeException.java index 8db3c68488..10de2baca9 100644 --- a/shared/src/main/java/cn/edu/tsinghua/iginx/exception/IginxRuntimeException.java +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/exception/IginxRuntimeException.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + /* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/exception/StatusCode.java b/shared/src/main/java/cn/edu/tsinghua/iginx/exception/StatusCode.java index 2dcdf4836d..e5aa81971f 100644 --- a/shared/src/main/java/cn/edu/tsinghua/iginx/exception/StatusCode.java +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/exception/StatusCode.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.exception; diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/exception/UnsupportedDataTypeException.java b/shared/src/main/java/cn/edu/tsinghua/iginx/exception/UnsupportedDataTypeException.java index 9665d97ce9..b850c93e93 100644 --- a/shared/src/main/java/cn/edu/tsinghua/iginx/exception/UnsupportedDataTypeException.java +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/exception/UnsupportedDataTypeException.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.exception; diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/exception/package-info.java b/shared/src/main/java/cn/edu/tsinghua/iginx/exception/package-info.java index 8e523880b3..4108ad30eb 100644 --- a/shared/src/main/java/cn/edu/tsinghua/iginx/exception/package-info.java +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/exception/package-info.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + /** * IGinX异常处理规范。 * diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/Bitmap.java b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/Bitmap.java index b02ae40ded..8638cba727 100644 --- a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/Bitmap.java +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/Bitmap.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.utils; diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/ByteUtils.java b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/ByteUtils.java index 626229346b..b966d9c35b 100644 --- a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/ByteUtils.java +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/ByteUtils.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.utils; diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/CSVUtils.java b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/CSVUtils.java index 1978c94e89..25636093b3 100644 --- a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/CSVUtils.java +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/CSVUtils.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.utils; import cn.edu.tsinghua.iginx.thrift.ExportCSV; diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/CheckedFunction.java b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/CheckedFunction.java index 39842e55e6..7c59d906cf 100644 --- a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/CheckedFunction.java +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/CheckedFunction.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.utils; diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/CompressionUtils.java b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/CompressionUtils.java index da7fd16174..2edf585817 100644 --- a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/CompressionUtils.java +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/CompressionUtils.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.utils; import java.io.*; diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/CurveMatchUtils.java b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/CurveMatchUtils.java index ca1c38d158..3ca544db76 100644 --- a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/CurveMatchUtils.java +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/CurveMatchUtils.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.utils; diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/DataTypeInferenceUtils.java b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/DataTypeInferenceUtils.java index 8bc61c2ef1..07c6662f5a 100644 --- a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/DataTypeInferenceUtils.java +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/DataTypeInferenceUtils.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.utils; diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/DataTypeUtils.java b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/DataTypeUtils.java index b144e217d5..ea0efd44b8 100644 --- a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/DataTypeUtils.java +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/DataTypeUtils.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.utils; diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/EnvUtils.java b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/EnvUtils.java index a94d924851..77214db983 100644 --- a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/EnvUtils.java +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/EnvUtils.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.utils; diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/Escaper.java b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/Escaper.java index 3dcd250d39..6f9c7011a6 100644 --- a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/Escaper.java +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/Escaper.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.utils; import java.text.ParseException; diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/FileCompareUtils.java b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/FileCompareUtils.java index a86a777c25..31a0492d30 100644 --- a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/FileCompareUtils.java +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/FileCompareUtils.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.utils; import java.io.BufferedReader; diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/FileReader.java b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/FileReader.java index 9d680823de..4dc17b00bb 100644 --- a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/FileReader.java +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/FileReader.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.utils; import java.io.*; diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/FileUtils.java b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/FileUtils.java index 03bc5a7446..4e7496538c 100644 --- a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/FileUtils.java +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/FileUtils.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.utils; import java.io.File; diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/FormatUtils.java b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/FormatUtils.java index 28a4ecd991..4bfef95735 100644 --- a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/FormatUtils.java +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/FormatUtils.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.utils; import java.text.SimpleDateFormat; diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/HostUtils.java b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/HostUtils.java index e714b8186d..31adc88681 100644 --- a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/HostUtils.java +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/HostUtils.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.utils; import java.net.Inet4Address; diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/JobFromYAML.java b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/JobFromYAML.java index bc758c2a0d..a2eda0e13b 100644 --- a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/JobFromYAML.java +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/JobFromYAML.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.utils; import java.util.List; @@ -7,6 +25,7 @@ public class JobFromYAML { private List taskList; private String exportFile; private String exportType; + private String schedule; public JobFromYAML() {} @@ -33,4 +52,12 @@ public String getExportType() { public void setExportType(String exportType) { this.exportType = exportType; } + + public String getSchedule() { + return schedule; + } + + public void setSchedule(String schedule) { + this.schedule = schedule; + } } diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/JsonUtils.java b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/JsonUtils.java index edbb6ff952..5669ea211b 100644 --- a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/JsonUtils.java +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/JsonUtils.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.utils; import com.alibaba.fastjson2.JSON; diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/Pair.java b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/Pair.java index d15d0dd292..05158f9982 100644 --- a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/Pair.java +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/Pair.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.utils; diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/RpcUtils.java b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/RpcUtils.java index 69f524bbc1..078fa94303 100644 --- a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/RpcUtils.java +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/RpcUtils.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.utils; diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/SerializeUtils.java b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/SerializeUtils.java index 9d75ffb1fb..733cee3ae5 100644 --- a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/SerializeUtils.java +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/SerializeUtils.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.utils; diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/ShellRunner.java b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/ShellRunner.java index 4be241aaa4..efd9f5a21c 100644 --- a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/ShellRunner.java +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/ShellRunner.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.utils; import java.io.BufferedReader; diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/SnowFlakeUtils.java b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/SnowFlakeUtils.java index 01ffbdd71d..5f305f2cd9 100644 --- a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/SnowFlakeUtils.java +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/SnowFlakeUtils.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.utils; diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/SortUtils.java b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/SortUtils.java index 10c05386f9..d59ecb2fae 100644 --- a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/SortUtils.java +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/SortUtils.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.utils; diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/StringUtils.java b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/StringUtils.java index 0453342c2a..4321a8ea94 100644 --- a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/StringUtils.java +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/StringUtils.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.utils; @@ -23,8 +22,11 @@ import java.time.DateTimeException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Objects; +import java.util.Set; import java.util.function.Predicate; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -188,4 +190,180 @@ public static Long tryParse2Key(String value) { } return res; } + + public static Set cutSchemaPrefix(String schemaPrefix, Set patterns) { + // show columns的patterns为空,查询所有列 + if (patterns.isEmpty()) { + return Collections.singleton("*"); + } + // 该数据源没有schema prefix,直接匹配patterns + if (schemaPrefix == null || schemaPrefix.isEmpty()) { + return patterns; + } + + Set patternsCutSchemaPrefix = new HashSet<>(); + for (String pattern : patterns) { + Set tmp = cutSchemaPrefix(schemaPrefix, pattern); + if (tmp.contains("*")) { + return Collections.singleton("*"); + } + patternsCutSchemaPrefix.addAll(tmp); + } + return patternsCutSchemaPrefix; + } + + private static Set cutSchemaPrefix(String schemaPrefix, String pattern) { + String[] prefixSplit = schemaPrefix.split("\\."); + String[] patternSplit = pattern.split("\\."); + int minLen = Math.min(prefixSplit.length, patternSplit.length); + int index = 0; + // 逐级匹配pattern和schemaPrefix + while (index < minLen && prefixSplit[index].equals(patternSplit[index])) { + index++; + } + + // pattern匹配结束,schemaPrefix还有剩余,则该storageEngine下没有该pattern + if (index == patternSplit.length) { + return Collections.emptySet(); + } + + // schemaPrefix匹配结束,pattern还有剩余,则把该pattern减去前缀schemaPrefix + if (index == prefixSplit.length) { + return Collections.singleton(joinWithDot(patternSplit, index)); + } + + // pattern和schemaPrefix不匹配 + if (!patternSplit[index].equals("*")) { + return Collections.emptySet(); + } + + Set target = new HashSet<>(); + // 将pattern的'*'视为部分匹配该前缀,即把'*'下推到数据源 + target.add(joinWithDot(patternSplit, index)); + if (index + 1 < patternSplit.length) { + // 将pattern的'*'视为完全匹配该前缀,即不把'*'下推到数据源 + String patternRemain = joinWithDot(patternSplit, index + 1); + target.add(patternRemain); + + // 将schemaPrefix的每一级分别匹配'*' + for (int i = index + 1; i < prefixSplit.length; i++) { + String prefixRemain = joinWithDot(prefixSplit, i); + target.addAll(cutSchemaPrefix(prefixRemain, patternRemain)); + } + } + + return target; + } + + private static String joinWithDot(String[] strings, int begin) { + if (begin >= strings.length) { + return ""; + } + StringBuilder sb = new StringBuilder(); + for (int i = begin; i < strings.length; i++) { + sb.append(strings[i]).append("."); + } + sb.setLength(sb.length() - 1); + return sb.toString(); + } + + public static Set intersectDataPrefix(String dataPrefix, Set patterns) { + if (dataPrefix == null || dataPrefix.isEmpty() || patterns.isEmpty()) { + return patterns; + } + String dataPrefixRegex = dataPrefix + ".*"; + if (patterns.contains("*")) { + return Collections.singleton(dataPrefixRegex); + } + + Set target = new HashSet<>(); + for (String pattern : patterns) { + Set tmp = intersectDataPrefix(dataPrefix, pattern); + if (tmp == null) { + continue; + } + if (tmp.contains(dataPrefixRegex)) { + return Collections.singleton(dataPrefixRegex); + } + target.addAll(tmp); + } + + // 移除不必要的pattern + return mergePatterns(target); + } + + private static Set intersectDataPrefix(String dataPrefix, String pattern) { + String[] prefixSplit = dataPrefix.split("\\."); + String[] patternSplit = pattern.split("\\."); + StringBuilder commonPrefix = new StringBuilder(); + int minLen = Math.min(prefixSplit.length, patternSplit.length); + int index = 0; + // 逐级匹配pattern和dataPrefix + while (index < minLen && prefixSplit[index].equals(patternSplit[index])) { + commonPrefix.append(prefixSplit[index]); + index++; + } + + // pattern匹配结束,dataPrefix还有剩余,则交集取dataPrefix + if (index == patternSplit.length) { + return Collections.singleton(dataPrefix + ".*"); + } + + // dataPrefix匹配结束,pattern还有剩余,则交集取pattern + if (index == prefixSplit.length) { + return Collections.singleton(pattern); + } + + // pattern和dataPrefix不匹配 + if (!patternSplit[index].equals("*")) { + return Collections.emptySet(); + } + + Set target = new HashSet<>(); + // 将pattern的'*'视为部分匹配该前缀,即把'*'下推到数据源 + target.add(dataPrefix + "." + joinWithDot(patternSplit, index)); + if (index + 1 < patternSplit.length) { + // 将pattern的'*'视为完全匹配该前缀,即不把'*'下推到数据源 + String patternRemain = joinWithDot(patternSplit, index + 1); + target.add(dataPrefix + "." + patternRemain); + + if (commonPrefix.length() > 0) { + commonPrefix.append("."); + } + // 将dataPrefix的每一级分别匹配'*' + for (int i = index + 1; i < prefixSplit.length; i++) { + commonPrefix.append(prefixSplit[i - 1]).append("."); + String prefixRemain = joinWithDot(prefixSplit, i); + Set ret = intersectDataPrefix(prefixRemain, patternRemain); + ret.forEach(str -> target.add(commonPrefix + str)); + } + } + + return target; + } + + private static Set mergePatterns(Set patterns) { + if (patterns.size() <= 1) { + return patterns; + } + Set target = new HashSet<>(); + List list = new ArrayList<>(patterns); + int size = list.size(); + List toBeRemoved = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + boolean removed = false; + for (int j = 0; j != i && !toBeRemoved.contains(j) && j < size; j++) { + // 第j个pattern包含第i个pattern + if (match(list.get(i), list.get(j))) { + toBeRemoved.add(i); + removed = true; + break; + } + } + if (!removed) { + target.add(list.get(i)); + } + } + return target; + } } diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/TagKVUtils.java b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/TagKVUtils.java index fc8ca48ed8..317c462805 100644 --- a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/TagKVUtils.java +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/TagKVUtils.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.utils; diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/TaskFromYAML.java b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/TaskFromYAML.java index 9a52bf75bb..5d3bfdf87d 100644 --- a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/TaskFromYAML.java +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/TaskFromYAML.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.utils; import java.util.List; diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/ThriftConnPool.java b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/ThriftConnPool.java index da9695d2a2..4997e7c72b 100644 --- a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/ThriftConnPool.java +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/ThriftConnPool.java @@ -1,9 +1,28 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.utils; import java.time.Duration; import java.util.Map; import org.apache.commons.pool2.PooledObject; import org.apache.commons.pool2.PooledObjectFactory; +import org.apache.commons.pool2.impl.BaseObjectPoolConfig; import org.apache.commons.pool2.impl.DefaultPooledObject; import org.apache.commons.pool2.impl.GenericObjectPool; import org.apache.commons.pool2.impl.GenericObjectPoolConfig; @@ -15,32 +34,35 @@ public class ThriftConnPool { private static final Logger LOGGER = LoggerFactory.getLogger(ThriftConnPool.class); - private static final int DEFAULT_MAX_SIZE = 100; + private static final int DEFAULT_MAX_SIZE = GenericObjectPoolConfig.DEFAULT_MAX_TOTAL; - private static final int MAX_WAIT_TIME = 30000; + private static final int MAX_WAIT_TIME = 0; // infinite, same as Socket default - private static final long IDLE_TIMEOUT = 10L * 60L * 1000L; + private static final long IDLE_TIMEOUT = + BaseObjectPoolConfig.DEFAULT_MIN_EVICTABLE_IDLE_DURATION.toMillis(); private final GenericObjectPool pool; public ThriftConnPool(String ip, int port) { - this(ip, port, DEFAULT_MAX_SIZE, MAX_WAIT_TIME); + this(ip, port, DEFAULT_MAX_SIZE, MAX_WAIT_TIME, IDLE_TIMEOUT); } public ThriftConnPool(String ip, int port, Map extraParams) { this( ip, port, - DEFAULT_MAX_SIZE, Integer.parseInt( - extraParams.getOrDefault("thrift_timeout", String.valueOf(MAX_WAIT_TIME)))); + extraParams.getOrDefault("thrift_pool_max_size", String.valueOf(DEFAULT_MAX_SIZE))), + Integer.parseInt(extraParams.getOrDefault("thrift_timeout", String.valueOf(MAX_WAIT_TIME))), + Long.parseLong( + extraParams.getOrDefault( + "thrift_pool_min_evictable_idle_time_millis", String.valueOf(IDLE_TIMEOUT)))); } - public ThriftConnPool(String ip, int port, int maxSize, int maxWaitTime) { - + public ThriftConnPool(String ip, int port, int maxSize, int maxWaitTime, long idleTimeout) { GenericObjectPoolConfig poolConfig = new GenericObjectPoolConfig<>(); poolConfig.setMaxTotal(maxSize); - poolConfig.setMinEvictableIdleDuration(Duration.ofMillis(IDLE_TIMEOUT)); // 设置空闲连接的超时时间 + poolConfig.setMinEvictableIdleDuration(Duration.ofMillis(idleTimeout)); // 设置空闲连接的超时时间 TSocketFactory socketFactory = new TSocketFactory(ip, port, maxWaitTime); pool = new GenericObjectPool<>(socketFactory, poolConfig); @@ -76,7 +98,7 @@ public static class TSocketFactory implements PooledObjectFactory { private final String ip; private final int port; - private final int maxWaitTime; // 连接超时时间(毫秒) + private final int maxWaitTime; // 连接超时时间(毫秒) 以及 socket 超时时间 public TSocketFactory(String ip, int port, int maxWaitTime) { this.ip = ip; diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/TimeUtils.java b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/TimeUtils.java index 9c1efcabee..c15497430b 100644 --- a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/TimeUtils.java +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/TimeUtils.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.utils; diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/ValueUtils.java b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/ValueUtils.java index 2346fad6a9..d57568d19b 100644 --- a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/ValueUtils.java +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/ValueUtils.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.utils; public class ValueUtils { diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/YAMLReader.java b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/YAMLReader.java index 46bd13abe6..bb3cb761a9 100644 --- a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/YAMLReader.java +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/YAMLReader.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.utils; import java.io.*; @@ -41,10 +59,12 @@ public String normalize(String conf) { String exportType = "(?i)exportType"; String exportFile = "(?i)exportFile"; String exportNameList = "(?i)exportNameList"; + String schedule = "(?i)schedule"; conf = conf.replaceAll(exportType, "exportType"); conf = conf.replaceAll(exportFile, "exportFile"); conf = conf.replaceAll(exportNameList, "exportNameList"); + conf = conf.replaceAll(schedule, "schedule"); return conf; } diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/YAMLWriter.java b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/YAMLWriter.java index cf8a9c5283..35f774c945 100644 --- a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/YAMLWriter.java +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/YAMLWriter.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.utils; import java.io.BufferedWriter; diff --git a/shared/src/test/java/cn/edu/tsinghua/iginx/utils/CompressionUtilsTest.java b/shared/src/test/java/cn/edu/tsinghua/iginx/utils/CompressionUtilsTest.java index cdf145a76e..dcc269ccbc 100644 --- a/shared/src/test/java/cn/edu/tsinghua/iginx/utils/CompressionUtilsTest.java +++ b/shared/src/test/java/cn/edu/tsinghua/iginx/utils/CompressionUtilsTest.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.utils; import static org.junit.Assert.fail; diff --git a/shared/src/test/java/cn/edu/tsinghua/iginx/utils/EscaperTest.java b/shared/src/test/java/cn/edu/tsinghua/iginx/utils/EscaperTest.java index 87bf1d4265..78926dd14c 100644 --- a/shared/src/test/java/cn/edu/tsinghua/iginx/utils/EscaperTest.java +++ b/shared/src/test/java/cn/edu/tsinghua/iginx/utils/EscaperTest.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.utils; import static org.junit.Assert.*; diff --git a/shared/src/test/java/cn/edu/tsinghua/iginx/utils/StringUtilsTest.java b/shared/src/test/java/cn/edu/tsinghua/iginx/utils/StringUtilsTest.java index 90cad48265..1ac452ae30 100644 --- a/shared/src/test/java/cn/edu/tsinghua/iginx/utils/StringUtilsTest.java +++ b/shared/src/test/java/cn/edu/tsinghua/iginx/utils/StringUtilsTest.java @@ -1,7 +1,29 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.utils; import static org.junit.Assert.assertEquals; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; import org.junit.Before; import org.junit.Test; @@ -37,4 +59,53 @@ public void toRegexExpr() { assertEquals(".*\\Q.\\E.*\\Q.c\\E", StringUtils.toRegexExpr("**.**.c")); assertEquals(".*\\Q.\\E.*\\Q.\\E.*", StringUtils.toRegexExpr("**.**.**")); } + + @Test + public void cutSchemaPrefix() { + assertEquals(toSet("*"), StringUtils.cutSchemaPrefix(null, Collections.emptySet())); + assertEquals(toSet("*"), StringUtils.cutSchemaPrefix("a.b", Collections.emptySet())); + assertEquals(toSet("a.*", "b.*"), StringUtils.cutSchemaPrefix(null, toSet("a.*", "b.*"))); + assertEquals(toSet("*"), StringUtils.cutSchemaPrefix("a.b.c", toSet("a.*"))); + assertEquals(toSet("*"), StringUtils.cutSchemaPrefix("a.b.c", toSet("a.b.c.*"))); + assertEquals(Collections.emptySet(), StringUtils.cutSchemaPrefix("a.b.c", toSet("a.b.c"))); + assertEquals(Collections.emptySet(), StringUtils.cutSchemaPrefix("a.b.c", toSet("d.*"))); + assertEquals(Collections.emptySet(), StringUtils.cutSchemaPrefix("a.bb.c", toSet("a.b.*"))); + assertEquals(Collections.emptySet(), StringUtils.cutSchemaPrefix("a.b.c", toSet("a.bb.*"))); + assertEquals(toSet("*"), StringUtils.cutSchemaPrefix("a.b.c", toSet("*.b.*"))); + assertEquals(toSet("b.c", "*.b.c"), StringUtils.cutSchemaPrefix("a.b.c", toSet("*.b.c"))); + } + + @Test + public void intersectDataPrefix() { + assertEquals( + toSet("a.b.c.*"), StringUtils.intersectDataPrefix(null, Collections.singleton("a.b.c.*"))); + assertEquals( + toSet("a.b.*"), StringUtils.intersectDataPrefix("a.b", Collections.singleton("*"))); + assertEquals( + toSet("a.b.c.*"), StringUtils.intersectDataPrefix("a.b", Collections.singleton("a.b.c.*"))); + assertEquals( + toSet("a.b.c"), StringUtils.intersectDataPrefix("a.b", Collections.singleton("a.b.c"))); + assertEquals( + toSet("a.b.*"), StringUtils.intersectDataPrefix("a.b", Collections.singleton("a.*"))); + assertEquals( + toSet("a.b.c.*", "a.b.*.c.*"), + StringUtils.intersectDataPrefix("a.b", Collections.singleton("*.c.*"))); + assertEquals( + toSet("a.b.*"), StringUtils.intersectDataPrefix("a.b", Collections.singleton("*.b.*"))); + assertEquals( + toSet("a.b.c", "a.b.b.c", "a.b.*.b.c"), + StringUtils.intersectDataPrefix("a.b", Collections.singleton("*.b.c"))); + assertEquals( + toSet("a.b.*.c"), StringUtils.intersectDataPrefix("a.b", Collections.singleton("*.b.*.c"))); + assertEquals( + Collections.emptySet(), + StringUtils.intersectDataPrefix("a.b", Collections.singleton("b.*.b.*.c"))); + assertEquals( + Collections.emptySet(), + StringUtils.intersectDataPrefix("a.b", Collections.singleton("a.c.b.*"))); + } + + private Set toSet(String... items) { + return new HashSet<>(Arrays.asList(items)); + } } diff --git a/test/pom.xml b/test/pom.xml index d2d0c8dc6c..598fdb8aff 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -1,4 +1,23 @@ + 4.0.0 @@ -7,15 +26,12 @@ cn.edu.tsinghua iginx ${revision} - ../pom.xml test IGinX Test - 8 - 8 true diff --git a/test/src/main/java/cn/edu/tsinghua/iginx/shared/MockClassGenerator.java b/test/src/main/java/cn/edu/tsinghua/iginx/shared/MockClassGenerator.java index 48a62d6b56..b7c933fabd 100644 --- a/test/src/main/java/cn/edu/tsinghua/iginx/shared/MockClassGenerator.java +++ b/test/src/main/java/cn/edu/tsinghua/iginx/shared/MockClassGenerator.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.shared; import cn.edu.tsinghua.iginx.conf.Constants; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/client/ExportFileIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/client/ExportFileIT.java index 0497d370da..95a93bfad1 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/client/ExportFileIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/client/ExportFileIT.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.client; import static org.junit.Assert.assertEquals; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/client/ImportFileIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/client/ImportFileIT.java index d040dd4cdb..2af25690d4 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/client/ImportFileIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/client/ImportFileIT.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.client; import cn.edu.tsinghua.iginx.exception.SessionException; @@ -16,7 +34,6 @@ public class ImportFileIT { @BeforeClass public static void setUp() throws SessionException { MultiConnection session = new MultiConnection(new Session("127.0.0.1", 6888, "root", "root")); - ; executor = new SQLExecutor(session); executor.open(); } diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/compaction/CompactionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/compaction/CompactionIT.java index 636ad9b3af..087faf9ffa 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/compaction/CompactionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/compaction/CompactionIT.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.compaction; import static org.junit.Assert.assertEquals; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/controller/Controller.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/controller/Controller.java index b5231a1d69..d62a87cda1 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/controller/Controller.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/controller/Controller.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.controller; import static cn.edu.tsinghua.iginx.constant.GlobalConstant.CLEAR_DUMMY_DATA_CAUTION; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/controller/TestEnvironmentController.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/controller/TestEnvironmentController.java index 41aff1d677..b1dda6b233 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/controller/TestEnvironmentController.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/controller/TestEnvironmentController.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.controller; import cn.edu.tsinghua.iginx.metadata.entity.StorageEngineMeta; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/BaseCapacityExpansionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/BaseCapacityExpansionIT.java index 1577c7587b..8a5f6179dc 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/BaseCapacityExpansionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/BaseCapacityExpansionIT.java @@ -1,10 +1,27 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.expansion; import static cn.edu.tsinghua.iginx.integration.controller.Controller.SUPPORT_KEY; import static cn.edu.tsinghua.iginx.integration.expansion.constant.Constant.*; import static cn.edu.tsinghua.iginx.integration.expansion.utils.SQLTestTools.executeShellScript; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; +import static org.junit.Assert.*; import cn.edu.tsinghua.iginx.exception.SessionException; import cn.edu.tsinghua.iginx.integration.controller.Controller; @@ -17,11 +34,10 @@ import cn.edu.tsinghua.iginx.session.QueryDataSet; import cn.edu.tsinghua.iginx.session.Session; import cn.edu.tsinghua.iginx.thrift.RemovedStorageEngineInfo; +import cn.edu.tsinghua.iginx.thrift.StorageEngineInfo; import cn.edu.tsinghua.iginx.thrift.StorageEngineType; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; +import java.util.*; +import java.util.stream.Collectors; import org.junit.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,6 +49,8 @@ public abstract class BaseCapacityExpansionIT { protected static Session session; + protected static final String ALTER_ENGINE_STRING = "alter storageengine %d with params \"%s\";"; + private static final ConfLoader testConf = new ConfLoader(Controller.CONFIG_FILE); protected StorageEngineType type; @@ -41,6 +59,8 @@ public abstract class BaseCapacityExpansionIT { protected List wrongExtraParams = new ArrayList<>(); + protected Map updatedParams = new HashMap<>(); + private final boolean IS_PARQUET_OR_FILE_SYSTEM = this instanceof FileSystemCapacityExpansionIT || this instanceof ParquetCapacityExpansionIT; @@ -50,6 +70,8 @@ public abstract class BaseCapacityExpansionIT { public static final String DBCE_PARQUET_FS_TEST_DIR = "test"; + protected static final String updateParamsScriptDir = ".github/scripts/dataSources/update/"; + protected static BaseHistoryDataGenerator generator; public BaseCapacityExpansionIT( @@ -85,9 +107,10 @@ protected String addStorageEngine( if (IS_PARQUET_OR_FILE_SYSTEM) { statement.append(String.format(", dummy_dir:%s/", DBCE_PARQUET_FS_TEST_DIR)); statement.append(PORT_TO_ROOT.get(port)); - statement.append(String.format(", dir:%s/iginx_", DBCE_PARQUET_FS_TEST_DIR)); + statement.append( + String.format(", dir:%s/" + IGINX_DATA_PATH_PREFIX_NAME, DBCE_PARQUET_FS_TEST_DIR)); statement.append(PORT_TO_ROOT.get(port)); - statement.append(", iginx_port:" + oriPortIginx); + statement.append(", iginx_port:").append(oriPortIginx); } if (extraParams != null) { statement.append(", "); @@ -146,8 +169,7 @@ public void clearData() { } private void addStorageEngineInProgress( - int port, boolean hasData, boolean isReadOnly, String dataPrefix, String schemaPrefix) - throws InterruptedException { + int port, boolean hasData, boolean isReadOnly, String dataPrefix, String schemaPrefix) { if (IS_PARQUET_OR_FILE_SYSTEM) { startStorageEngineWithIginx(port, hasData, isReadOnly); } else { @@ -157,7 +179,7 @@ private void addStorageEngineInProgress( } @Test - public void oriHasDataExpHasData() throws InterruptedException, SessionException { + public void oriHasDataExpHasData() { // 查询原始节点的历史数据,结果不为空 testQueryHistoryDataOriHasData(); // 写入并查询新数据 @@ -176,7 +198,7 @@ public void oriHasDataExpHasData() throws InterruptedException, SessionException } @Test - public void oriHasDataExpNoData() throws InterruptedException { + public void oriHasDataExpNoData() { // 查询原始节点的历史数据,结果不为空 testQueryHistoryDataOriHasData(); // 写入并查询新数据 @@ -192,7 +214,7 @@ public void oriHasDataExpNoData() throws InterruptedException { } @Test - public void oriNoDataExpHasData() throws InterruptedException { + public void oriNoDataExpHasData() { // 查询原始节点的历史数据,结果为空 testQueryHistoryDataOriNoData(); // 写入并查询新数据 @@ -210,7 +232,7 @@ public void oriNoDataExpHasData() throws InterruptedException { } @Test - public void oriNoDataExpNoData() throws InterruptedException { + public void oriNoDataExpNoData() { // 查询原始节点的历史数据,结果为空 testQueryHistoryDataOriNoData(); // 写入并查询新数据 @@ -226,11 +248,13 @@ public void oriNoDataExpNoData() throws InterruptedException { } @Test - public void testReadOnly() throws InterruptedException { + public void testReadOnly() throws SessionException { // 查询原始只读节点的历史数据,结果不为空 testQueryHistoryDataOriHasData(); + // 测试只读节点的参数修改 + testUpdateEngineParams(); // 测试参数错误的只读节点扩容 - testInvalidDummyParams(readOnlyPort, true, false, null, EXP_SCHEMA_PREFIX); + testInvalidDummyParams(readOnlyPort, true, true, null, READ_ONLY_SCHEMA_PREFIX); // 扩容只读节点 addStorageEngineInProgress(readOnlyPort, true, true, null, READ_ONLY_SCHEMA_PREFIX); // 查询扩容只读节点的历史数据,结果不为空 @@ -302,6 +326,69 @@ protected void testInvalidDummyParams( } } + /** 测试引擎修改参数(目前仅支持dummy & read-only) */ + protected void testUpdateEngineParams() throws SessionException { + // 修改前后通过相同schema_prefix查询判断引擎成功更新 + LOGGER.info("Testing updating engine params..."); + if (updatedParams.isEmpty()) { + LOGGER.info("Engine {} skipped this test.", type); + return; + } + + String prefix = "prefix"; + // 添加只读节点 + addStorageEngine(readOnlyPort, true, true, null, prefix, extraParams); + // 查询 + String statement = "select wt01.status, wt01.temperature from " + prefix + ".tm.wf05;"; + List pathList = + READ_ONLY_PATH_LIST.stream().map(s -> prefix + "." + s).collect(Collectors.toList()); + List> valuesList = READ_ONLY_VALUES_LIST; + SQLTestTools.executeAndCompare(session, statement, pathList, valuesList); + + // 修改数据库参数 + updateParams(readOnlyPort); + + // 修改 + List engineInfoList = session.getClusterInfo().getStorageEngineInfos(); + long id = -1; + for (StorageEngineInfo info : engineInfoList) { + if (info.getIp().equals("127.0.0.1") + && info.getPort() == readOnlyPort + && info.getDataPrefix().equals("null") + && info.getSchemaPrefix().equals(prefix) + && info.getType().equals(type)) { + id = info.getId(); + } + } + assertTrue(id != -1); + + String newParams = + updatedParams.entrySet().stream() + .map(entry -> entry.getKey() + ":" + entry.getValue()) + .collect(Collectors.joining(", ")); + session.executeSql(String.format(ALTER_ENGINE_STRING, id, newParams)); + + // 重新查询 + statement = "select wt01.status, wt01.temperature from " + prefix + ".tm.wf05;"; + pathList = READ_ONLY_PATH_LIST.stream().map(s -> prefix + "." + s).collect(Collectors.toList()); + valuesList = READ_ONLY_VALUES_LIST; + SQLTestTools.executeAndCompare(session, statement, pathList, valuesList); + + // 删除,不影响后续测试 + session.removeHistoryDataSource( + Collections.singletonList( + new RemovedStorageEngineInfo("127.0.0.1", readOnlyPort, prefix, ""))); + + // 改回数据库参数 + restoreParams(readOnlyPort); + } + + /** 这个方法需要实现:通过脚本修改port对应数据源的可变参数,如密码等 */ + protected abstract void updateParams(int port); + + /** 这个方法需要实现:通过脚本恢复updateParams中修改的可变参数 */ + protected abstract void restoreParams(int port); + protected void queryExtendedKeyDummy() { // ori // extended key queryable @@ -466,7 +553,9 @@ private void testAddAndRemoveStorageEngineWithPrefix() { List> valuesList = EXP_VALUES_LIST1; // 添加不同 schemaPrefix,相同 dataPrefix + testShowColumnsInExpansion(true); addStorageEngine(expPort, true, true, dataPrefix1, schemaPrefix1, extraParams); + testShowColumnsInExpansion(false); // 添加节点 dataPrefix = dataPrefix1 && schemaPrefix = p1 后查询 String statement = "select status2 from *;"; @@ -512,6 +601,7 @@ private void testAddAndRemoveStorageEngineWithPrefix() { SQLTestTools.executeAndCompare(session, statement, pathList, valuesList); // 通过 session 接口测试移除节点 + testShowColumnsRemoveStorageEngine(true); List removedStorageEngineList = new ArrayList<>(); removedStorageEngineList.add( new RemovedStorageEngineInfo("127.0.0.1", expPort, "p2" + schemaPrefixSuffix, dataPrefix1)); @@ -523,6 +613,8 @@ private void testAddAndRemoveStorageEngineWithPrefix() { } catch (SessionException e) { LOGGER.error("remove history data source through session api error: ", e); } + testShowColumnsRemoveStorageEngine(false); + // 移除节点 dataPrefix = dataPrefix1 && schemaPrefix = p2 + schemaPrefixSuffix 后再查询 statement = "select * from p2.nt.wf03;"; String expect = @@ -565,6 +657,125 @@ private void testAddAndRemoveStorageEngineWithPrefix() { testShowClusterInfo(2); } + protected void testShowColumnsInExpansion(boolean before) { + String statement = "SHOW COLUMNS nt.wf03.*;"; + String expected = + "Columns:\n" + + "+--------------------+--------+\n" + + "| Path|DataType|\n" + + "+--------------------+--------+\n" + + "|nt.wf03.wt01.status2| LONG|\n" + + "+--------------------+--------+\n" + + "Total line number = 1\n"; + SQLTestTools.executeAndCompare(session, statement, expected); + + statement = "SHOW COLUMNS;"; + if (before) { + expected = + "Columns:\n" + + "+--------------------------------------------------------------------------------------+--------+\n" + + "| Path|DataType|\n" + + "+--------------------------------------------------------------------------------------+--------+\n" + + "| b.b.b| LONG|\n" + + "| ln.wf02.status| BOOLEAN|\n" + + "| ln.wf02.version| BINARY|\n" + + "| nt.wf03.wt01.status2| LONG|\n" + + "| nt.wf04.wt01.temperature| DOUBLE|\n" + + "|zzzzzzzzzzzzzzzzzzzzzzzzzzzz.zzzzzzzzzzzzzzzzzzzzzzzzzzz.zzzzzzzzzzzzzzzzzzzzzzzzzzzzz| LONG|\n" + + "+--------------------------------------------------------------------------------------+--------+\n" + + "Total line number = 6\n"; + } else { // 添加schemaPrefix为p1,dataPrefix为nt.wf03的数据源 + expected = + "Columns:\n" + + "+--------------------------------------------------------------------------------------+--------+\n" + + "| Path|DataType|\n" + + "+--------------------------------------------------------------------------------------+--------+\n" + + "| b.b.b| LONG|\n" + + "| ln.wf02.status| BOOLEAN|\n" + + "| ln.wf02.version| BINARY|\n" + + "| nt.wf03.wt01.status2| LONG|\n" + + "| nt.wf04.wt01.temperature| DOUBLE|\n" + + "| p1.nt.wf03.wt01.status2| LONG|\n" + + "|zzzzzzzzzzzzzzzzzzzzzzzzzzzz.zzzzzzzzzzzzzzzzzzzzzzzzzzz.zzzzzzzzzzzzzzzzzzzzzzzzzzzzz| LONG|\n" + + "+--------------------------------------------------------------------------------------+--------+\n" + + "Total line number = 7\n"; + } + SQLTestTools.executeAndCompare(session, statement, expected); + + statement = "SHOW COLUMNS p1.*;"; + if (before) { + expected = + "Columns:\n" + + "+----+--------+\n" + + "|Path|DataType|\n" + + "+----+--------+\n" + + "+----+--------+\n" + + "Empty set.\n"; + } else { // 添加schemaPrefix为p1,dataPrefix为nt.wf03的数据源 + expected = + "Columns:\n" + + "+-----------------------+--------+\n" + + "| Path|DataType|\n" + + "+-----------------------+--------+\n" + + "|p1.nt.wf03.wt01.status2| LONG|\n" + + "+-----------------------+--------+\n" + + "Total line number = 1\n"; + } + SQLTestTools.executeAndCompare(session, statement, expected); + + statement = "SHOW COLUMNS *.wf03.wt01.*;"; + if (before) { + expected = + "Columns:\n" + + "+--------------------+--------+\n" + + "| Path|DataType|\n" + + "+--------------------+--------+\n" + + "|nt.wf03.wt01.status2| LONG|\n" + + "+--------------------+--------+\n" + + "Total line number = 1\n"; + } else { // 添加schemaPrefix为p1,dataPrefix为nt.wf03的数据源 + expected = + "Columns:\n" + + "+-----------------------+--------+\n" + + "| Path|DataType|\n" + + "+-----------------------+--------+\n" + + "| nt.wf03.wt01.status2| LONG|\n" + + "|p1.nt.wf03.wt01.status2| LONG|\n" + + "+-----------------------+--------+\n" + + "Total line number = 2\n"; + } + SQLTestTools.executeAndCompare(session, statement, expected); + } + + protected void testShowColumnsRemoveStorageEngine(boolean before) { + String statement = "SHOW COLUMNS p1.*, p2.*, p3.*;"; + String expected; + if (before) { + expected = + "Columns:\n" + + "+---------------------------+--------+\n" + + "| Path|DataType|\n" + + "+---------------------------+--------+\n" + + "| p1.nt.wf03.wt01.status2| LONG|\n" + + "| p2.nt.wf03.wt01.status2| LONG|\n" + + "| p3.nt.wf03.wt01.status2| LONG|\n" + + "|p3.nt.wf04.wt01.temperature| DOUBLE|\n" + + "+---------------------------+--------+\n" + + "Total line number = 4\n"; + } else { // schemaPrefix为p2及p3,dataPrefix为nt.wf03的数据源被移除 + expected = + "Columns:\n" + + "+---------------------------+--------+\n" + + "| Path|DataType|\n" + + "+---------------------------+--------+\n" + + "| p1.nt.wf03.wt01.status2| LONG|\n" + + "|p3.nt.wf04.wt01.temperature| DOUBLE|\n" + + "+---------------------------+--------+\n" + + "Total line number = 2\n"; + } + SQLTestTools.executeAndCompare(session, statement, expected); + } + private void testShowClusterInfo(int expected) { try { ClusterInfo clusterInfo = session.getClusterInfo(); @@ -699,8 +910,7 @@ private void testSameKeyWarning() { } } - protected void startStorageEngineWithIginx(int port, boolean hasData, boolean isReadOnly) - throws InterruptedException { + protected void startStorageEngineWithIginx(int port, boolean hasData, boolean isReadOnly) { String scriptPath, iginxPath = ".github/scripts/iginx/iginx.sh"; String os = System.getProperty("os.name").toLowerCase(); boolean isOnMac = false; @@ -713,15 +923,15 @@ protected void startStorageEngineWithIginx(int port, boolean hasData, boolean is if (this instanceof FileSystemCapacityExpansionIT) { if (isOnMac) { - scriptPath = ".github/scripts/dataSources/filesystem_macos.sh"; + scriptPath = ".github/scripts/dataSources/startup/filesystem_macos.sh"; } else { - scriptPath = ".github/scripts/dataSources/filesystem_linux_windows.sh"; + scriptPath = ".github/scripts/dataSources/startup/filesystem_linux_windows.sh"; } } else if (this instanceof ParquetCapacityExpansionIT) { if (isOnMac) { - scriptPath = ".github/scripts/dataSources/parquet_macos.sh"; + scriptPath = ".github/scripts/dataSources/startup/parquet_macos.sh"; } else { - scriptPath = ".github/scripts/dataSources/parquet_linux_windows.sh"; + scriptPath = ".github/scripts/dataSources/startup/parquet_linux_windows.sh"; } } else { throw new IllegalStateException("Only support file system and parquet"); @@ -746,7 +956,7 @@ protected void startStorageEngineWithIginx(int port, boolean hasData, boolean is hasData ? DBCE_PARQUET_FS_TEST_DIR + "/" + PORT_TO_ROOT.get(port) : DBCE_PARQUET_FS_TEST_DIR + "/" + INIT_PATH_LIST.get(0).replace(".", "/"), - DBCE_PARQUET_FS_TEST_DIR + "/iginx_" + PORT_TO_ROOT.get(port), + DBCE_PARQUET_FS_TEST_DIR + "/" + IGINX_DATA_PATH_PREFIX_NAME + PORT_TO_ROOT.get(port), String.valueOf(hasData), String.valueOf(isReadOnly), "core/target/iginx-core-*/conf/config.properties", diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/BaseHistoryDataGenerator.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/BaseHistoryDataGenerator.java index b2660298d5..a5d483dffe 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/BaseHistoryDataGenerator.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/BaseHistoryDataGenerator.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.expansion; import static cn.edu.tsinghua.iginx.integration.expansion.constant.Constant.*; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/constant/Constant.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/constant/Constant.java index 7ef5adbc14..2a1d221631 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/constant/Constant.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/constant/Constant.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.expansion.constant; import cn.edu.tsinghua.iginx.integration.controller.Controller; @@ -13,6 +31,8 @@ public class Constant { public static final String EXP_PORT_NAME = "exp_port"; public static final String READ_ONLY_PORT_NAME = "read_only_port"; + public static final String IGINX_DATA_PATH_PREFIX_NAME = "iginx_"; + // port public static int oriPort = 6667; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/filesystem/FileSystemCapacityExpansionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/filesystem/FileSystemCapacityExpansionIT.java index 657bf4de45..b7b2ce1bd4 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/filesystem/FileSystemCapacityExpansionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/filesystem/FileSystemCapacityExpansionIT.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.expansion.filesystem; import static cn.edu.tsinghua.iginx.thrift.StorageEngineType.filesystem; @@ -22,6 +40,127 @@ protected void testInvalidDummyParams( LOGGER.info("filesystem skips test for wrong dummy engine params."); } + // filesystem中,所有dummy数据都识别为BINARY + @Override + protected void testShowColumnsInExpansion(boolean before) { + String statement = "SHOW COLUMNS nt.wf03.*;"; + String expected = + "Columns:\n" + + "+--------------------+--------+\n" + + "| Path|DataType|\n" + + "+--------------------+--------+\n" + + "|nt.wf03.wt01.status2| BINARY|\n" + + "+--------------------+--------+\n" + + "Total line number = 1\n"; + SQLTestTools.executeAndCompare(session, statement, expected); + + statement = "SHOW COLUMNS;"; + if (before) { + expected = + "Columns:\n" + + "+------------------------+--------+\n" + + "| Path|DataType|\n" + + "+------------------------+--------+\n" + + "| ln.wf02.status| BOOLEAN|\n" + + "| ln.wf02.version| BINARY|\n" + + "| nt.wf03.wt01.status2| BINARY|\n" + + "|nt.wf04.wt01.temperature| BINARY|\n" + + "+------------------------+--------+\n" + + "Total line number = 4\n"; + } else { // 添加schemaPrefix为p1,dataPrefix为nt.wf03的数据源 + expected = + "Columns:\n" + + "+------------------------+--------+\n" + + "| Path|DataType|\n" + + "+------------------------+--------+\n" + + "| ln.wf02.status| BOOLEAN|\n" + + "| ln.wf02.version| BINARY|\n" + + "| nt.wf03.wt01.status2| BINARY|\n" + + "|nt.wf04.wt01.temperature| BINARY|\n" + + "| p1.nt.wf03.wt01.status2| BINARY|\n" + + "+------------------------+--------+\n" + + "Total line number = 5\n"; + } + SQLTestTools.executeAndCompare(session, statement, expected); + + if (before) { + statement = "SHOW COLUMNS p1.*;"; + expected = + "Columns:\n" + + "+----+--------+\n" + + "|Path|DataType|\n" + + "+----+--------+\n" + + "+----+--------+\n" + + "Empty set.\n"; + } else { // 添加schemaPrefix为p1,dataPrefix为nt.wf03的数据源 + statement = "SHOW COLUMNS p1.*;"; + expected = + "Columns:\n" + + "+-----------------------+--------+\n" + + "| Path|DataType|\n" + + "+-----------------------+--------+\n" + + "|p1.nt.wf03.wt01.status2| BINARY|\n" + + "+-----------------------+--------+\n" + + "Total line number = 1\n"; + } + SQLTestTools.executeAndCompare(session, statement, expected); + + statement = "SHOW COLUMNS *.wf03.wt01.*;"; + if (before) { + expected = + "Columns:\n" + + "+--------------------+--------+\n" + + "| Path|DataType|\n" + + "+--------------------+--------+\n" + + "|nt.wf03.wt01.status2| BINARY|\n" + + "+--------------------+--------+\n" + + "Total line number = 1\n"; + } else { // 添加schemaPrefix为p1,dataPrefix为nt.wf03的数据源 + expected = + "Columns:\n" + + "+-----------------------+--------+\n" + + "| Path|DataType|\n" + + "+-----------------------+--------+\n" + + "| nt.wf03.wt01.status2| BINARY|\n" + + "|p1.nt.wf03.wt01.status2| BINARY|\n" + + "+-----------------------+--------+\n" + + "Total line number = 2\n"; + } + SQLTestTools.executeAndCompare(session, statement, expected); + } + + // filesystem中,所有dummy数据都识别为BINARY + @Override + protected void testShowColumnsRemoveStorageEngine(boolean before) { + String statement = "SHOW COLUMNS p1.*, p2.*, p3.*;"; + String expected; + if (before) { + expected = + "Columns:\n" + + "+---------------------------+--------+\n" + + "| Path|DataType|\n" + + "+---------------------------+--------+\n" + + "| p1.nt.wf03.wt01.status2| BINARY|\n" + + "| p2.nt.wf03.wt01.status2| BINARY|\n" + + "| p3.nt.wf03.wt01.status2| BINARY|\n" + + "|p3.nt.wf04.wt01.temperature| BINARY|\n" + + "+---------------------------+--------+\n" + + "Total line number = 4\n"; + } else { // 移除schemaPrefix为p2及p3,dataPrefix为nt.wf03的数据源 + expected = + "Columns:\n" + + "+---------------------------+--------+\n" + + "| Path|DataType|\n" + + "+---------------------------+--------+\n" + + "| p1.nt.wf03.wt01.status2| BINARY|\n" + + "|p3.nt.wf04.wt01.temperature| BINARY|\n" + + "+---------------------------+--------+\n" + + "Total line number = 2\n"; + } + SQLTestTools.executeAndCompare(session, statement, expected); + } + + // filesystem中,所有dummy数据都识别为BINARY @Override public void testShowColumns() { String statement = "SHOW COLUMNS mn.*;"; @@ -94,4 +233,11 @@ public void testShowColumns() { + "Total line number = 3\n"; SQLTestTools.executeAndCompare(session, statement, expected); } + + // no param is allowed to be updated + @Override + protected void updateParams(int port) {} + + @Override + protected void restoreParams(int port) {} } diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/filesystem/FileSystemHistoryDataGenerator.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/filesystem/FileSystemHistoryDataGenerator.java index 0b3ebfea51..789619a2ae 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/filesystem/FileSystemHistoryDataGenerator.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/filesystem/FileSystemHistoryDataGenerator.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.expansion.filesystem; import static cn.edu.tsinghua.iginx.integration.expansion.constant.Constant.*; @@ -43,14 +61,23 @@ public void writeHistoryData( @Override public void clearHistoryDataForGivenPort(int port) { - Path rootPath = Paths.get(PORT_TO_ROOT.get(port)); - if (!Files.exists(rootPath)) { - return; - } - try (Stream walk = Files.walk(rootPath)) { - walk.sorted(Comparator.reverseOrder()).forEach(this::deleteDirectoryStream); - } catch (IOException e) { - LOGGER.error("delete {} failure", rootPath); + Path rootPath; + for (int i = 0; i < 2; i++) { + if (i == 0) { + rootPath = Paths.get(PORT_TO_ROOT.get(port)); + } else { + rootPath = Paths.get(IGINX_DATA_PATH_PREFIX_NAME + PORT_TO_ROOT.get(port)); + } + LOGGER.info("clear path {}", rootPath.toFile().getAbsolutePath()); + if (!Files.exists(rootPath)) { + LOGGER.info("path {} does not exist", rootPath.toFile().getAbsolutePath()); + continue; + } + try (Stream walk = Files.walk(rootPath)) { + walk.sorted(Comparator.reverseOrder()).forEach(this::deleteDirectoryStream); + } catch (IOException e) { + LOGGER.error("delete {} failure", rootPath); + } } } diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/datasource/DataSourceIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/filesystem/datasource/DataSourceIT.java similarity index 74% rename from test/src/test/java/cn/edu/tsinghua/iginx/integration/datasource/DataSourceIT.java rename to test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/filesystem/datasource/DataSourceIT.java index aede81abd9..b5378f9eb8 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/datasource/DataSourceIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/filesystem/datasource/DataSourceIT.java @@ -1,4 +1,22 @@ -package cn.edu.tsinghua.iginx.integration.datasource; +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.integration.expansion.filesystem.datasource; import static org.junit.Assert.assertNull; import static org.junit.Assert.fail; @@ -11,10 +29,15 @@ import cn.edu.tsinghua.iginx.engine.shared.data.read.Row; import cn.edu.tsinghua.iginx.engine.shared.data.read.RowStream; import cn.edu.tsinghua.iginx.engine.shared.data.write.DataView; +import cn.edu.tsinghua.iginx.engine.shared.function.FunctionCall; +import cn.edu.tsinghua.iginx.engine.shared.function.FunctionParams; +import cn.edu.tsinghua.iginx.engine.shared.function.system.Count; import cn.edu.tsinghua.iginx.engine.shared.operator.Delete; import cn.edu.tsinghua.iginx.engine.shared.operator.Insert; import cn.edu.tsinghua.iginx.engine.shared.operator.Project; +import cn.edu.tsinghua.iginx.engine.shared.operator.SetTransform; import cn.edu.tsinghua.iginx.engine.shared.source.FragmentSource; +import cn.edu.tsinghua.iginx.engine.shared.source.OperatorSource; import cn.edu.tsinghua.iginx.integration.controller.Controller; import cn.edu.tsinghua.iginx.integration.tool.ConfLoader; import cn.edu.tsinghua.iginx.integration.tool.DBConf; @@ -241,4 +264,41 @@ public void deleteRowsInMultiPart() throws PhysicalException { checkResult(result); checkRowCount(result.getRowStream(), 500000); } + + @Test + public void frequentCountAndInsertOverlapData() throws PhysicalException { + FragmentSource source = MockClassGenerator.genFragmentSource(); + DataArea dataArea = MockClassGenerator.genDataArea(); + + Project project = new Project(source, Collections.singletonList("*"), null); + OperatorSource projectSource = new OperatorSource(project); + FunctionParams countParams = new FunctionParams(Collections.singletonList("*")); + FunctionCall countCall = new FunctionCall(Count.getInstance(), countParams); + SetTransform countOperator = + new SetTransform(projectSource, Collections.singletonList(countCall)); + + Assume.assumeTrue(storage.isSupportProjectWithSetTransform(countOperator, dataArea)); + + for (int seed = 0; seed < 10; seed++) { + Random random = new Random(seed); + HashSet keys = new HashSet<>(); + for (long key = 0; key < 1000; key += 100) { + int rows = random.nextInt(150) + 1; + for (int i = 0; i < rows; i++) { + keys.add(key + i); + } + insertData(key, rows, "us.d1.s1", "us.d1.s2", "us.d1.s3", "us.d1.s4"); + TaskExecuteResult result = + storage.executeProjectWithSetTransform(project, countOperator, dataArea); + checkResult(result); + RowStream rowStream = result.getRowStream(); + Assert.assertTrue(rowStream.hasNext()); + Row row = rowStream.next(); + long count = keys.size(); + Assert.assertArrayEquals(new Object[] {count, count, count, count}, row.getValues()); + Assert.assertFalse(rowStream.hasNext()); + } + clear(); + } + } } diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/influxdb/InfluxDBCapacityExpansionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/influxdb/InfluxDBCapacityExpansionIT.java index 47d638e437..3a089246c7 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/influxdb/InfluxDBCapacityExpansionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/influxdb/InfluxDBCapacityExpansionIT.java @@ -1,6 +1,26 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.expansion.influxdb; +import static cn.edu.tsinghua.iginx.integration.expansion.utils.SQLTestTools.executeShellScript; import static cn.edu.tsinghua.iginx.thrift.StorageEngineType.influxdb; +import static org.junit.Assert.fail; import cn.edu.tsinghua.iginx.integration.controller.Controller; import cn.edu.tsinghua.iginx.integration.expansion.BaseCapacityExpansionIT; @@ -26,9 +46,35 @@ public InfluxDBCapacityExpansionIT() { Constant.readOnlyPort = dbConf.getDBCEPortMap().get(Constant.READ_ONLY_PORT_NAME); wrongExtraParams.add( "username:user, password:12345678, token:testToken, organization:wrongOrg"); + updatedParams.put("organization", "newOrg"); } // dummy key range cannot be extended yet @Override protected void queryExtendedKeyDummy() {} + + @Override + protected void updateParams(int port) { + changeParams(port, "newOrg"); + } + + @Override + protected void restoreParams(int port) { + changeParams(port, "testOrg"); + } + + private void changeParams(int port, String newOrgName) { + String scriptPath = updateParamsScriptDir + "influxdb.sh"; + String os = System.getProperty("os.name").toLowerCase(); + if (os.contains("mac")) { + scriptPath = updateParamsScriptDir + "influxdb_macos.sh"; + } else if (os.contains("win")) { + scriptPath = updateParamsScriptDir + "influxdb_windows.sh"; + } + // 脚本参数:对应端口,新参数 + int res = executeShellScript(scriptPath, String.valueOf(port), newOrgName); + if (res != 0) { + fail("Fail to update influxdb params."); + } + } } diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/influxdb/InfluxDBHistoryDataGenerator.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/influxdb/InfluxDBHistoryDataGenerator.java index bd246a7c9d..8498008528 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/influxdb/InfluxDBHistoryDataGenerator.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/influxdb/InfluxDBHistoryDataGenerator.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.expansion.influxdb; import cn.edu.tsinghua.iginx.integration.expansion.BaseHistoryDataGenerator; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/iotdb/IoTDB12CapacityExpansionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/iotdb/IoTDB12CapacityExpansionIT.java index 0361e3d49f..1242fbcbb0 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/iotdb/IoTDB12CapacityExpansionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/iotdb/IoTDB12CapacityExpansionIT.java @@ -1,6 +1,26 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.expansion.iotdb; +import static cn.edu.tsinghua.iginx.integration.expansion.utils.SQLTestTools.executeShellScript; import static cn.edu.tsinghua.iginx.thrift.StorageEngineType.iotdb12; +import static org.junit.Assert.fail; import cn.edu.tsinghua.iginx.integration.expansion.BaseCapacityExpansionIT; import org.slf4j.Logger; @@ -17,5 +37,31 @@ public IoTDB12CapacityExpansionIT() { new IoTDB12HistoryDataGenerator()); wrongExtraParams.add("username:root, password:wrong, sessionPoolSize:20"); wrongExtraParams.add("username:wrong, password:root, sessionPoolSize:20"); + updatedParams.put("password", "newPassword"); + } + + @Override + protected void updateParams(int port) { + changeParams(port, "root", "newPassword"); + } + + @Override + protected void restoreParams(int port) { + changeParams(port, "newPassword", "root"); + } + + private void changeParams(int port, String oldPw, String newPw) { + String scriptPath = updateParamsScriptDir + "iotdb.sh"; + String os = System.getProperty("os.name").toLowerCase(); + if (os.contains("mac")) { + scriptPath = updateParamsScriptDir + "iotdb_macos.sh"; + } else if (os.contains("win")) { + scriptPath = updateParamsScriptDir + "iotdb_windows.sh"; + } + // 脚本参数:对应端口,旧密码,新密码 + int res = executeShellScript(scriptPath, String.valueOf(port), oldPw, newPw); + if (res != 0) { + fail("Fail to update iotdb params."); + } } } diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/iotdb/IoTDB12HistoryDataGenerator.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/iotdb/IoTDB12HistoryDataGenerator.java index df9f3cdfc4..33680c0968 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/iotdb/IoTDB12HistoryDataGenerator.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/iotdb/IoTDB12HistoryDataGenerator.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.expansion.iotdb; import cn.edu.tsinghua.iginx.integration.expansion.BaseHistoryDataGenerator; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/mongodb/MongoDBCapacityExpansionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/mongodb/MongoDBCapacityExpansionIT.java index 603db64f97..1b6bf7b3c7 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/mongodb/MongoDBCapacityExpansionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/mongodb/MongoDBCapacityExpansionIT.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.expansion.mongodb; import static cn.edu.tsinghua.iginx.thrift.StorageEngineType.mongodb; @@ -23,6 +41,145 @@ public MongoDBCapacityExpansionIT() { Constant.readOnlyPort = dbConf.getDBCEPortMap().get(Constant.READ_ONLY_PORT_NAME); } + @Override + protected void testShowColumnsInExpansion(boolean before) { + String statement = "SHOW COLUMNS nt.wf03.*;"; + String expected = + "Columns:\n" + + "+--------------------+--------+\n" + + "| Path|DataType|\n" + + "+--------------------+--------+\n" + + "| nt.wf03._id| INTEGER|\n" + + "|nt.wf03.wt01.status2| LONG|\n" + + "+--------------------+--------+\n" + + "Total line number = 2\n"; + SQLTestTools.executeAndCompare(session, statement, expected); + + statement = "SHOW COLUMNS;"; + if (before) { + expected = + "Columns:\n" + + "+--------------------------------------------------------------------------------------+--------+\n" + + "| Path|DataType|\n" + + "+--------------------------------------------------------------------------------------+--------+\n" + + "| b.b._id| INTEGER|\n" + + "| b.b.b| LONG|\n" + + "| ln.wf02.status| BOOLEAN|\n" + + "| ln.wf02.version| BINARY|\n" + + "| nt.wf03._id| INTEGER|\n" + + "| nt.wf03.wt01.status2| LONG|\n" + + "| nt.wf04._id| INTEGER|\n" + + "| nt.wf04.wt01.temperature| DOUBLE|\n" + + "| zzzzzzzzzzzzzzzzzzzzzzzzzzzz.zzzzzzzzzzzzzzzzzzzzzzzzzzz._id| INTEGER|\n" + + "|zzzzzzzzzzzzzzzzzzzzzzzzzzzz.zzzzzzzzzzzzzzzzzzzzzzzzzzz.zzzzzzzzzzzzzzzzzzzzzzzzzzzzz| LONG|\n" + + "+--------------------------------------------------------------------------------------+--------+\n" + + "Total line number = 10\n"; + } else { // 添加schemaPrefix为p1,dataPrefix为nt.wf03的数据源 + expected = + "Columns:\n" + + "+--------------------------------------------------------------------------------------+--------+\n" + + "| Path|DataType|\n" + + "+--------------------------------------------------------------------------------------+--------+\n" + + "| b.b._id| INTEGER|\n" + + "| b.b.b| LONG|\n" + + "| ln.wf02.status| BOOLEAN|\n" + + "| ln.wf02.version| BINARY|\n" + + "| nt.wf03._id| INTEGER|\n" + + "| nt.wf03.wt01.status2| LONG|\n" + + "| nt.wf04._id| INTEGER|\n" + + "| nt.wf04.wt01.temperature| DOUBLE|\n" + + "| p1.nt.wf03._id| INTEGER|\n" + + "| p1.nt.wf03.wt01.status2| LONG|\n" + + "| zzzzzzzzzzzzzzzzzzzzzzzzzzzz.zzzzzzzzzzzzzzzzzzzzzzzzzzz._id| INTEGER|\n" + + "|zzzzzzzzzzzzzzzzzzzzzzzzzzzz.zzzzzzzzzzzzzzzzzzzzzzzzzzz.zzzzzzzzzzzzzzzzzzzzzzzzzzzzz| LONG|\n" + + "+--------------------------------------------------------------------------------------+--------+\n" + + "Total line number = 12\n"; + } + SQLTestTools.executeAndCompare(session, statement, expected); + + if (before) { + statement = "SHOW COLUMNS p1.*;"; + expected = + "Columns:\n" + + "+----+--------+\n" + + "|Path|DataType|\n" + + "+----+--------+\n" + + "+----+--------+\n" + + "Empty set.\n"; + } else { // 添加schemaPrefix为p1,dataPrefix为nt.wf03的数据源 + statement = "SHOW COLUMNS p1.*;"; + expected = + "Columns:\n" + + "+-----------------------+--------+\n" + + "| Path|DataType|\n" + + "+-----------------------+--------+\n" + + "| p1.nt.wf03._id| INTEGER|\n" + + "|p1.nt.wf03.wt01.status2| LONG|\n" + + "+-----------------------+--------+\n" + + "Total line number = 2\n"; + } + SQLTestTools.executeAndCompare(session, statement, expected); + + statement = "SHOW COLUMNS *.wf03.wt01.*;"; + if (before) { + expected = + "Columns:\n" + + "+--------------------+--------+\n" + + "| Path|DataType|\n" + + "+--------------------+--------+\n" + + "|nt.wf03.wt01.status2| LONG|\n" + + "+--------------------+--------+\n" + + "Total line number = 1\n"; + } else { // 添加schemaPrefix为p1,dataPrefix为nt.wf03的数据源 + expected = + "Columns:\n" + + "+-----------------------+--------+\n" + + "| Path|DataType|\n" + + "+-----------------------+--------+\n" + + "| nt.wf03.wt01.status2| LONG|\n" + + "|p1.nt.wf03.wt01.status2| LONG|\n" + + "+-----------------------+--------+\n" + + "Total line number = 2\n"; + } + SQLTestTools.executeAndCompare(session, statement, expected); + } + + @Override + protected void testShowColumnsRemoveStorageEngine(boolean before) { + String statement = "SHOW COLUMNS p1.*, p2.*, p3.*;"; + String expected; + if (before) { + expected = + "Columns:\n" + + "+---------------------------+--------+\n" + + "| Path|DataType|\n" + + "+---------------------------+--------+\n" + + "| p1.nt.wf03._id| INTEGER|\n" + + "| p1.nt.wf03.wt01.status2| LONG|\n" + + "| p2.nt.wf03._id| INTEGER|\n" + + "| p2.nt.wf03.wt01.status2| LONG|\n" + + "| p3.nt.wf03._id| INTEGER|\n" + + "| p3.nt.wf03.wt01.status2| LONG|\n" + + "| p3.nt.wf04._id| INTEGER|\n" + + "|p3.nt.wf04.wt01.temperature| DOUBLE|\n" + + "+---------------------------+--------+\n" + + "Total line number = 8\n"; + } else { // 移除schemaPrefix为p2及p3,dataPrefix为nt.wf03的数据源 + expected = + "Columns:\n" + + "+---------------------------+--------+\n" + + "| Path|DataType|\n" + + "+---------------------------+--------+\n" + + "| p1.nt.wf03._id| INTEGER|\n" + + "| p1.nt.wf03.wt01.status2| LONG|\n" + + "| p3.nt.wf04._id| INTEGER|\n" + + "|p3.nt.wf04.wt01.temperature| DOUBLE|\n" + + "+---------------------------+--------+\n" + + "Total line number = 4\n"; + } + SQLTestTools.executeAndCompare(session, statement, expected); + } + @Override protected void testQuerySpecialHistoryData() { testProject(); @@ -335,4 +492,11 @@ public void testShowColumns() { + "Total line number = 3\n"; SQLTestTools.executeAndCompare(session, statement, expected); } + + // no param is allowed to be updated + @Override + protected void updateParams(int port) {} + + @Override + protected void restoreParams(int port) {} } diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/mongodb/MongoDBHistoryDataGenerator.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/mongodb/MongoDBHistoryDataGenerator.java index b2ffb3b2cb..7c764705ad 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/mongodb/MongoDBHistoryDataGenerator.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/mongodb/MongoDBHistoryDataGenerator.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.expansion.mongodb; import static cn.edu.tsinghua.iginx.integration.expansion.constant.Constant.readOnlyPort; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/mysql/MySQLCapacityExpansionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/mysql/MySQLCapacityExpansionIT.java index f75fbea476..a0d8944aeb 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/mysql/MySQLCapacityExpansionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/mysql/MySQLCapacityExpansionIT.java @@ -1,5 +1,26 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.expansion.mysql; +import static cn.edu.tsinghua.iginx.integration.expansion.utils.SQLTestTools.executeShellScript; +import static org.junit.Assert.fail; + import cn.edu.tsinghua.iginx.integration.controller.Controller; import cn.edu.tsinghua.iginx.integration.expansion.BaseCapacityExpansionIT; import cn.edu.tsinghua.iginx.integration.expansion.constant.Constant; @@ -17,12 +38,40 @@ public MySQLCapacityExpansionIT() { super( StorageEngineType.relational, "engine:mysql, username:root," - + "meta_properties_path:dataSources/relational/src/main/resources/mysql-meta-template.properties", + + "meta_properties_path:dataSource/relational/src/main/resources/mysql-meta-template.properties", new MySQLHistoryDataGenerator()); ConfLoader conf = new ConfLoader(Controller.CONFIG_FILE); DBConf dbConf = conf.loadDBConf(conf.getStorageType()); Constant.oriPort = dbConf.getDBCEPortMap().get(Constant.ORI_PORT_NAME); Constant.expPort = dbConf.getDBCEPortMap().get(Constant.EXP_PORT_NAME); Constant.readOnlyPort = dbConf.getDBCEPortMap().get(Constant.READ_ONLY_PORT_NAME); + updatedParams.put("password", "newPassword"); + } + + @Override + protected void updateParams(int port) { + changeParams(port, null, "newPassword"); + } + + @Override + protected void restoreParams(int port) { + changeParams(port, "newPassword", null); + } + + private void changeParams(int port, String oldPw, String newPw) { + String scriptPath = updateParamsScriptDir + "mysql.sh"; + String mode = oldPw == null ? "set" : "unset"; + String os = System.getProperty("os.name").toLowerCase(); + if (os.contains("mac")) { + scriptPath = updateParamsScriptDir + "mysql_macos.sh"; + } else if (os.contains("win")) { + scriptPath = updateParamsScriptDir + "mysql_windows.sh"; + } + // 脚本参数:对应端口,模式(是在无密码条件下设置密码,还是在有密码条件下去掉密码),需要设置的密码/需要被去掉的密码 + int res = + executeShellScript(scriptPath, String.valueOf(port), mode, oldPw == null ? newPw : oldPw); + if (res != 0) { + fail("Fail to update mysql params."); + } } } diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/mysql/MySQLHistoryDataGenerator.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/mysql/MySQLHistoryDataGenerator.java index 298b03100f..b7d79f0ba1 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/mysql/MySQLHistoryDataGenerator.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/mysql/MySQLHistoryDataGenerator.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.expansion.mysql; import cn.edu.tsinghua.iginx.integration.expansion.BaseHistoryDataGenerator; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/parquet/ParquetCapacityExpansionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/parquet/ParquetCapacityExpansionIT.java index 178847bcd6..d518928b89 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/parquet/ParquetCapacityExpansionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/parquet/ParquetCapacityExpansionIT.java @@ -1,8 +1,27 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.expansion.parquet; import static cn.edu.tsinghua.iginx.thrift.StorageEngineType.parquet; import cn.edu.tsinghua.iginx.integration.expansion.BaseCapacityExpansionIT; +import cn.edu.tsinghua.iginx.integration.expansion.utils.SQLTestTools; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -20,4 +39,98 @@ protected void testInvalidDummyParams( int port, boolean hasData, boolean isReadOnly, String dataPrefix, String schemaPrefix) { LOGGER.info("parquet skips test for wrong dummy engine params."); } + + @Override + protected void testShowColumnsInExpansion(boolean before) { + String statement = "SHOW COLUMNS nt.wf03.*;"; + String expected = + "Columns:\n" + + "+--------------------+--------+\n" + + "| Path|DataType|\n" + + "+--------------------+--------+\n" + + "|nt.wf03.wt01.status2| LONG|\n" + + "+--------------------+--------+\n" + + "Total line number = 1\n"; + SQLTestTools.executeAndCompare(session, statement, expected); + + statement = "SHOW COLUMNS;"; + if (before) { + expected = + "Columns:\n" + + "+------------------------+--------+\n" + + "| Path|DataType|\n" + + "+------------------------+--------+\n" + + "| ln.wf02.status| BOOLEAN|\n" + + "| ln.wf02.version| BINARY|\n" + + "| nt.wf03.wt01.status2| LONG|\n" + + "|nt.wf04.wt01.temperature| DOUBLE|\n" + + "+------------------------+--------+\n" + + "Total line number = 4\n"; + } else { // 添加schemaPrefix为p1,dataPrefix为nt.wf03的数据源 + expected = + "Columns:\n" + + "+------------------------+--------+\n" + + "| Path|DataType|\n" + + "+------------------------+--------+\n" + + "| ln.wf02.status| BOOLEAN|\n" + + "| ln.wf02.version| BINARY|\n" + + "| nt.wf03.wt01.status2| LONG|\n" + + "|nt.wf04.wt01.temperature| DOUBLE|\n" + + "| p1.nt.wf03.wt01.status2| LONG|\n" + + "+------------------------+--------+\n" + + "Total line number = 5\n"; + } + SQLTestTools.executeAndCompare(session, statement, expected); + + statement = "SHOW COLUMNS p1.*;"; + if (before) { + expected = + "Columns:\n" + + "+----+--------+\n" + + "|Path|DataType|\n" + + "+----+--------+\n" + + "+----+--------+\n" + + "Empty set.\n"; + } else { // 添加schemaPrefix为p1,dataPrefix为nt.wf03的数据源 + expected = + "Columns:\n" + + "+-----------------------+--------+\n" + + "| Path|DataType|\n" + + "+-----------------------+--------+\n" + + "|p1.nt.wf03.wt01.status2| LONG|\n" + + "+-----------------------+--------+\n" + + "Total line number = 1\n"; + } + SQLTestTools.executeAndCompare(session, statement, expected); + + statement = "SHOW COLUMNS *.wf03.wt01.*;"; + if (before) { + expected = + "Columns:\n" + + "+--------------------+--------+\n" + + "| Path|DataType|\n" + + "+--------------------+--------+\n" + + "|nt.wf03.wt01.status2| LONG|\n" + + "+--------------------+--------+\n" + + "Total line number = 1\n"; + } else { // 添加schemaPrefix为p1,dataPrefix为nt.wf03的数据源 + expected = + "Columns:\n" + + "+-----------------------+--------+\n" + + "| Path|DataType|\n" + + "+-----------------------+--------+\n" + + "| nt.wf03.wt01.status2| LONG|\n" + + "|p1.nt.wf03.wt01.status2| LONG|\n" + + "+-----------------------+--------+\n" + + "Total line number = 2\n"; + } + SQLTestTools.executeAndCompare(session, statement, expected); + } + + // no param is allowed to be updated + @Override + protected void updateParams(int port) {} + + @Override + protected void restoreParams(int port) {} } diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/parquet/ParquetHistoryDataGenerator.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/parquet/ParquetHistoryDataGenerator.java index fb8edc8443..0601f77886 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/parquet/ParquetHistoryDataGenerator.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/parquet/ParquetHistoryDataGenerator.java @@ -1,17 +1,43 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.expansion.parquet; import static cn.edu.tsinghua.iginx.integration.expansion.BaseCapacityExpansionIT.DBCE_PARQUET_FS_TEST_DIR; -import static cn.edu.tsinghua.iginx.integration.expansion.constant.Constant.*; +import static cn.edu.tsinghua.iginx.integration.expansion.constant.Constant.IGINX_DATA_PATH_PREFIX_NAME; +import static cn.edu.tsinghua.iginx.integration.expansion.constant.Constant.PARQUET_PARAMS; +import cn.edu.tsinghua.iginx.engine.physical.storage.domain.ColumnKey; import cn.edu.tsinghua.iginx.format.parquet.ParquetWriter; import cn.edu.tsinghua.iginx.format.parquet.example.ExampleParquetWriter; import cn.edu.tsinghua.iginx.integration.expansion.BaseHistoryDataGenerator; import cn.edu.tsinghua.iginx.thrift.DataType; import java.io.File; import java.io.IOException; -import java.nio.file.*; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.SortedMap; import java.util.TreeMap; @@ -148,17 +174,24 @@ public void clearHistoryDataForGivenPort(int port) { "delete {}/{} error: does not exist or is not a file.", dir, file.getAbsoluteFile()); } + List pathList = + Arrays.asList(IT_DATA_DIR, IGINX_DATA_PATH_PREFIX_NAME + PARQUET_PARAMS.get(port).get(0)); // delete the normal IT data - dir = DBCE_PARQUET_FS_TEST_DIR + System.getProperty("file.separator") + IT_DATA_DIR; - parquetPath = Paths.get("../" + dir); - - try { - Files.walkFileTree(parquetPath, new DeleteFileVisitor()); - } catch (NoSuchFileException e) { - LOGGER.warn( - "no such file or directory: {}", new File(parquetPath.toString()).getAbsoluteFile()); - } catch (IOException e) { - LOGGER.warn("delete {} error: ", new File(parquetPath.toString()).getAbsoluteFile(), e); + for (String path : pathList) { + Path dataPath = Paths.get(path); + if (Files.exists(dataPath)) { + try { + Files.walkFileTree(dataPath, new DeleteFileVisitor()); + } catch (NoSuchFileException e) { + LOGGER.warn( + "no such file or directory: {}", new File(dataPath.toString()).getAbsoluteFile()); + } catch (IOException e) { + LOGGER.warn("delete {} error: ", new File(dataPath.toString()).getAbsoluteFile(), e); + } + } else { + LOGGER.warn( + "delete {} error: does not exist.", new File(dataPath.toString()).getAbsoluteFile()); + } } } @@ -176,7 +209,7 @@ public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOEx } } - public static final String KEY_FIELD_NAME = "*"; + public static final String KEY_FIELD_NAME = ColumnKey.KEY.getPath(); private static void flushRows( List pathList, diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLCapacityExpansionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLCapacityExpansionIT.java index 0ced1e22a1..03d62aaeca 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLCapacityExpansionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLCapacityExpansionIT.java @@ -1,5 +1,26 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.expansion.postgresql; +import static cn.edu.tsinghua.iginx.integration.expansion.utils.SQLTestTools.executeShellScript; +import static org.junit.Assert.fail; + import cn.edu.tsinghua.iginx.integration.controller.Controller; import cn.edu.tsinghua.iginx.integration.expansion.BaseCapacityExpansionIT; import cn.edu.tsinghua.iginx.integration.expansion.constant.Constant; @@ -7,6 +28,8 @@ import cn.edu.tsinghua.iginx.integration.tool.ConfLoader; import cn.edu.tsinghua.iginx.integration.tool.DBConf; import cn.edu.tsinghua.iginx.thrift.StorageEngineType; +import java.sql.Connection; +import java.sql.Statement; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -26,11 +49,18 @@ public PostgreSQLCapacityExpansionIT() { Constant.readOnlyPort = dbConf.getDBCEPortMap().get(Constant.READ_ONLY_PORT_NAME); wrongExtraParams.add("username:wrong, password:postgres"); // wrong password situation cannot be tested because trust mode is used + + updatedParams.put("password", "newPassword"); } - /** 执行一个简单查询1000次,测试是否会使连接池耗尽,来验证PG的dummy查询是否正确释放连接 */ @Override protected void testQuerySpecialHistoryData() { + testRepeatsQuery(); + testConcatFunction(); + } + + /** 执行一个简单查询1000次,测试是否会使连接池耗尽,来验证PG的dummy查询是否正确释放连接 */ + private void testRepeatsQuery() { String statement = "select * from ln;"; String expect = "ResultSets:\n" @@ -49,4 +79,93 @@ protected void testQuerySpecialHistoryData() { SQLTestTools.executeAndCompare(session, statement, expect); } } + + /** 创建一个1000列的表,进行查询,看concat函数是否会报错。 */ + protected void testConcatFunction() { + int cols = 1000; + // 用pg的session创建一个1000列的表 + String createTableStatement = "CREATE TABLE test_concat ("; + for (int i = 0; i < cols; i++) { + createTableStatement += "col" + i + " text,"; + } + createTableStatement = createTableStatement.substring(0, createTableStatement.length() - 1); + createTableStatement += ");"; + + StringBuilder insert1 = new StringBuilder(); + insert1.append("test_concat ("); + for (int i = 0; i < cols; i++) { + insert1.append("col" + i + ","); + } + insert1.deleteCharAt(insert1.length() - 1); + insert1.append(") "); + StringBuilder insert2 = new StringBuilder(); + insert2.append("("); + for (int i = 0; i < cols; i++) { + insert2.append("'test',"); + } + insert2.deleteCharAt(insert2.length() - 1); + insert2.append(");"); + String insertStatement = + String.format(PostgreSQLHistoryDataGenerator.INSERT_STATEMENT, insert1, insert2); + + try { + Connection connection = + PostgreSQLHistoryDataGenerator.connect(5432, true, null, "postgres", "postgres"); + Statement stmt = connection.createStatement(); + stmt.execute( + String.format(PostgreSQLHistoryDataGenerator.CREATE_DATABASE_STATEMENT, "test_concat")); + + Connection testConnection = + PostgreSQLHistoryDataGenerator.connect( + 5432, false, "test_concat", "postgres", "postgres"); + Statement testStmt = testConnection.createStatement(); + testStmt.execute(createTableStatement); + testStmt.execute(insertStatement); + + // 不用插入数据直接查,不报错即可,不需要比较结果 + String query = "SELECT col1 FROM test_concat.*;"; + String expect = + "ResultSets:\n" + + "+---------+----------------------------+\n" + + "| key|test_concat.test_concat.col1|\n" + + "+---------+----------------------------+\n" + + "|262526998| test|\n" + + "+---------+----------------------------+\n" + + "Total line number = 1\n"; + SQLTestTools.executeAndCompare(session, query, expect); + + testConnection.close(); + stmt.execute( + String.format(PostgreSQLHistoryDataGenerator.DROP_DATABASE_STATEMENT, "test_concat")); + connection.close(); + } catch (Exception e) { + LOGGER.error("create table failed", e); + assert false; + } + } + + @Override + protected void updateParams(int port) { + changeParams(port, "postgres", "newPassword"); + } + + @Override + protected void restoreParams(int port) { + changeParams(port, "newPassword", "postgres"); + } + + private void changeParams(int port, String oldPw, String newPw) { + String scriptPath = updateParamsScriptDir + "postgresql.sh"; + String os = System.getProperty("os.name").toLowerCase(); + if (os.contains("mac")) { + scriptPath = updateParamsScriptDir + "postgresql_macos.sh"; + } else if (os.contains("win")) { + scriptPath = updateParamsScriptDir + "postgresql_windows.sh"; + } + // 脚本参数:对应端口,旧密码,新密码 + int res = executeShellScript(scriptPath, String.valueOf(port), oldPw, newPw); + if (res != 0) { + fail("Fail to update postgresql params."); + } + } } diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGenerator.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGenerator.java index 69d45ecc94..afeb554045 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGenerator.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGenerator.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.expansion.postgresql; import cn.edu.tsinghua.iginx.integration.expansion.BaseHistoryDataGenerator; @@ -17,13 +35,13 @@ public class PostgreSQLHistoryDataGenerator extends BaseHistoryDataGenerator { private static final String QUERY_DATABASES_STATEMENT = "SELECT datname FROM pg_database;"; - private static final String CREATE_DATABASE_STATEMENT = "CREATE DATABASE \"%s\";"; + public static final String CREATE_DATABASE_STATEMENT = "CREATE DATABASE \"%s\";"; - private static final String CREATE_TABLE_STATEMENT = "CREATE TABLE %s (%s);"; + public static final String CREATE_TABLE_STATEMENT = "CREATE TABLE %s (%s);"; - private static final String INSERT_STATEMENT = "INSERT INTO %s VALUES %s;"; + public static final String INSERT_STATEMENT = "INSERT INTO %s VALUES %s;"; - private static final String DROP_DATABASE_STATEMENT = "DROP DATABASE \"%s\" WITH (FORCE);"; + public static final String DROP_DATABASE_STATEMENT = "DROP DATABASE \"%s\" WITH (FORCE);"; private static final String USERNAME = "postgres"; @@ -35,7 +53,8 @@ public PostgreSQLHistoryDataGenerator() { Constant.readOnlyPort = 5434; } - private Connection connect(int port, boolean useSystemDatabase, String databaseName) { + public static Connection connect( + int port, boolean useSystemDatabase, String databaseName, String username, String password) { try { String url; if (useSystemDatabase) { @@ -44,7 +63,7 @@ private Connection connect(int port, boolean useSystemDatabase, String databaseN url = String.format("jdbc:postgresql://127.0.0.1:%d/%s", port, databaseName); } Class.forName("org.postgresql.Driver"); - return DriverManager.getConnection(url, USERNAME, PASSWORD); + return DriverManager.getConnection(url, username, password); } catch (SQLException | ClassNotFoundException e) { throw new RuntimeException(e); } @@ -59,7 +78,7 @@ public void writeHistoryData( List> valuesList) { Connection connection = null; try { - connection = connect(port, true, null); + connection = connect(port, true, null, USERNAME, PASSWORD); if (connection == null) { LOGGER.error("cannot connect to 127.0.0.1:{}!", port); return; @@ -92,7 +111,7 @@ public void writeHistoryData( } stmt.close(); - Connection conn = connect(port, false, databaseName); + Connection conn = connect(port, false, databaseName, USERNAME, PASSWORD); stmt = conn.createStatement(); for (Map.Entry> item : entry.getValue().entrySet()) { String tableName = item.getKey(); @@ -161,7 +180,7 @@ public void writeHistoryData( public void clearHistoryDataForGivenPort(int port) { Connection conn = null; try { - conn = connect(port, true, null); + conn = connect(port, true, null, USERNAME, PASSWORD); Statement stmt = conn.createStatement(); ResultSet databaseSet = stmt.executeQuery(QUERY_DATABASES_STATEMENT); Statement dropDatabaseStatement = conn.createStatement(); diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/redis/RedisCapacityExpansionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/redis/RedisCapacityExpansionIT.java index fdfb57e0ad..540d6d62b8 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/redis/RedisCapacityExpansionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/redis/RedisCapacityExpansionIT.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.expansion.redis; import static cn.edu.tsinghua.iginx.thrift.StorageEngineType.redis; @@ -39,6 +57,130 @@ protected void testQuerySpecialHistoryData() { SQLTestTools.executeAndCompare(session, statement, expect); } + // redis中,所有dummy数据都识别为BINARY + @Override + protected void testShowColumnsInExpansion(boolean before) { + String statement = "SHOW COLUMNS nt.wf03.*;"; + String expected = + "Columns:\n" + + "+--------------------+--------+\n" + + "| Path|DataType|\n" + + "+--------------------+--------+\n" + + "|nt.wf03.wt01.status2| BINARY|\n" + + "+--------------------+--------+\n" + + "Total line number = 1\n"; + SQLTestTools.executeAndCompare(session, statement, expected); + + statement = "SHOW COLUMNS;"; + if (before) { + expected = + "Columns:\n" + + "+--------------------------------------------------------------------------------------+--------+\n" + + "| Path|DataType|\n" + + "+--------------------------------------------------------------------------------------+--------+\n" + + "| b.b.b| BINARY|\n" + + "| ln.wf02.status| BOOLEAN|\n" + + "| ln.wf02.version| BINARY|\n" + + "| nt.wf03.wt01.status2| BINARY|\n" + + "| nt.wf04.wt01.temperature| BINARY|\n" + + "|zzzzzzzzzzzzzzzzzzzzzzzzzzzz.zzzzzzzzzzzzzzzzzzzzzzzzzzz.zzzzzzzzzzzzzzzzzzzzzzzzzzzzz| BINARY|\n" + + "+--------------------------------------------------------------------------------------+--------+\n" + + "Total line number = 6\n"; + } else { // 添加schemaPrefix为p1,dataPrefix为nt.wf03的数据源 + expected = + "Columns:\n" + + "+--------------------------------------------------------------------------------------+--------+\n" + + "| Path|DataType|\n" + + "+--------------------------------------------------------------------------------------+--------+\n" + + "| b.b.b| BINARY|\n" + + "| ln.wf02.status| BOOLEAN|\n" + + "| ln.wf02.version| BINARY|\n" + + "| nt.wf03.wt01.status2| BINARY|\n" + + "| nt.wf04.wt01.temperature| BINARY|\n" + + "| p1.nt.wf03.wt01.status2| BINARY|\n" + + "|zzzzzzzzzzzzzzzzzzzzzzzzzzzz.zzzzzzzzzzzzzzzzzzzzzzzzzzz.zzzzzzzzzzzzzzzzzzzzzzzzzzzzz| BINARY|\n" + + "+--------------------------------------------------------------------------------------+--------+\n" + + "Total line number = 7\n"; + } + SQLTestTools.executeAndCompare(session, statement, expected); + + if (before) { + statement = "SHOW COLUMNS p1.*;"; + expected = + "Columns:\n" + + "+----+--------+\n" + + "|Path|DataType|\n" + + "+----+--------+\n" + + "+----+--------+\n" + + "Empty set.\n"; + } else { // 添加schemaPrefix为p1,dataPrefix为nt.wf03的数据源 + statement = "SHOW COLUMNS p1.*;"; + expected = + "Columns:\n" + + "+-----------------------+--------+\n" + + "| Path|DataType|\n" + + "+-----------------------+--------+\n" + + "|p1.nt.wf03.wt01.status2| BINARY|\n" + + "+-----------------------+--------+\n" + + "Total line number = 1\n"; + } + SQLTestTools.executeAndCompare(session, statement, expected); + + statement = "SHOW COLUMNS *.wf03.wt01.*;"; + if (before) { + expected = + "Columns:\n" + + "+--------------------+--------+\n" + + "| Path|DataType|\n" + + "+--------------------+--------+\n" + + "|nt.wf03.wt01.status2| BINARY|\n" + + "+--------------------+--------+\n" + + "Total line number = 1\n"; + } else { // 添加schemaPrefix为p1,dataPrefix为nt.wf03的数据源 + expected = + "Columns:\n" + + "+-----------------------+--------+\n" + + "| Path|DataType|\n" + + "+-----------------------+--------+\n" + + "| nt.wf03.wt01.status2| BINARY|\n" + + "|p1.nt.wf03.wt01.status2| BINARY|\n" + + "+-----------------------+--------+\n" + + "Total line number = 2\n"; + } + SQLTestTools.executeAndCompare(session, statement, expected); + } + + // redis中,所有dummy数据都识别为BINARY + @Override + protected void testShowColumnsRemoveStorageEngine(boolean before) { + String statement = "SHOW COLUMNS p1.*, p2.*, p3.*;"; + String expected; + if (before) { + expected = + "Columns:\n" + + "+---------------------------+--------+\n" + + "| Path|DataType|\n" + + "+---------------------------+--------+\n" + + "| p1.nt.wf03.wt01.status2| BINARY|\n" + + "| p2.nt.wf03.wt01.status2| BINARY|\n" + + "| p3.nt.wf03.wt01.status2| BINARY|\n" + + "|p3.nt.wf04.wt01.temperature| BINARY|\n" + + "+---------------------------+--------+\n" + + "Total line number = 4\n"; + } else { // 移除schemaPrefix为p2及p3,dataPrefix为nt.wf03的数据源 + expected = + "Columns:\n" + + "+---------------------------+--------+\n" + + "| Path|DataType|\n" + + "+---------------------------+--------+\n" + + "| p1.nt.wf03.wt01.status2| BINARY|\n" + + "|p3.nt.wf04.wt01.temperature| BINARY|\n" + + "+---------------------------+--------+\n" + + "Total line number = 2\n"; + } + SQLTestTools.executeAndCompare(session, statement, expected); + } + // redis中,所有dummy数据都识别为BINARY @Override public void testShowColumns() { @@ -78,4 +220,11 @@ public void testShowColumns() { + "Total line number = 2\n"; SQLTestTools.executeAndCompare(session, statement, expected); } + + // no param is allowed to be updated + @Override + protected void updateParams(int port) {} + + @Override + protected void restoreParams(int port) {} } diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/redis/RedisHistoryDataGenerator.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/redis/RedisHistoryDataGenerator.java index 3f01f7c569..b80dc2fdd9 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/redis/RedisHistoryDataGenerator.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/redis/RedisHistoryDataGenerator.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.expansion.redis; import static cn.edu.tsinghua.iginx.integration.expansion.constant.Constant.readOnlyPort; @@ -65,7 +83,7 @@ public void writeSpecialHistoryData() { @Override public void clearHistoryDataForGivenPort(int port) { Jedis jedis = new Jedis(LOCAL_IP, port); - jedis.flushDB(); + jedis.flushAll(); jedis.close(); LOGGER.info("clear data on 127.0.0.1:{} success!", port); } diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/utils/SQLTestTools.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/utils/SQLTestTools.java index 6085f6798c..c421de3683 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/utils/SQLTestTools.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/utils/SQLTestTools.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.expansion.utils; import static org.junit.Assert.*; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/rest/RestAnnotationIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/rest/RestAnnotationIT.java index 54552baa2f..d6609829de 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/rest/RestAnnotationIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/rest/RestAnnotationIT.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.func.rest; import static org.junit.Assert.assertEquals; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/rest/RestIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/rest/RestIT.java index 62211e88fd..5ad12b0435 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/rest/RestIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/rest/RestIT.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.func.rest; import static cn.edu.tsinghua.iginx.integration.controller.Controller.clearAllData; @@ -238,7 +256,7 @@ public void testQueryWrongTime() { public void testQueryAvg() { String json = "testQueryAvg.json"; String result = - "{\"queries\":[{\"sample_size\": 3,\"results\": [{ \"name\": \"archive.file.tracked\",\"group_by\": [{\"name\": \"type\",\"type\": \"number\"}], \"tags\": {\"dc\": [\"DC1\"],\"host\": [\"server1\"]}, \"values\": [[1359788298001,13.2],[1359788398001,123.3],[1359788408001,23.1]]}]}]}"; + "{\"queries\":[{\"sample_size\": 9,\"results\": [{ \"names\": [\"window_start\", \"window_end\", \"archive.file.tracked\"],\"group_by\": [{\"name\": \"type\",\"type\": \"number\"}], \"tags\": {\"dc\": [\"DC1\"],\"host\": [\"server1\"]}, \"values\": [[1359788298001,1359788298001],[1359788398001,1359788398001],[1359788408001,1359788408001],[1359788298001,1359788300000],[1359788398001,1359788400000],[1359788408001,1359788410000],[1359788298001,13.2],[1359788398001,123.3],[1359788408001,23.1]]}]}]}"; executeAndCompare(json, result); } @@ -246,7 +264,7 @@ public void testQueryAvg() { public void testQueryCount() { String json = "testQueryCount.json"; String result = - "{\"queries\":[{\"sample_size\": 1,\"results\": [{ \"name\": \"archive.file.tracked\",\"group_by\": [{\"name\": \"type\",\"type\": \"number\"}], \"tags\": {\"dc\": [\"DC1\"],\"host\": [\"server1\"]}, \"values\": [[1359763200001,3]]}]}]}"; + "{\"queries\":[{\"sample_size\": 3,\"results\": [{ \"names\": [\"window_start\", \"window_end\", \"archive.file.tracked\"],\"group_by\": [{\"name\": \"type\",\"type\": \"number\"}], \"tags\": {\"dc\": [\"DC1\"],\"host\": [\"server1\"]}, \"values\": [[1359763200001,1359763200001],[1359763200001,1359849600000],[1359763200001,3]]}]}]}"; executeAndCompare(json, result); } @@ -254,7 +272,7 @@ public void testQueryCount() { public void testQueryFirst() { String json = "testQueryFirst.json"; String result = - "{\"queries\":[{\"sample_size\": 1,\"results\": [{ \"name\": \"archive.file.tracked\",\"group_by\": [{\"name\": \"type\",\"type\": \"number\"}], \"tags\": {\"dc\": [\"DC1\"],\"host\": [\"server1\"]}, \"values\": [[1359763200001,13.2]]}]}]}"; + "{\"queries\":[{\"sample_size\": 3,\"results\": [{ \"names\": [\"window_start\", \"window_end\", \"archive.file.tracked\"],\"group_by\": [{\"name\": \"type\",\"type\": \"number\"}], \"tags\": {\"dc\": [\"DC1\"],\"host\": [\"server1\"]}, \"values\": [[1359763200001,1359763200001],[1359763200001,1359936000000],[1359763200001,13.2]]}]}]}"; executeAndCompare(json, result); } @@ -262,7 +280,7 @@ public void testQueryFirst() { public void testQueryLast() { String json = "testQueryLast.json"; String result = - "{\"queries\":[{\"sample_size\": 1,\"results\": [{ \"name\": \"archive.file.tracked\",\"group_by\": [{\"name\": \"type\",\"type\": \"number\"}], \"tags\": {\"dc\": [\"DC1\"],\"host\": [\"server1\"]}, \"values\": [[1359763200001,23.1]]}]}]}"; + "{\"queries\":[{\"sample_size\": 3,\"results\": [{ \"names\": [\"window_start\", \"window_end\", \"archive.file.tracked\"],\"group_by\": [{\"name\": \"type\",\"type\": \"number\"}], \"tags\": {\"dc\": [\"DC1\"],\"host\": [\"server1\"]}, \"values\": [[1359763200001,1359763200001],[1359763200001,1359936000000],[1359763200001,23.1]]}]}]}"; executeAndCompare(json, result); } @@ -270,7 +288,7 @@ public void testQueryLast() { public void testQueryMax() { String json = "testQueryMax.json"; String result = - "{\"queries\":[{\"sample_size\": 1,\"results\": [{ \"name\": \"archive.file.tracked\",\"group_by\": [{\"name\": \"type\",\"type\": \"number\"}], \"tags\": {\"dc\": [\"DC1\"],\"host\": [\"server1\"]}, \"values\": [[1359763200001,123.3]]}]}]}"; + "{\"queries\":[{\"sample_size\": 3,\"results\": [{ \"names\": [\"window_start\", \"window_end\", \"archive.file.tracked\"],\"group_by\": [{\"name\": \"type\",\"type\": \"number\"}], \"tags\": {\"dc\": [\"DC1\"],\"host\": [\"server1\"]}, \"values\": [[1359763200001,1359763200001],[1359763200001,1359936000000],[1359763200001,123.3]]}]}]}"; executeAndCompare(json, result); } @@ -278,7 +296,7 @@ public void testQueryMax() { public void testQueryMin() { String json = "testQueryMin.json"; String result = - "{\"queries\":[{\"sample_size\": 1,\"results\": [{ \"name\": \"archive.file.tracked\",\"group_by\": [{\"name\": \"type\",\"type\": \"number\"}], \"tags\": {\"dc\": [\"DC1\"],\"host\": [\"server1\"]}, \"values\": [[1359763200001,13.2]]}]}]}"; + "{\"queries\":[{\"sample_size\": 3,\"results\": [{ \"names\": [\"window_start\", \"window_end\", \"archive.file.tracked\"],\"group_by\": [{\"name\": \"type\",\"type\": \"number\"}], \"tags\": {\"dc\": [\"DC1\"],\"host\": [\"server1\"]}, \"values\": [[1359763200001,1359763200001],[1359763200001,1359936000000],[1359763200001,13.2]]}]}]}"; executeAndCompare(json, result); } @@ -286,7 +304,7 @@ public void testQueryMin() { public void testQuerySum() { String json = "testQuerySum.json"; String result = - "{\"queries\":[{\"sample_size\": 1,\"results\": [{ \"name\": \"archive.file.tracked\",\"group_by\": [{\"name\": \"type\",\"type\": \"number\"}], \"tags\": {\"dc\": [\"DC1\"],\"host\": [\"server1\"]}, \"values\": [[1359763200001,159.6]]}]}]}"; + "{\"queries\":[{\"sample_size\": 3,\"results\": [{ \"names\": [\"window_start\", \"window_end\", \"archive.file.tracked\"],\"group_by\": [{\"name\": \"type\",\"type\": \"number\"}], \"tags\": {\"dc\": [\"DC1\"],\"host\": [\"server1\"]}, \"values\": [[1359763200001,1359763200001],[1359763200001,1359936000000],[1359763200001,159.6]]}]}]}"; executeAndCompare(json, result); } diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/BaseSessionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/BaseSessionIT.java index 2e28b2bc64..337bf2e181 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/BaseSessionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/BaseSessionIT.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.func.session; import static org.junit.Assert.fail; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/InsertAPIType.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/InsertAPIType.java index a257c77507..207ba90857 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/InsertAPIType.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/InsertAPIType.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.func.session; public enum InsertAPIType { diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/NewSessionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/NewSessionIT.java index 45aaa44c2e..37ad705d16 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/NewSessionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/NewSessionIT.java @@ -1,5 +1,25 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.func.session; +import static cn.edu.tsinghua.iginx.engine.shared.Constants.WINDOW_END_COL; +import static cn.edu.tsinghua.iginx.engine.shared.Constants.WINDOW_START_COL; import static cn.edu.tsinghua.iginx.integration.controller.Controller.SUPPORT_KEY; import static cn.edu.tsinghua.iginx.integration.controller.Controller.clearAllData; import static cn.edu.tsinghua.iginx.integration.func.session.InsertAPIType.*; @@ -374,8 +394,8 @@ public void testCancelSession() { List existsSessionIDs = conn.executeSql("show sessionid;").getSessionIDs(); - if (!new HashSet<>(existsSessionIDs).equals(new HashSet<>(sessionIDs))) { - LOGGER.error("server session_id_list does not equal to active_session_id_list."); + if (!existsSessionIDs.containsAll(sessionIDs)) { + LOGGER.error("server session_id_list doesn't contain all session IDs."); fail(); } @@ -399,6 +419,7 @@ public void testCancelSession() { public void testCancelClient() { File clientDir = new File("../client/target/"); File[] matchingFiles = clientDir.listFiles((dir, name) -> name.startsWith("iginx-client-")); + matchingFiles = Arrays.stream(matchingFiles).filter(File::isDirectory).toArray(File[]::new); String version = matchingFiles[0].getName(); version = version.contains(".jar") ? version.substring(0, version.lastIndexOf(".")) : version; // use .sh on unix & .bat on windows(absolute path) @@ -420,7 +441,9 @@ public void testCancelClient() { if (ShellRunner.isOnWin()) { pb.command(clientWinPath); } else { - Runtime.getRuntime().exec(new String[] {"chmod", "+x", clientUnixPath}); + Process before = Runtime.getRuntime().exec(new String[] {"chmod", "+x", clientUnixPath}); + before.waitFor(); + LOGGER.info("before start a client, exit value: {}", before.exitValue()); pb.command("bash", "-c", clientUnixPath); } Process p = pb.start(); @@ -635,6 +658,9 @@ public void testAggregateQuery() { @Test public void testDownsampleQuery() { List paths = Arrays.asList("us.d1.s2", "us.d1.s3", "us.d1.s4", "us.d1.s5"); + List resPaths = + Arrays.asList( + WINDOW_START_COL, WINDOW_END_COL, "us.d1.s2", "us.d1.s3", "us.d1.s4", "us.d1.s5"); List types = Arrays.asList(DataType.INTEGER, DataType.LONG, DataType.FLOAT, DataType.DOUBLE); List keys = Arrays.asList(0L, 4000L, 8000L, 12000L); @@ -655,72 +681,114 @@ public void testDownsampleQuery() { new TestDataSection( keys, types, - paths.stream().map(s -> "max(" + s + ")").collect(Collectors.toList()), + resPaths.stream() + .map( + s -> + s.equals(WINDOW_START_COL) || s.equals(WINDOW_END_COL) + ? s + : "max(" + s + ")") + .collect(Collectors.toList()), Arrays.asList( - Arrays.asList(3999, 3999L, 3999.1f, 3999.2d), - Arrays.asList(7999, 7999L, 7999.1f, 7999.2d), - Arrays.asList(11999, 11999L, 11999.1f, 11999.2d), - Arrays.asList(15999, 15999L, 15999.1f, 15999.2d)), + Arrays.asList(0L, 3999L, 3999, 3999L, 3999.1f, 3999.2d), + Arrays.asList(4000L, 7999L, 7999, 7999L, 7999.1f, 7999.2d), + Arrays.asList(8000L, 11999L, 11999, 11999L, 11999.1f, 11999.2d), + Arrays.asList(12000L, 15999L, 15999, 15999L, 15999.1f, 15999.2d)), baseDataSection.getTagsList()), new TestDataSection( keys, types, - paths.stream().map(s -> "min(" + s + ")").collect(Collectors.toList()), + resPaths.stream() + .map( + s -> + s.equals(WINDOW_START_COL) || s.equals(WINDOW_END_COL) + ? s + : "min(" + s + ")") + .collect(Collectors.toList()), Arrays.asList( - Arrays.asList(0, 0L, 0.1f, 0.2d), - Arrays.asList(4000, 4000L, 4000.1f, 4000.2d), - Arrays.asList(8000, 8000L, 8000.1f, 8000.2d), - Arrays.asList(12000, 12000L, 12000.1f, 12000.2d)), + Arrays.asList(0L, 3999L, 0, 0L, 0.1f, 0.2d), + Arrays.asList(4000L, 7999L, 4000, 4000L, 4000.1f, 4000.2d), + Arrays.asList(8000L, 11999L, 8000, 8000L, 8000.1f, 8000.2d), + Arrays.asList(12000L, 15999L, 12000, 12000L, 12000.1f, 12000.2d)), baseDataSection.getTagsList()), new TestDataSection( keys, types, - paths.stream().map(s -> "sum(" + s + ")").collect(Collectors.toList()), + resPaths.stream() + .map( + s -> + s.equals(WINDOW_START_COL) || s.equals(WINDOW_END_COL) + ? s + : "sum(" + s + ")") + .collect(Collectors.toList()), Arrays.asList( - Arrays.asList(7998000L, 7998000L, 7998400.0d, 7998800.0d), - Arrays.asList(23998000L, 23998000L, 23998400.0d, 23998800.0d), - Arrays.asList(39998000L, 39998000L, 39998400.0d, 39998800.0d), - Arrays.asList(55998000L, 55998000L, 55998400.0d, 55998800.0d)), + Arrays.asList(0L, 3999L, 7998000L, 7998000L, 7998400.0d, 7998800.0d), + Arrays.asList(4000L, 7999L, 23998000L, 23998000L, 23998400.0d, 23998800.0d), + Arrays.asList(8000L, 11999L, 39998000L, 39998000L, 39998400.0d, 39998800.0d), + Arrays.asList(12000L, 15999L, 55998000L, 55998000L, 55998400.0d, 55998800.0d)), baseDataSection.getTagsList()), new TestDataSection( keys, types, - paths.stream().map(s -> "count(" + s + ")").collect(Collectors.toList()), + resPaths.stream() + .map( + s -> + s.equals(WINDOW_START_COL) || s.equals(WINDOW_END_COL) + ? s + : "count(" + s + ")") + .collect(Collectors.toList()), Arrays.asList( - Arrays.asList(4000L, 4000L, 4000L, 4000L), - Arrays.asList(4000L, 4000L, 4000L, 4000L), - Arrays.asList(4000L, 4000L, 4000L, 4000L), - Arrays.asList(4000L, 4000L, 4000L, 4000L)), + Arrays.asList(0L, 3999L, 4000L, 4000L, 4000L, 4000L), + Arrays.asList(4000L, 7999L, 4000L, 4000L, 4000L, 4000L), + Arrays.asList(8000L, 11999L, 4000L, 4000L, 4000L, 4000L), + Arrays.asList(12000L, 15999L, 4000L, 4000L, 4000L, 4000L)), baseDataSection.getTagsList()), new TestDataSection( keys, types, - paths.stream().map(s -> "avg(" + s + ")").collect(Collectors.toList()), + resPaths.stream() + .map( + s -> + s.equals(WINDOW_START_COL) || s.equals(WINDOW_END_COL) + ? s + : "avg(" + s + ")") + .collect(Collectors.toList()), Arrays.asList( - Arrays.asList(1999.5d, 1999.5d, 1999.6d, 1999.7d), - Arrays.asList(5999.5d, 5999.5d, 5999.6d, 5999.7d), - Arrays.asList(9999.5d, 9999.5d, 9999.6d, 9999.7d), - Arrays.asList(13999.5d, 13999.5d, 13999.6d, 13999.7d)), + Arrays.asList(0L, 3999L, 1999.5d, 1999.5d, 1999.6d, 1999.7d), + Arrays.asList(4000L, 7999L, 5999.5d, 5999.5d, 5999.6d, 5999.7d), + Arrays.asList(8000L, 11999L, 9999.5d, 9999.5d, 9999.6d, 9999.7d), + Arrays.asList(12000L, 15999L, 13999.5d, 13999.5d, 13999.6d, 13999.7d)), baseDataSection.getTagsList()), new TestDataSection( keys, types, - paths.stream().map(s -> "first_value(" + s + ")").collect(Collectors.toList()), + resPaths.stream() + .map( + s -> + s.equals(WINDOW_START_COL) || s.equals(WINDOW_END_COL) + ? s + : "first_value(" + s + ")") + .collect(Collectors.toList()), Arrays.asList( - Arrays.asList(0, 0L, 0.1f, 0.2d), - Arrays.asList(4000, 4000L, 4000.1f, 4000.2d), - Arrays.asList(8000, 8000L, 8000.1f, 8000.2d), - Arrays.asList(12000, 12000L, 12000.1f, 12000.2d)), + Arrays.asList(0L, 3999L, 0, 0L, 0.1f, 0.2d), + Arrays.asList(4000L, 7999L, 4000, 4000L, 4000.1f, 4000.2d), + Arrays.asList(8000L, 11999L, 8000, 8000L, 8000.1f, 8000.2d), + Arrays.asList(12000L, 15999L, 12000, 12000L, 12000.1f, 12000.2d)), baseDataSection.getTagsList()), new TestDataSection( keys, types, - paths.stream().map(s -> "last_value(" + s + ")").collect(Collectors.toList()), + resPaths.stream() + .map( + s -> + s.equals(WINDOW_START_COL) || s.equals(WINDOW_END_COL) + ? s + : "last_value(" + s + ")") + .collect(Collectors.toList()), Arrays.asList( - Arrays.asList(3999, 3999L, 3999.1f, 3999.2d), - Arrays.asList(7999, 7999L, 7999.1f, 7999.2d), - Arrays.asList(11999, 11999L, 11999.1f, 11999.2d), - Arrays.asList(15999, 15999L, 15999.1f, 15999.2d)), + Arrays.asList(0L, 3999L, 3999, 3999L, 3999.1f, 3999.2d), + Arrays.asList(4000L, 7999L, 7999, 7999L, 7999.1f, 7999.2d), + Arrays.asList(8000L, 11999L, 11999, 11999L, 11999.1f, 11999.2d), + Arrays.asList(12000L, 15999L, 15999, 15999L, 15999.1f, 15999.2d)), baseDataSection.getTagsList())); for (int i = 0; i < aggregateTypes.size(); i++) { @@ -736,6 +804,154 @@ public void testDownsampleQuery() { } } + @Test + public void testDownsampleQueryNoInterval() { + List paths = Arrays.asList("us.d1.s2", "us.d1.s3", "us.d1.s4", "us.d1.s5"); + List resPaths = + Arrays.asList( + WINDOW_START_COL, WINDOW_END_COL, "us.d1.s2", "us.d1.s3", "us.d1.s4", "us.d1.s5"); + List types = + Arrays.asList(DataType.INTEGER, DataType.LONG, DataType.FLOAT, DataType.DOUBLE); + List keys = Arrays.asList(0L, 4000L, 8000L, 12000L); + + List aggregateTypes = + Arrays.asList( + AggregateType.MAX, + AggregateType.MIN, + AggregateType.SUM, + AggregateType.COUNT, + AggregateType.AVG, + AggregateType.FIRST_VALUE, + AggregateType.LAST_VALUE); + long precision = 4000; + + List expectedResults = + Arrays.asList( + new TestDataSection( + keys, + types, + resPaths.stream() + .map( + s -> + s.equals(WINDOW_START_COL) || s.equals(WINDOW_END_COL) + ? s + : "max(" + s + ")") + .collect(Collectors.toList()), + Arrays.asList( + Arrays.asList(0L, 3999L, 3999, 3999L, 3999.1f, 3999.2d), + Arrays.asList(4000L, 7999L, 7999, 7999L, 7999.1f, 7999.2d), + Arrays.asList(8000L, 11999L, 11999, 11999L, 11999.1f, 11999.2d), + Arrays.asList(12000L, 15999L, 15999, 15999L, 15999.1f, 15999.2d)), + baseDataSection.getTagsList()), + new TestDataSection( + keys, + types, + resPaths.stream() + .map( + s -> + s.equals(WINDOW_START_COL) || s.equals(WINDOW_END_COL) + ? s + : "min(" + s + ")") + .collect(Collectors.toList()), + Arrays.asList( + Arrays.asList(0L, 3999L, 0, 0L, 0.1f, 0.2d), + Arrays.asList(4000L, 7999L, 4000, 4000L, 4000.1f, 4000.2d), + Arrays.asList(8000L, 11999L, 8000, 8000L, 8000.1f, 8000.2d), + Arrays.asList(12000L, 15999L, 12000, 12000L, 12000.1f, 12000.2d)), + baseDataSection.getTagsList()), + new TestDataSection( + keys, + types, + resPaths.stream() + .map( + s -> + s.equals(WINDOW_START_COL) || s.equals(WINDOW_END_COL) + ? s + : "sum(" + s + ")") + .collect(Collectors.toList()), + Arrays.asList( + Arrays.asList(0L, 3999L, 7998000L, 7998000L, 7998400.0d, 7998800.0d), + Arrays.asList(4000L, 7999L, 23998000L, 23998000L, 23998400.0d, 23998800.0d), + Arrays.asList(8000L, 11999L, 39998000L, 39998000L, 39998400.0d, 39998800.0d), + Arrays.asList(12000L, 15999L, 55998000L, 55998000L, 55998400.0d, 55998800.0d)), + baseDataSection.getTagsList()), + new TestDataSection( + keys, + types, + resPaths.stream() + .map( + s -> + s.equals(WINDOW_START_COL) || s.equals(WINDOW_END_COL) + ? s + : "count(" + s + ")") + .collect(Collectors.toList()), + Arrays.asList( + Arrays.asList(0L, 3999L, 4000L, 4000L, 4000L, 4000L), + Arrays.asList(4000L, 7999L, 4000L, 4000L, 4000L, 4000L), + Arrays.asList(8000L, 11999L, 4000L, 4000L, 4000L, 4000L), + Arrays.asList(12000L, 15999L, 4000L, 4000L, 4000L, 4000L)), + baseDataSection.getTagsList()), + new TestDataSection( + keys, + types, + resPaths.stream() + .map( + s -> + s.equals(WINDOW_START_COL) || s.equals(WINDOW_END_COL) + ? s + : "avg(" + s + ")") + .collect(Collectors.toList()), + Arrays.asList( + Arrays.asList(0L, 3999L, 1999.5d, 1999.5d, 1999.6d, 1999.7d), + Arrays.asList(4000L, 7999L, 5999.5d, 5999.5d, 5999.6d, 5999.7d), + Arrays.asList(8000L, 11999L, 9999.5d, 9999.5d, 9999.6d, 9999.7d), + Arrays.asList(12000L, 15999L, 13999.5d, 13999.5d, 13999.6d, 13999.7d)), + baseDataSection.getTagsList()), + new TestDataSection( + keys, + types, + resPaths.stream() + .map( + s -> + s.equals(WINDOW_START_COL) || s.equals(WINDOW_END_COL) + ? s + : "first_value(" + s + ")") + .collect(Collectors.toList()), + Arrays.asList( + Arrays.asList(0L, 3999L, 0, 0L, 0.1f, 0.2d), + Arrays.asList(4000L, 7999L, 4000, 4000L, 4000.1f, 4000.2d), + Arrays.asList(8000L, 11999L, 8000, 8000L, 8000.1f, 8000.2d), + Arrays.asList(12000L, 15999L, 12000, 12000L, 12000.1f, 12000.2d)), + baseDataSection.getTagsList()), + new TestDataSection( + keys, + types, + resPaths.stream() + .map( + s -> + s.equals(WINDOW_START_COL) || s.equals(WINDOW_END_COL) + ? s + : "last_value(" + s + ")") + .collect(Collectors.toList()), + Arrays.asList( + Arrays.asList(0L, 3999L, 3999, 3999L, 3999.1f, 3999.2d), + Arrays.asList(4000L, 7999L, 7999, 7999L, 7999.1f, 7999.2d), + Arrays.asList(8000L, 11999L, 11999, 11999L, 11999.1f, 11999.2d), + Arrays.asList(12000L, 15999L, 15999, 15999L, 15999.1f, 15999.2d)), + baseDataSection.getTagsList())); + + for (int i = 0; i < aggregateTypes.size(); i++) { + AggregateType type = aggregateTypes.get(i); + try { + SessionQueryDataSet dataSet = conn.downsampleQuery(paths, type, precision); + compare(expectedResults.get(i), dataSet); + } catch (SessionException e) { + LOGGER.error("execute downsample query failed, AggType={}, Precision={}", type, precision); + fail(); + } + } + } + @Test public void testQueryAfterDelete() { if (!isAbleToDelete) return; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/PySessionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/PySessionIT.java index a38caa3f97..1dcea36696 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/PySessionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/PySessionIT.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.func.session; import static cn.edu.tsinghua.iginx.integration.controller.Controller.clearAllData; @@ -46,7 +64,7 @@ public class PySessionIT { protected static String defaultTestPass = "root"; private static final Config config = ConfigDescriptor.getInstance().getConfig(); - private static String pythonCMD = config.getPythonCMD(); + private static final String pythonCMD = config.getPythonCMD(); private static boolean isAbleToDelete = true; private static PythonInterpreter interpreter; @@ -164,9 +182,37 @@ public void testDownSampleQuery() { } // 检查Python脚本的输出是否符合预期 String expected = - " key count(test.a.a) count(test.a.b) count(test.b.b) count(test.c.c)\n" - + "0 0 1 1 1 1\n" - + "1 3 1 1 1 1\n"; + " key window_start window_end count(test.a.a) count(test.a.b) \\\n" + + "0 0 0 2 1 1 \n" + + "1 3 3 5 1 1 \n" + + "\n" + + " count(test.b.b) count(test.c.c) \n" + + "0 1 1 \n" + + "1 1 1 \n"; + assertEquals(expected, result); + } + + @Test + public void testDownSampleQueryNoInterval() { + String result = ""; + try { + // 设置Python脚本路径 + logger.info("Test downsample query without time interval"); + result = runPythonScript("downsampleQueryNoInterval"); + logger.info(result); + } catch (IOException | InterruptedException e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + // 检查Python脚本的输出是否符合预期 + String expected = + " key window_start window_end count(test.a.a) count(test.a.b) \\\n" + + "0 0 0 2 1 1 \n" + + "1 3 3 5 1 1 \n" + + "\n" + + " count(test.b.b) count(test.c.c) \n" + + "0 1 1 \n" + + "1 1 1 \n"; assertEquals(expected, result); } @@ -347,6 +393,34 @@ public void testInsert() { assertEquals(expected, result); } + @Test + public void testInsertDF() { + String result = ""; + try { + logger.info("insert dataframe"); + result = runPythonScript("insertDF"); + logger.info(result); + } catch (IOException | InterruptedException e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + // 检查Python脚本的输出是否符合预期 + String expected = + " key dftestdata.value1 dftestdata.value2 dftestdata.value3 dftestdata.value4\n" + + " 10 b'A' 1.1 b'B' 2.2\n" + + " 11 b'A' 1.1 b'B' 2.2\n" + + " 12 b'A' 1.1 b'B' 2.2\n" + + " 13 b'A' 1.1 b'B' 2.2\n" + + " 14 b'A' 1.1 b'B' 2.2\n" + + " 15 b'A' 1.1 b'B' 2.2\n" + + " 16 b'A' 1.1 b'B' 2.2\n" + + " 17 b'A' 1.1 b'B' 2.2\n" + + " 18 b'A' 1.1 b'B' 2.2\n" + + " 19 b'A' 1.1 b'B' 2.2\n"; + + assertEquals(expected, result); + } + @Test public void testDeleteRow() { if (!isAbleToDelete) { diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/SQLCompareIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/SQLCompareIT.java index 32bddea191..98a231f1f8 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/SQLCompareIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/SQLCompareIT.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.func.session; import static org.junit.Assert.assertEquals; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/SessionConcurrencyIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/SessionConcurrencyIT.java index d88f5fad9b..6e66593d39 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/SessionConcurrencyIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/SessionConcurrencyIT.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.func.session; import static org.junit.Assert.*; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/SessionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/SessionIT.java index a83297ee1e..8e8f302ef7 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/SessionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/SessionIT.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.integration.func.session; @@ -522,7 +521,8 @@ public void sessionTest() throws SessionException, InterruptedException { long dsStartKey = delDsAvgDataSet.getKeys()[i]; assertEquals(START_KEY + i * PRECISION, dsStartKey); List dsResult = delDsAvgDataSet.getValues().get(i); - for (int j = 0; j < delDsResPaths.size(); j++) { + // j starts from 2 to skip WINDOW_START & WINDOW_END + for (int j = 2; j < delDsResPaths.size(); j++) { long dsEndKey = Math.min((START_KEY + (i + 1) * PRECISION - 1), END_KEY); double delDsAvg = (dsStartKey + dsEndKey) / 2.0; int pathNum = getPathNum(delDsResPaths.get(j)); @@ -636,7 +636,8 @@ public void sessionTest() throws SessionException, InterruptedException { long dsKey = dsDelDataInColSet.getKeys()[i]; assertEquals(START_KEY + i * PRECISION, dsKey); List dsResult = dsDelDataInColSet.getValues().get(i); - for (int j = 0; j < dsDelDataResPaths.size(); j++) { + // j starts from 2 to skip WINDOW_START & WINDOW_END + for (int j = 2; j < dsDelDataResPaths.size(); j++) { long maxNum = Math.min((START_KEY + (i + 1) * PRECISION - 1), END_KEY); double avg = (dsKey + maxNum) / 2.0; int pathNum = getPathNum(dsDelDataResPaths.get(j)); diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/SessionPoolIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/SessionPoolIT.java index 1fec5af133..5c8b777752 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/SessionPoolIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/SessionPoolIT.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.func.session; import cn.edu.tsinghua.iginx.integration.tool.MultiConnection; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/SessionV2IT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/SessionV2IT.java index 2a3771bf83..e206003c78 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/SessionV2IT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/SessionV2IT.java @@ -1,5 +1,25 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.func.session; +import static cn.edu.tsinghua.iginx.engine.shared.Constants.WINDOW_END_COL; +import static cn.edu.tsinghua.iginx.engine.shared.Constants.WINDOW_START_COL; import static cn.edu.tsinghua.iginx.integration.controller.Controller.SUPPORT_KEY; import static cn.edu.tsinghua.iginx.thrift.StorageEngineType.influxdb; import static org.junit.Assert.*; @@ -616,6 +636,22 @@ public void testDownsampleQuery() { .startKey(startKey) .endKey(endKey + (endKey - startKey)) .build(); + executeDownsampleQuery(query); + } + + @Test + public void testDownsampleQueryNoInterval() { + Query query = + DownsampleQuery.builder() + .addMeasurement("test.session.v2.long") + .addMeasurement("test.session.v2.double") + .aggregate(AggregateType.SUM) + .precision((endKey - startKey) / 10) + .build(); + executeDownsampleQuery(query); + } + + private void executeDownsampleQuery(Query query) { IginXTable table = queryClient.query(query); if (!needCompareResult) { return; @@ -624,7 +660,7 @@ public void testDownsampleQuery() { IginXHeader header = table.getHeader(); assertTrue(header.hasTimestamp()); List columns = header.getColumns(); - assertEquals(2, columns.size()); + assertEquals(4, columns.size()); for (IginXColumn column : columns) { switch (column.getName()) { case "sum(test.session.v2.long)": @@ -633,6 +669,9 @@ public void testDownsampleQuery() { case "sum(test.session.v2.double)": assertEquals(DataType.DOUBLE, column.getDataType()); break; + case WINDOW_START_COL: + case WINDOW_END_COL: + break; default: fail(); } diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/TestDataSection.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/TestDataSection.java index fac441696e..e7ff77187d 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/TestDataSection.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/TestDataSection.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.func.session; import cn.edu.tsinghua.iginx.thrift.DataType; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/SQLSessionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/SQLSessionIT.java index 1c1f7ec600..fbf0ef62a6 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/SQLSessionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/SQLSessionIT.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.func.sql; import static cn.edu.tsinghua.iginx.integration.controller.Controller.SUPPORT_KEY; @@ -7,22 +25,14 @@ import cn.edu.tsinghua.iginx.exception.SessionException; import cn.edu.tsinghua.iginx.integration.controller.Controller; import cn.edu.tsinghua.iginx.integration.func.session.InsertAPIType; -import cn.edu.tsinghua.iginx.integration.tool.ConfLoader; -import cn.edu.tsinghua.iginx.integration.tool.DBConf; +import cn.edu.tsinghua.iginx.integration.tool.*; import cn.edu.tsinghua.iginx.integration.tool.DBConf.DBConfType; -import cn.edu.tsinghua.iginx.integration.tool.MultiConnection; -import cn.edu.tsinghua.iginx.integration.tool.SQLExecutor; import cn.edu.tsinghua.iginx.pool.IginxInfo; import cn.edu.tsinghua.iginx.pool.SessionPool; import cn.edu.tsinghua.iginx.session.Session; import cn.edu.tsinghua.iginx.thrift.DataType; import cn.edu.tsinghua.iginx.utils.Pair; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; import org.apache.commons.lang3.RandomStringUtils; import org.junit.*; @@ -941,34 +951,34 @@ public void testDistinct() { executor.executeAndCompare(statement, expected); statement = - "SELECT COUNT(a), AVG(a), SUM(a), MIN(a), MAX(a) FROM test OVER (RANGE 2 IN (0, 10]);"; - expected = - "ResultSets:\n" - + "+---+-------------+-----------+-----------+-----------+-----------+\n" - + "|key|count(test.a)|avg(test.a)|sum(test.a)|min(test.a)|max(test.a)|\n" - + "+---+-------------+-----------+-----------+-----------+-----------+\n" - + "| 1| 2| 1.5| 3| 1| 2|\n" - + "| 3| 2| 2.5| 5| 2| 3|\n" - + "| 5| 2| 3.0| 6| 3| 3|\n" - + "| 7| 2| 4.0| 8| 4| 4|\n" - + "| 9| 2| 4.0| 8| 4| 4|\n" - + "+---+-------------+-----------+-----------+-----------+-----------+\n" + "SELECT COUNT(a), AVG(a), SUM(a), MIN(a), MAX(a) FROM test OVER WINDOW (size 2 IN (0, 10]);"; + expected = + "ResultSets:\n" + + "+---+------------+----------+-------------+-----------+-----------+-----------+-----------+\n" + + "|key|window_start|window_end|count(test.a)|avg(test.a)|sum(test.a)|min(test.a)|max(test.a)|\n" + + "+---+------------+----------+-------------+-----------+-----------+-----------+-----------+\n" + + "| 1| 1| 2| 2| 1.5| 3| 1| 2|\n" + + "| 3| 3| 4| 2| 2.5| 5| 2| 3|\n" + + "| 5| 5| 6| 2| 3.0| 6| 3| 3|\n" + + "| 7| 7| 8| 2| 4.0| 8| 4| 4|\n" + + "| 9| 9| 10| 2| 4.0| 8| 4| 4|\n" + + "+---+------------+----------+-------------+-----------+-----------+-----------+-----------+\n" + "Total line number = 5\n"; executor.executeAndCompare(statement, expected); statement = - "SELECT COUNT(DISTINCT a), AVG(DISTINCT a), SUM(DISTINCT a), MIN(DISTINCT a), MAX(DISTINCT a) FROM test OVER (RANGE 2 IN (0, 10]);"; - expected = - "ResultSets:\n" - + "+---+----------------------+--------------------+--------------------+--------------------+--------------------+\n" - + "|key|count(distinct test.a)|avg(distinct test.a)|sum(distinct test.a)|min(distinct test.a)|max(distinct test.a)|\n" - + "+---+----------------------+--------------------+--------------------+--------------------+--------------------+\n" - + "| 1| 2| 1.5| 3| 1| 2|\n" - + "| 3| 2| 2.5| 5| 2| 3|\n" - + "| 5| 1| 3.0| 3| 3| 3|\n" - + "| 7| 1| 4.0| 4| 4| 4|\n" - + "| 9| 1| 4.0| 4| 4| 4|\n" - + "+---+----------------------+--------------------+--------------------+--------------------+--------------------+\n" + "SELECT COUNT(DISTINCT a), AVG(DISTINCT a), SUM(DISTINCT a), MIN(DISTINCT a), MAX(DISTINCT a) FROM test OVER WINDOW (size 2 IN (0, 10]);"; + expected = + "ResultSets:\n" + + "+---+------------+----------+----------------------+--------------------+--------------------+--------------------+--------------------+\n" + + "|key|window_start|window_end|count(distinct test.a)|avg(distinct test.a)|sum(distinct test.a)|min(distinct test.a)|max(distinct test.a)|\n" + + "+---+------------+----------+----------------------+--------------------+--------------------+--------------------+--------------------+\n" + + "| 1| 1| 2| 2| 1.5| 3| 1| 2|\n" + + "| 3| 3| 4| 2| 2.5| 5| 2| 3|\n" + + "| 5| 5| 6| 1| 3.0| 3| 3| 3|\n" + + "| 7| 7| 8| 1| 4.0| 4| 4| 4|\n" + + "| 9| 9| 10| 1| 4.0| 4| 4| 4|\n" + + "+---+------------+----------+----------------------+--------------------+--------------------+--------------------+--------------------+\n" + "Total line number = 5\n"; executor.executeAndCompare(statement, expected); } @@ -1427,122 +1437,122 @@ public void testAggregateQuery() { @Test public void testDownSampleQuery() { - String statement = "SELECT %s(s1), %s(s4) FROM us.d1 OVER (RANGE 100 IN (0, 1000));"; + String statement = "SELECT %s(s1), %s(s4) FROM us.d1 OVER WINDOW (size 100 IN (0, 1000));"; List funcTypeList = Arrays.asList("MAX", "MIN", "FIRST_VALUE", "LAST_VALUE", "SUM", "AVG", "COUNT"); List expectedList = Arrays.asList( "ResultSets:\n" - + "+---+-------------+-------------+\n" - + "|key|max(us.d1.s1)|max(us.d1.s4)|\n" - + "+---+-------------+-------------+\n" - + "| 1| 100| 100.1|\n" - + "|101| 200| 200.1|\n" - + "|201| 300| 300.1|\n" - + "|301| 400| 400.1|\n" - + "|401| 500| 500.1|\n" - + "|501| 600| 600.1|\n" - + "|601| 700| 700.1|\n" - + "|701| 800| 800.1|\n" - + "|801| 900| 900.1|\n" - + "|901| 999| 999.1|\n" - + "+---+-------------+-------------+\n" + + "+---+------------+----------+-------------+-------------+\n" + + "|key|window_start|window_end|max(us.d1.s1)|max(us.d1.s4)|\n" + + "+---+------------+----------+-------------+-------------+\n" + + "| 1| 1| 100| 100| 100.1|\n" + + "|101| 101| 200| 200| 200.1|\n" + + "|201| 201| 300| 300| 300.1|\n" + + "|301| 301| 400| 400| 400.1|\n" + + "|401| 401| 500| 500| 500.1|\n" + + "|501| 501| 600| 600| 600.1|\n" + + "|601| 601| 700| 700| 700.1|\n" + + "|701| 701| 800| 800| 800.1|\n" + + "|801| 801| 900| 900| 900.1|\n" + + "|901| 901| 1000| 999| 999.1|\n" + + "+---+------------+----------+-------------+-------------+\n" + "Total line number = 10\n", "ResultSets:\n" - + "+---+-------------+-------------+\n" - + "|key|min(us.d1.s1)|min(us.d1.s4)|\n" - + "+---+-------------+-------------+\n" - + "| 1| 1| 1.1|\n" - + "|101| 101| 101.1|\n" - + "|201| 201| 201.1|\n" - + "|301| 301| 301.1|\n" - + "|401| 401| 401.1|\n" - + "|501| 501| 501.1|\n" - + "|601| 601| 601.1|\n" - + "|701| 701| 701.1|\n" - + "|801| 801| 801.1|\n" - + "|901| 901| 901.1|\n" - + "+---+-------------+-------------+\n" + + "+---+------------+----------+-------------+-------------+\n" + + "|key|window_start|window_end|min(us.d1.s1)|min(us.d1.s4)|\n" + + "+---+------------+----------+-------------+-------------+\n" + + "| 1| 1| 100| 1| 1.1|\n" + + "|101| 101| 200| 101| 101.1|\n" + + "|201| 201| 300| 201| 201.1|\n" + + "|301| 301| 400| 301| 301.1|\n" + + "|401| 401| 500| 401| 401.1|\n" + + "|501| 501| 600| 501| 501.1|\n" + + "|601| 601| 700| 601| 601.1|\n" + + "|701| 701| 800| 701| 701.1|\n" + + "|801| 801| 900| 801| 801.1|\n" + + "|901| 901| 1000| 901| 901.1|\n" + + "+---+------------+----------+-------------+-------------+\n" + "Total line number = 10\n", "ResultSets:\n" - + "+---+---------------------+---------------------+\n" - + "|key|first_value(us.d1.s1)|first_value(us.d1.s4)|\n" - + "+---+---------------------+---------------------+\n" - + "| 1| 1| 1.1|\n" - + "|101| 101| 101.1|\n" - + "|201| 201| 201.1|\n" - + "|301| 301| 301.1|\n" - + "|401| 401| 401.1|\n" - + "|501| 501| 501.1|\n" - + "|601| 601| 601.1|\n" - + "|701| 701| 701.1|\n" - + "|801| 801| 801.1|\n" - + "|901| 901| 901.1|\n" - + "+---+---------------------+---------------------+\n" + + "+---+------------+----------+---------------------+---------------------+\n" + + "|key|window_start|window_end|first_value(us.d1.s1)|first_value(us.d1.s4)|\n" + + "+---+------------+----------+---------------------+---------------------+\n" + + "| 1| 1| 100| 1| 1.1|\n" + + "|101| 101| 200| 101| 101.1|\n" + + "|201| 201| 300| 201| 201.1|\n" + + "|301| 301| 400| 301| 301.1|\n" + + "|401| 401| 500| 401| 401.1|\n" + + "|501| 501| 600| 501| 501.1|\n" + + "|601| 601| 700| 601| 601.1|\n" + + "|701| 701| 800| 701| 701.1|\n" + + "|801| 801| 900| 801| 801.1|\n" + + "|901| 901| 1000| 901| 901.1|\n" + + "+---+------------+----------+---------------------+---------------------+\n" + "Total line number = 10\n", "ResultSets:\n" - + "+---+--------------------+--------------------+\n" - + "|key|last_value(us.d1.s1)|last_value(us.d1.s4)|\n" - + "+---+--------------------+--------------------+\n" - + "| 1| 100| 100.1|\n" - + "|101| 200| 200.1|\n" - + "|201| 300| 300.1|\n" - + "|301| 400| 400.1|\n" - + "|401| 500| 500.1|\n" - + "|501| 600| 600.1|\n" - + "|601| 700| 700.1|\n" - + "|701| 800| 800.1|\n" - + "|801| 900| 900.1|\n" - + "|901| 999| 999.1|\n" - + "+---+--------------------+--------------------+\n" + + "+---+------------+----------+--------------------+--------------------+\n" + + "|key|window_start|window_end|last_value(us.d1.s1)|last_value(us.d1.s4)|\n" + + "+---+------------+----------+--------------------+--------------------+\n" + + "| 1| 1| 100| 100| 100.1|\n" + + "|101| 101| 200| 200| 200.1|\n" + + "|201| 201| 300| 300| 300.1|\n" + + "|301| 301| 400| 400| 400.1|\n" + + "|401| 401| 500| 500| 500.1|\n" + + "|501| 501| 600| 600| 600.1|\n" + + "|601| 601| 700| 700| 700.1|\n" + + "|701| 701| 800| 800| 800.1|\n" + + "|801| 801| 900| 900| 900.1|\n" + + "|901| 901| 1000| 999| 999.1|\n" + + "+---+------------+----------+--------------------+--------------------+\n" + "Total line number = 10\n", "ResultSets:\n" - + "+---+-------------+------------------+\n" - + "|key|sum(us.d1.s1)| sum(us.d1.s4)|\n" - + "+---+-------------+------------------+\n" - + "| 1| 5050| 5060.0|\n" - + "|101| 15050|15060.000000000022|\n" - + "|201| 25050| 25059.99999999997|\n" - + "|301| 35050| 35059.99999999994|\n" - + "|401| 45050| 45059.99999999992|\n" - + "|501| 55050| 55059.99999999991|\n" - + "|601| 65050| 65059.9999999999|\n" - + "|701| 75050| 75059.99999999999|\n" - + "|801| 85050| 85060.00000000004|\n" - + "|901| 94050| 94059.9000000001|\n" - + "+---+-------------+------------------+\n" + + "+---+------------+----------+-------------+------------------+\n" + + "|key|window_start|window_end|sum(us.d1.s1)| sum(us.d1.s4)|\n" + + "+---+------------+----------+-------------+------------------+\n" + + "| 1| 1| 100| 5050| 5060.0|\n" + + "|101| 101| 200| 15050|15060.000000000022|\n" + + "|201| 201| 300| 25050| 25059.99999999997|\n" + + "|301| 301| 400| 35050| 35059.99999999994|\n" + + "|401| 401| 500| 45050| 45059.99999999992|\n" + + "|501| 501| 600| 55050| 55059.99999999991|\n" + + "|601| 601| 700| 65050| 65059.9999999999|\n" + + "|701| 701| 800| 75050| 75059.99999999999|\n" + + "|801| 801| 900| 85050| 85060.00000000004|\n" + + "|901| 901| 1000| 94050| 94059.9000000001|\n" + + "+---+------------+----------+-------------+------------------+\n" + "Total line number = 10\n", "ResultSets:\n" - + "+---+-------------+------------------+\n" - + "|key|avg(us.d1.s1)| avg(us.d1.s4)|\n" - + "+---+-------------+------------------+\n" - + "| 1| 50.5| 50.6|\n" - + "|101| 150.5|150.60000000000022|\n" - + "|201| 250.5| 250.5999999999997|\n" - + "|301| 350.5| 350.5999999999994|\n" - + "|401| 450.5| 450.5999999999992|\n" - + "|501| 550.5| 550.5999999999991|\n" - + "|601| 650.5| 650.599999999999|\n" - + "|701| 750.5| 750.5999999999999|\n" - + "|801| 850.5| 850.6000000000005|\n" - + "|901| 950.0| 950.1000000000009|\n" - + "+---+-------------+------------------+\n" + + "+---+------------+----------+-------------+------------------+\n" + + "|key|window_start|window_end|avg(us.d1.s1)| avg(us.d1.s4)|\n" + + "+---+------------+----------+-------------+------------------+\n" + + "| 1| 1| 100| 50.5| 50.6|\n" + + "|101| 101| 200| 150.5|150.60000000000022|\n" + + "|201| 201| 300| 250.5| 250.5999999999997|\n" + + "|301| 301| 400| 350.5| 350.5999999999994|\n" + + "|401| 401| 500| 450.5| 450.5999999999992|\n" + + "|501| 501| 600| 550.5| 550.5999999999991|\n" + + "|601| 601| 700| 650.5| 650.599999999999|\n" + + "|701| 701| 800| 750.5| 750.5999999999999|\n" + + "|801| 801| 900| 850.5| 850.6000000000005|\n" + + "|901| 901| 1000| 950.0| 950.1000000000009|\n" + + "+---+------------+----------+-------------+------------------+\n" + "Total line number = 10\n", "ResultSets:\n" - + "+---+---------------+---------------+\n" - + "|key|count(us.d1.s1)|count(us.d1.s4)|\n" - + "+---+---------------+---------------+\n" - + "| 1| 100| 100|\n" - + "|101| 100| 100|\n" - + "|201| 100| 100|\n" - + "|301| 100| 100|\n" - + "|401| 100| 100|\n" - + "|501| 100| 100|\n" - + "|601| 100| 100|\n" - + "|701| 100| 100|\n" - + "|801| 100| 100|\n" - + "|901| 99| 99|\n" - + "+---+---------------+---------------+\n" + + "+---+------------+----------+---------------+---------------+\n" + + "|key|window_start|window_end|count(us.d1.s1)|count(us.d1.s4)|\n" + + "+---+------------+----------+---------------+---------------+\n" + + "| 1| 1| 100| 100| 100|\n" + + "|101| 101| 200| 100| 100|\n" + + "|201| 201| 300| 100| 100|\n" + + "|301| 301| 400| 100| 100|\n" + + "|401| 401| 500| 100| 100|\n" + + "|501| 501| 600| 100| 100|\n" + + "|601| 601| 700| 100| 100|\n" + + "|701| 701| 800| 100| 100|\n" + + "|801| 801| 900| 100| 100|\n" + + "|901| 901| 1000| 99| 99|\n" + + "+---+------------+----------+---------------+---------------+\n" + "Total line number = 10\n"); for (int i = 0; i < funcTypeList.size(); i++) { String type = funcTypeList.get(i); @@ -1555,7 +1565,7 @@ public void testDownSampleQuery() { } statement = - "explain SELECT avg(s1), count(s4) FROM us.d1 OVER (RANGE 100 IN (0, 1000) STEP 50);"; + "explain SELECT avg(s1), count(s4) FROM us.d1 OVER WINDOW (size 100 IN (0, 1000) SLIDE 50);"; assertTrue( Arrays.stream(executor.execute(statement).split("\\n")) .anyMatch(s -> s.contains("Downsample") && s.contains("avg") && s.contains("count"))); @@ -1564,73 +1574,73 @@ public void testDownSampleQuery() { @Test public void testRangeDownSampleQuery() { String statement = - "SELECT %s(s1), %s(s4) FROM us.d1 WHERE key > 600 AND s1 <= 900 OVER (RANGE 100 IN (0, 1000));"; + "SELECT %s(s1), %s(s4) FROM us.d1 WHERE key > 600 AND s1 <= 900 OVER WINDOW (size 100 IN (0, 1000));"; List funcTypeList = Arrays.asList("MAX", "MIN", "FIRST_VALUE", "LAST_VALUE", "SUM", "AVG", "COUNT"); List expectedList = Arrays.asList( "ResultSets:\n" - + "+---+-------------+-------------+\n" - + "|key|max(us.d1.s1)|max(us.d1.s4)|\n" - + "+---+-------------+-------------+\n" - + "|601| 700| 700.1|\n" - + "|701| 800| 800.1|\n" - + "|801| 900| 900.1|\n" - + "+---+-------------+-------------+\n" + + "+---+------------+----------+-------------+-------------+\n" + + "|key|window_start|window_end|max(us.d1.s1)|max(us.d1.s4)|\n" + + "+---+------------+----------+-------------+-------------+\n" + + "|601| 601| 700| 700| 700.1|\n" + + "|701| 701| 800| 800| 800.1|\n" + + "|801| 801| 900| 900| 900.1|\n" + + "+---+------------+----------+-------------+-------------+\n" + "Total line number = 3\n", "ResultSets:\n" - + "+---+-------------+-------------+\n" - + "|key|min(us.d1.s1)|min(us.d1.s4)|\n" - + "+---+-------------+-------------+\n" - + "|601| 601| 601.1|\n" - + "|701| 701| 701.1|\n" - + "|801| 801| 801.1|\n" - + "+---+-------------+-------------+\n" + + "+---+------------+----------+-------------+-------------+\n" + + "|key|window_start|window_end|min(us.d1.s1)|min(us.d1.s4)|\n" + + "+---+------------+----------+-------------+-------------+\n" + + "|601| 601| 700| 601| 601.1|\n" + + "|701| 701| 800| 701| 701.1|\n" + + "|801| 801| 900| 801| 801.1|\n" + + "+---+------------+----------+-------------+-------------+\n" + "Total line number = 3\n", "ResultSets:\n" - + "+---+---------------------+---------------------+\n" - + "|key|first_value(us.d1.s1)|first_value(us.d1.s4)|\n" - + "+---+---------------------+---------------------+\n" - + "|601| 601| 601.1|\n" - + "|701| 701| 701.1|\n" - + "|801| 801| 801.1|\n" - + "+---+---------------------+---------------------+\n" + + "+---+------------+----------+---------------------+---------------------+\n" + + "|key|window_start|window_end|first_value(us.d1.s1)|first_value(us.d1.s4)|\n" + + "+---+------------+----------+---------------------+---------------------+\n" + + "|601| 601| 700| 601| 601.1|\n" + + "|701| 701| 800| 701| 701.1|\n" + + "|801| 801| 900| 801| 801.1|\n" + + "+---+------------+----------+---------------------+---------------------+\n" + "Total line number = 3\n", "ResultSets:\n" - + "+---+--------------------+--------------------+\n" - + "|key|last_value(us.d1.s1)|last_value(us.d1.s4)|\n" - + "+---+--------------------+--------------------+\n" - + "|601| 700| 700.1|\n" - + "|701| 800| 800.1|\n" - + "|801| 900| 900.1|\n" - + "+---+--------------------+--------------------+\n" + + "+---+------------+----------+--------------------+--------------------+\n" + + "|key|window_start|window_end|last_value(us.d1.s1)|last_value(us.d1.s4)|\n" + + "+---+------------+----------+--------------------+--------------------+\n" + + "|601| 601| 700| 700| 700.1|\n" + + "|701| 701| 800| 800| 800.1|\n" + + "|801| 801| 900| 900| 900.1|\n" + + "+---+------------+----------+--------------------+--------------------+\n" + "Total line number = 3\n", "ResultSets:\n" - + "+---+-------------+-----------------+\n" - + "|key|sum(us.d1.s1)| sum(us.d1.s4)|\n" - + "+---+-------------+-----------------+\n" - + "|601| 65050| 65059.9999999999|\n" - + "|701| 75050|75059.99999999999|\n" - + "|801| 85050|85060.00000000004|\n" - + "+---+-------------+-----------------+\n" + + "+---+------------+----------+-------------+-----------------+\n" + + "|key|window_start|window_end|sum(us.d1.s1)| sum(us.d1.s4)|\n" + + "+---+------------+----------+-------------+-----------------+\n" + + "|601| 601| 700| 65050| 65059.9999999999|\n" + + "|701| 701| 800| 75050|75059.99999999999|\n" + + "|801| 801| 900| 85050|85060.00000000004|\n" + + "+---+------------+----------+-------------+-----------------+\n" + "Total line number = 3\n", "ResultSets:\n" - + "+---+-------------+-----------------+\n" - + "|key|avg(us.d1.s1)| avg(us.d1.s4)|\n" - + "+---+-------------+-----------------+\n" - + "|601| 650.5| 650.599999999999|\n" - + "|701| 750.5|750.5999999999999|\n" - + "|801| 850.5|850.6000000000005|\n" - + "+---+-------------+-----------------+\n" + + "+---+------------+----------+-------------+-----------------+\n" + + "|key|window_start|window_end|avg(us.d1.s1)| avg(us.d1.s4)|\n" + + "+---+------------+----------+-------------+-----------------+\n" + + "|601| 601| 700| 650.5| 650.599999999999|\n" + + "|701| 701| 800| 750.5|750.5999999999999|\n" + + "|801| 801| 900| 850.5|850.6000000000005|\n" + + "+---+------------+----------+-------------+-----------------+\n" + "Total line number = 3\n", "ResultSets:\n" - + "+---+---------------+---------------+\n" - + "|key|count(us.d1.s1)|count(us.d1.s4)|\n" - + "+---+---------------+---------------+\n" - + "|601| 100| 100|\n" - + "|701| 100| 100|\n" - + "|801| 100| 100|\n" - + "+---+---------------+---------------+\n" + + "+---+------------+----------+---------------+---------------+\n" + + "|key|window_start|window_end|count(us.d1.s1)|count(us.d1.s4)|\n" + + "+---+------------+----------+---------------+---------------+\n" + + "|601| 601| 700| 100| 100|\n" + + "|701| 701| 800| 100| 100|\n" + + "|801| 801| 900| 100| 100|\n" + + "+---+------------+----------+---------------+---------------+\n" + "Total line number = 3\n"); for (int i = 0; i < funcTypeList.size(); i++) { String type = funcTypeList.get(i); @@ -1641,185 +1651,186 @@ public void testRangeDownSampleQuery() { @Test public void testSlideWindowByTimeQuery() { - String statement = "SELECT %s(s1), %s(s4) FROM us.d1 OVER (RANGE 100 IN (0, 1000) STEP 50);"; + String statement = + "SELECT %s(s1), %s(s4) FROM us.d1 OVER WINDOW (size 100 IN (0, 1000) SLIDE 50);"; List funcTypeList = Arrays.asList("MAX", "MIN", "FIRST_VALUE", "LAST_VALUE", "SUM", "AVG", "COUNT"); List expectedList = Arrays.asList( "ResultSets:\n" - + "+---+-------------+-------------+\n" - + "|key|max(us.d1.s1)|max(us.d1.s4)|\n" - + "+---+-------------+-------------+\n" - + "| 1| 100| 100.1|\n" - + "| 51| 150| 150.1|\n" - + "|101| 200| 200.1|\n" - + "|151| 250| 250.1|\n" - + "|201| 300| 300.1|\n" - + "|251| 350| 350.1|\n" - + "|301| 400| 400.1|\n" - + "|351| 450| 450.1|\n" - + "|401| 500| 500.1|\n" - + "|451| 550| 550.1|\n" - + "|501| 600| 600.1|\n" - + "|551| 650| 650.1|\n" - + "|601| 700| 700.1|\n" - + "|651| 750| 750.1|\n" - + "|701| 800| 800.1|\n" - + "|751| 850| 850.1|\n" - + "|801| 900| 900.1|\n" - + "|851| 950| 950.1|\n" - + "|901| 999| 999.1|\n" - + "+---+-------------+-------------+\n" + + "+---+------------+----------+-------------+-------------+\n" + + "|key|window_start|window_end|max(us.d1.s1)|max(us.d1.s4)|\n" + + "+---+------------+----------+-------------+-------------+\n" + + "| 1| 1| 100| 100| 100.1|\n" + + "| 51| 51| 150| 150| 150.1|\n" + + "|101| 101| 200| 200| 200.1|\n" + + "|151| 151| 250| 250| 250.1|\n" + + "|201| 201| 300| 300| 300.1|\n" + + "|251| 251| 350| 350| 350.1|\n" + + "|301| 301| 400| 400| 400.1|\n" + + "|351| 351| 450| 450| 450.1|\n" + + "|401| 401| 500| 500| 500.1|\n" + + "|451| 451| 550| 550| 550.1|\n" + + "|501| 501| 600| 600| 600.1|\n" + + "|551| 551| 650| 650| 650.1|\n" + + "|601| 601| 700| 700| 700.1|\n" + + "|651| 651| 750| 750| 750.1|\n" + + "|701| 701| 800| 800| 800.1|\n" + + "|751| 751| 850| 850| 850.1|\n" + + "|801| 801| 900| 900| 900.1|\n" + + "|851| 851| 950| 950| 950.1|\n" + + "|901| 901| 1000| 999| 999.1|\n" + + "+---+------------+----------+-------------+-------------+\n" + "Total line number = 19\n", "ResultSets:\n" - + "+---+-------------+-------------+\n" - + "|key|min(us.d1.s1)|min(us.d1.s4)|\n" - + "+---+-------------+-------------+\n" - + "| 1| 1| 1.1|\n" - + "| 51| 51| 51.1|\n" - + "|101| 101| 101.1|\n" - + "|151| 151| 151.1|\n" - + "|201| 201| 201.1|\n" - + "|251| 251| 251.1|\n" - + "|301| 301| 301.1|\n" - + "|351| 351| 351.1|\n" - + "|401| 401| 401.1|\n" - + "|451| 451| 451.1|\n" - + "|501| 501| 501.1|\n" - + "|551| 551| 551.1|\n" - + "|601| 601| 601.1|\n" - + "|651| 651| 651.1|\n" - + "|701| 701| 701.1|\n" - + "|751| 751| 751.1|\n" - + "|801| 801| 801.1|\n" - + "|851| 851| 851.1|\n" - + "|901| 901| 901.1|\n" - + "+---+-------------+-------------+\n" + + "+---+------------+----------+-------------+-------------+\n" + + "|key|window_start|window_end|min(us.d1.s1)|min(us.d1.s4)|\n" + + "+---+------------+----------+-------------+-------------+\n" + + "| 1| 1| 100| 1| 1.1|\n" + + "| 51| 51| 150| 51| 51.1|\n" + + "|101| 101| 200| 101| 101.1|\n" + + "|151| 151| 250| 151| 151.1|\n" + + "|201| 201| 300| 201| 201.1|\n" + + "|251| 251| 350| 251| 251.1|\n" + + "|301| 301| 400| 301| 301.1|\n" + + "|351| 351| 450| 351| 351.1|\n" + + "|401| 401| 500| 401| 401.1|\n" + + "|451| 451| 550| 451| 451.1|\n" + + "|501| 501| 600| 501| 501.1|\n" + + "|551| 551| 650| 551| 551.1|\n" + + "|601| 601| 700| 601| 601.1|\n" + + "|651| 651| 750| 651| 651.1|\n" + + "|701| 701| 800| 701| 701.1|\n" + + "|751| 751| 850| 751| 751.1|\n" + + "|801| 801| 900| 801| 801.1|\n" + + "|851| 851| 950| 851| 851.1|\n" + + "|901| 901| 1000| 901| 901.1|\n" + + "+---+------------+----------+-------------+-------------+\n" + "Total line number = 19\n", "ResultSets:\n" - + "+---+---------------------+---------------------+\n" - + "|key|first_value(us.d1.s1)|first_value(us.d1.s4)|\n" - + "+---+---------------------+---------------------+\n" - + "| 1| 1| 1.1|\n" - + "| 51| 51| 51.1|\n" - + "|101| 101| 101.1|\n" - + "|151| 151| 151.1|\n" - + "|201| 201| 201.1|\n" - + "|251| 251| 251.1|\n" - + "|301| 301| 301.1|\n" - + "|351| 351| 351.1|\n" - + "|401| 401| 401.1|\n" - + "|451| 451| 451.1|\n" - + "|501| 501| 501.1|\n" - + "|551| 551| 551.1|\n" - + "|601| 601| 601.1|\n" - + "|651| 651| 651.1|\n" - + "|701| 701| 701.1|\n" - + "|751| 751| 751.1|\n" - + "|801| 801| 801.1|\n" - + "|851| 851| 851.1|\n" - + "|901| 901| 901.1|\n" - + "+---+---------------------+---------------------+\n" + + "+---+------------+----------+---------------------+---------------------+\n" + + "|key|window_start|window_end|first_value(us.d1.s1)|first_value(us.d1.s4)|\n" + + "+---+------------+----------+---------------------+---------------------+\n" + + "| 1| 1| 100| 1| 1.1|\n" + + "| 51| 51| 150| 51| 51.1|\n" + + "|101| 101| 200| 101| 101.1|\n" + + "|151| 151| 250| 151| 151.1|\n" + + "|201| 201| 300| 201| 201.1|\n" + + "|251| 251| 350| 251| 251.1|\n" + + "|301| 301| 400| 301| 301.1|\n" + + "|351| 351| 450| 351| 351.1|\n" + + "|401| 401| 500| 401| 401.1|\n" + + "|451| 451| 550| 451| 451.1|\n" + + "|501| 501| 600| 501| 501.1|\n" + + "|551| 551| 650| 551| 551.1|\n" + + "|601| 601| 700| 601| 601.1|\n" + + "|651| 651| 750| 651| 651.1|\n" + + "|701| 701| 800| 701| 701.1|\n" + + "|751| 751| 850| 751| 751.1|\n" + + "|801| 801| 900| 801| 801.1|\n" + + "|851| 851| 950| 851| 851.1|\n" + + "|901| 901| 1000| 901| 901.1|\n" + + "+---+------------+----------+---------------------+---------------------+\n" + "Total line number = 19\n", "ResultSets:\n" - + "+---+--------------------+--------------------+\n" - + "|key|last_value(us.d1.s1)|last_value(us.d1.s4)|\n" - + "+---+--------------------+--------------------+\n" - + "| 1| 100| 100.1|\n" - + "| 51| 150| 150.1|\n" - + "|101| 200| 200.1|\n" - + "|151| 250| 250.1|\n" - + "|201| 300| 300.1|\n" - + "|251| 350| 350.1|\n" - + "|301| 400| 400.1|\n" - + "|351| 450| 450.1|\n" - + "|401| 500| 500.1|\n" - + "|451| 550| 550.1|\n" - + "|501| 600| 600.1|\n" - + "|551| 650| 650.1|\n" - + "|601| 700| 700.1|\n" - + "|651| 750| 750.1|\n" - + "|701| 800| 800.1|\n" - + "|751| 850| 850.1|\n" - + "|801| 900| 900.1|\n" - + "|851| 950| 950.1|\n" - + "|901| 999| 999.1|\n" - + "+---+--------------------+--------------------+\n" + + "+---+------------+----------+--------------------+--------------------+\n" + + "|key|window_start|window_end|last_value(us.d1.s1)|last_value(us.d1.s4)|\n" + + "+---+------------+----------+--------------------+--------------------+\n" + + "| 1| 1| 100| 100| 100.1|\n" + + "| 51| 51| 150| 150| 150.1|\n" + + "|101| 101| 200| 200| 200.1|\n" + + "|151| 151| 250| 250| 250.1|\n" + + "|201| 201| 300| 300| 300.1|\n" + + "|251| 251| 350| 350| 350.1|\n" + + "|301| 301| 400| 400| 400.1|\n" + + "|351| 351| 450| 450| 450.1|\n" + + "|401| 401| 500| 500| 500.1|\n" + + "|451| 451| 550| 550| 550.1|\n" + + "|501| 501| 600| 600| 600.1|\n" + + "|551| 551| 650| 650| 650.1|\n" + + "|601| 601| 700| 700| 700.1|\n" + + "|651| 651| 750| 750| 750.1|\n" + + "|701| 701| 800| 800| 800.1|\n" + + "|751| 751| 850| 850| 850.1|\n" + + "|801| 801| 900| 900| 900.1|\n" + + "|851| 851| 950| 950| 950.1|\n" + + "|901| 901| 1000| 999| 999.1|\n" + + "+---+------------+----------+--------------------+--------------------+\n" + "Total line number = 19\n", "ResultSets:\n" - + "+---+-------------+------------------+\n" - + "|key|sum(us.d1.s1)| sum(us.d1.s4)|\n" - + "+---+-------------+------------------+\n" - + "| 1| 5050| 5060.0|\n" - + "| 51| 10050|10060.000000000013|\n" - + "|101| 15050|15060.000000000022|\n" - + "|151| 20050|20059.999999999996|\n" - + "|201| 25050| 25059.99999999997|\n" - + "|251| 30050|30059.999999999953|\n" - + "|301| 35050| 35059.99999999994|\n" - + "|351| 40050| 40059.99999999993|\n" - + "|401| 45050| 45059.99999999992|\n" - + "|451| 50050| 50059.99999999992|\n" - + "|501| 55050| 55059.99999999991|\n" - + "|551| 60050|60059.999999999905|\n" - + "|601| 65050| 65059.9999999999|\n" - + "|651| 70050| 70059.99999999994|\n" - + "|701| 75050| 75059.99999999999|\n" - + "|751| 80050| 80060.00000000001|\n" - + "|801| 85050| 85060.00000000004|\n" - + "|851| 90050| 90060.00000000009|\n" - + "|901| 94050| 94059.9000000001|\n" - + "+---+-------------+------------------+\n" + + "+---+------------+----------+-------------+------------------+\n" + + "|key|window_start|window_end|sum(us.d1.s1)| sum(us.d1.s4)|\n" + + "+---+------------+----------+-------------+------------------+\n" + + "| 1| 1| 100| 5050| 5060.0|\n" + + "| 51| 51| 150| 10050|10060.000000000013|\n" + + "|101| 101| 200| 15050|15060.000000000022|\n" + + "|151| 151| 250| 20050|20059.999999999996|\n" + + "|201| 201| 300| 25050| 25059.99999999997|\n" + + "|251| 251| 350| 30050|30059.999999999953|\n" + + "|301| 301| 400| 35050| 35059.99999999994|\n" + + "|351| 351| 450| 40050| 40059.99999999993|\n" + + "|401| 401| 500| 45050| 45059.99999999992|\n" + + "|451| 451| 550| 50050| 50059.99999999992|\n" + + "|501| 501| 600| 55050| 55059.99999999991|\n" + + "|551| 551| 650| 60050|60059.999999999905|\n" + + "|601| 601| 700| 65050| 65059.9999999999|\n" + + "|651| 651| 750| 70050| 70059.99999999994|\n" + + "|701| 701| 800| 75050| 75059.99999999999|\n" + + "|751| 751| 850| 80050| 80060.00000000001|\n" + + "|801| 801| 900| 85050| 85060.00000000004|\n" + + "|851| 851| 950| 90050| 90060.00000000009|\n" + + "|901| 901| 1000| 94050| 94059.9000000001|\n" + + "+---+------------+----------+-------------+------------------+\n" + "Total line number = 19\n", "ResultSets:\n" - + "+---+-------------+------------------+\n" - + "|key|avg(us.d1.s1)| avg(us.d1.s4)|\n" - + "+---+-------------+------------------+\n" - + "| 1| 50.5| 50.6|\n" - + "| 51| 100.5|100.60000000000012|\n" - + "|101| 150.5|150.60000000000022|\n" - + "|151| 200.5|200.59999999999997|\n" - + "|201| 250.5| 250.5999999999997|\n" - + "|251| 300.5| 300.5999999999995|\n" - + "|301| 350.5| 350.5999999999994|\n" - + "|351| 400.5| 400.5999999999993|\n" - + "|401| 450.5| 450.5999999999992|\n" - + "|451| 500.5| 500.5999999999992|\n" - + "|501| 550.5| 550.5999999999991|\n" - + "|551| 600.5| 600.599999999999|\n" - + "|601| 650.5| 650.599999999999|\n" - + "|651| 700.5| 700.5999999999995|\n" - + "|701| 750.5| 750.5999999999999|\n" - + "|751| 800.5| 800.6000000000001|\n" - + "|801| 850.5| 850.6000000000005|\n" - + "|851| 900.5| 900.6000000000008|\n" - + "|901| 950.0| 950.1000000000009|\n" - + "+---+-------------+------------------+\n" + + "+---+------------+----------+-------------+------------------+\n" + + "|key|window_start|window_end|avg(us.d1.s1)| avg(us.d1.s4)|\n" + + "+---+------------+----------+-------------+------------------+\n" + + "| 1| 1| 100| 50.5| 50.6|\n" + + "| 51| 51| 150| 100.5|100.60000000000012|\n" + + "|101| 101| 200| 150.5|150.60000000000022|\n" + + "|151| 151| 250| 200.5|200.59999999999997|\n" + + "|201| 201| 300| 250.5| 250.5999999999997|\n" + + "|251| 251| 350| 300.5| 300.5999999999995|\n" + + "|301| 301| 400| 350.5| 350.5999999999994|\n" + + "|351| 351| 450| 400.5| 400.5999999999993|\n" + + "|401| 401| 500| 450.5| 450.5999999999992|\n" + + "|451| 451| 550| 500.5| 500.5999999999992|\n" + + "|501| 501| 600| 550.5| 550.5999999999991|\n" + + "|551| 551| 650| 600.5| 600.599999999999|\n" + + "|601| 601| 700| 650.5| 650.599999999999|\n" + + "|651| 651| 750| 700.5| 700.5999999999995|\n" + + "|701| 701| 800| 750.5| 750.5999999999999|\n" + + "|751| 751| 850| 800.5| 800.6000000000001|\n" + + "|801| 801| 900| 850.5| 850.6000000000005|\n" + + "|851| 851| 950| 900.5| 900.6000000000008|\n" + + "|901| 901| 1000| 950.0| 950.1000000000009|\n" + + "+---+------------+----------+-------------+------------------+\n" + "Total line number = 19\n", "ResultSets:\n" - + "+---+---------------+---------------+\n" - + "|key|count(us.d1.s1)|count(us.d1.s4)|\n" - + "+---+---------------+---------------+\n" - + "| 1| 100| 100|\n" - + "| 51| 100| 100|\n" - + "|101| 100| 100|\n" - + "|151| 100| 100|\n" - + "|201| 100| 100|\n" - + "|251| 100| 100|\n" - + "|301| 100| 100|\n" - + "|351| 100| 100|\n" - + "|401| 100| 100|\n" - + "|451| 100| 100|\n" - + "|501| 100| 100|\n" - + "|551| 100| 100|\n" - + "|601| 100| 100|\n" - + "|651| 100| 100|\n" - + "|701| 100| 100|\n" - + "|751| 100| 100|\n" - + "|801| 100| 100|\n" - + "|851| 100| 100|\n" - + "|901| 99| 99|\n" - + "+---+---------------+---------------+\n" + + "+---+------------+----------+---------------+---------------+\n" + + "|key|window_start|window_end|count(us.d1.s1)|count(us.d1.s4)|\n" + + "+---+------------+----------+---------------+---------------+\n" + + "| 1| 1| 100| 100| 100|\n" + + "| 51| 51| 150| 100| 100|\n" + + "|101| 101| 200| 100| 100|\n" + + "|151| 151| 250| 100| 100|\n" + + "|201| 201| 300| 100| 100|\n" + + "|251| 251| 350| 100| 100|\n" + + "|301| 301| 400| 100| 100|\n" + + "|351| 351| 450| 100| 100|\n" + + "|401| 401| 500| 100| 100|\n" + + "|451| 451| 550| 100| 100|\n" + + "|501| 501| 600| 100| 100|\n" + + "|551| 551| 650| 100| 100|\n" + + "|601| 601| 700| 100| 100|\n" + + "|651| 651| 750| 100| 100|\n" + + "|701| 701| 800| 100| 100|\n" + + "|751| 751| 850| 100| 100|\n" + + "|801| 801| 900| 100| 100|\n" + + "|851| 851| 950| 100| 100|\n" + + "|901| 901| 1000| 99| 99|\n" + + "+---+------------+----------+---------------+---------------+\n" + "Total line number = 19\n"); for (int i = 0; i < funcTypeList.size(); i++) { String type = funcTypeList.get(i); @@ -1831,101 +1842,101 @@ public void testSlideWindowByTimeQuery() { @Test public void testRangeSlideWindowByTimeQuery() { String statement = - "SELECT %s(s1), %s(s4) FROM us.d1 WHERE key > 300 AND s1 <= 600 OVER (RANGE 100 IN (0, 1000) STEP 50);"; + "SELECT %s(s1), %s(s4) FROM us.d1 WHERE key > 300 AND s1 <= 600 OVER WINDOW (size 100 IN (0, 1000) SLIDE 50);"; List funcTypeList = Arrays.asList("MAX", "MIN", "FIRST_VALUE", "LAST_VALUE", "SUM", "AVG", "COUNT"); List expectedList = Arrays.asList( "ResultSets:\n" - + "+---+-------------+-------------+\n" - + "|key|max(us.d1.s1)|max(us.d1.s4)|\n" - + "+---+-------------+-------------+\n" - + "|251| 350| 350.1|\n" - + "|301| 400| 400.1|\n" - + "|351| 450| 450.1|\n" - + "|401| 500| 500.1|\n" - + "|451| 550| 550.1|\n" - + "|501| 600| 600.1|\n" - + "|551| 600| 600.1|\n" - + "+---+-------------+-------------+\n" + + "+---+------------+----------+-------------+-------------+\n" + + "|key|window_start|window_end|max(us.d1.s1)|max(us.d1.s4)|\n" + + "+---+------------+----------+-------------+-------------+\n" + + "|251| 251| 350| 350| 350.1|\n" + + "|301| 301| 400| 400| 400.1|\n" + + "|351| 351| 450| 450| 450.1|\n" + + "|401| 401| 500| 500| 500.1|\n" + + "|451| 451| 550| 550| 550.1|\n" + + "|501| 501| 600| 600| 600.1|\n" + + "|551| 551| 650| 600| 600.1|\n" + + "+---+------------+----------+-------------+-------------+\n" + "Total line number = 7\n", "ResultSets:\n" - + "+---+-------------+-------------+\n" - + "|key|min(us.d1.s1)|min(us.d1.s4)|\n" - + "+---+-------------+-------------+\n" - + "|251| 301| 301.1|\n" - + "|301| 301| 301.1|\n" - + "|351| 351| 351.1|\n" - + "|401| 401| 401.1|\n" - + "|451| 451| 451.1|\n" - + "|501| 501| 501.1|\n" - + "|551| 551| 551.1|\n" - + "+---+-------------+-------------+\n" + + "+---+------------+----------+-------------+-------------+\n" + + "|key|window_start|window_end|min(us.d1.s1)|min(us.d1.s4)|\n" + + "+---+------------+----------+-------------+-------------+\n" + + "|251| 251| 350| 301| 301.1|\n" + + "|301| 301| 400| 301| 301.1|\n" + + "|351| 351| 450| 351| 351.1|\n" + + "|401| 401| 500| 401| 401.1|\n" + + "|451| 451| 550| 451| 451.1|\n" + + "|501| 501| 600| 501| 501.1|\n" + + "|551| 551| 650| 551| 551.1|\n" + + "+---+------------+----------+-------------+-------------+\n" + "Total line number = 7\n", "ResultSets:\n" - + "+---+---------------------+---------------------+\n" - + "|key|first_value(us.d1.s1)|first_value(us.d1.s4)|\n" - + "+---+---------------------+---------------------+\n" - + "|251| 301| 301.1|\n" - + "|301| 301| 301.1|\n" - + "|351| 351| 351.1|\n" - + "|401| 401| 401.1|\n" - + "|451| 451| 451.1|\n" - + "|501| 501| 501.1|\n" - + "|551| 551| 551.1|\n" - + "+---+---------------------+---------------------+\n" + + "+---+------------+----------+---------------------+---------------------+\n" + + "|key|window_start|window_end|first_value(us.d1.s1)|first_value(us.d1.s4)|\n" + + "+---+------------+----------+---------------------+---------------------+\n" + + "|251| 251| 350| 301| 301.1|\n" + + "|301| 301| 400| 301| 301.1|\n" + + "|351| 351| 450| 351| 351.1|\n" + + "|401| 401| 500| 401| 401.1|\n" + + "|451| 451| 550| 451| 451.1|\n" + + "|501| 501| 600| 501| 501.1|\n" + + "|551| 551| 650| 551| 551.1|\n" + + "+---+------------+----------+---------------------+---------------------+\n" + "Total line number = 7\n", "ResultSets:\n" - + "+---+--------------------+--------------------+\n" - + "|key|last_value(us.d1.s1)|last_value(us.d1.s4)|\n" - + "+---+--------------------+--------------------+\n" - + "|251| 350| 350.1|\n" - + "|301| 400| 400.1|\n" - + "|351| 450| 450.1|\n" - + "|401| 500| 500.1|\n" - + "|451| 550| 550.1|\n" - + "|501| 600| 600.1|\n" - + "|551| 600| 600.1|\n" - + "+---+--------------------+--------------------+\n" + + "+---+------------+----------+--------------------+--------------------+\n" + + "|key|window_start|window_end|last_value(us.d1.s1)|last_value(us.d1.s4)|\n" + + "+---+------------+----------+--------------------+--------------------+\n" + + "|251| 251| 350| 350| 350.1|\n" + + "|301| 301| 400| 400| 400.1|\n" + + "|351| 351| 450| 450| 450.1|\n" + + "|401| 401| 500| 500| 500.1|\n" + + "|451| 451| 550| 550| 550.1|\n" + + "|501| 501| 600| 600| 600.1|\n" + + "|551| 551| 650| 600| 600.1|\n" + + "+---+------------+----------+--------------------+--------------------+\n" + "Total line number = 7\n", "ResultSets:\n" - + "+---+-------------+------------------+\n" - + "|key|sum(us.d1.s1)| sum(us.d1.s4)|\n" - + "+---+-------------+------------------+\n" - + "|251| 16275|16280.000000000013|\n" - + "|301| 35050| 35059.99999999994|\n" - + "|351| 40050| 40059.99999999993|\n" - + "|401| 45050| 45059.99999999992|\n" - + "|451| 50050| 50059.99999999992|\n" - + "|501| 55050| 55059.99999999991|\n" - + "|551| 28775|28779.999999999975|\n" - + "+---+-------------+------------------+\n" + + "+---+------------+----------+-------------+------------------+\n" + + "|key|window_start|window_end|sum(us.d1.s1)| sum(us.d1.s4)|\n" + + "+---+------------+----------+-------------+------------------+\n" + + "|251| 251| 350| 16275|16280.000000000013|\n" + + "|301| 301| 400| 35050| 35059.99999999994|\n" + + "|351| 351| 450| 40050| 40059.99999999993|\n" + + "|401| 401| 500| 45050| 45059.99999999992|\n" + + "|451| 451| 550| 50050| 50059.99999999992|\n" + + "|501| 501| 600| 55050| 55059.99999999991|\n" + + "|551| 551| 650| 28775|28779.999999999975|\n" + + "+---+------------+----------+-------------+------------------+\n" + "Total line number = 7\n", "ResultSets:\n" - + "+---+-------------+------------------+\n" - + "|key|avg(us.d1.s1)| avg(us.d1.s4)|\n" - + "+---+-------------+------------------+\n" - + "|251| 325.5|325.60000000000025|\n" - + "|301| 350.5| 350.5999999999994|\n" - + "|351| 400.5| 400.5999999999993|\n" - + "|401| 450.5| 450.5999999999992|\n" - + "|451| 500.5| 500.5999999999992|\n" - + "|501| 550.5| 550.5999999999991|\n" - + "|551| 575.5| 575.5999999999995|\n" - + "+---+-------------+------------------+\n" + + "+---+------------+----------+-------------+------------------+\n" + + "|key|window_start|window_end|avg(us.d1.s1)| avg(us.d1.s4)|\n" + + "+---+------------+----------+-------------+------------------+\n" + + "|251| 251| 350| 325.5|325.60000000000025|\n" + + "|301| 301| 400| 350.5| 350.5999999999994|\n" + + "|351| 351| 450| 400.5| 400.5999999999993|\n" + + "|401| 401| 500| 450.5| 450.5999999999992|\n" + + "|451| 451| 550| 500.5| 500.5999999999992|\n" + + "|501| 501| 600| 550.5| 550.5999999999991|\n" + + "|551| 551| 650| 575.5| 575.5999999999995|\n" + + "+---+------------+----------+-------------+------------------+\n" + "Total line number = 7\n", "ResultSets:\n" - + "+---+---------------+---------------+\n" - + "|key|count(us.d1.s1)|count(us.d1.s4)|\n" - + "+---+---------------+---------------+\n" - + "|251| 50| 50|\n" - + "|301| 100| 100|\n" - + "|351| 100| 100|\n" - + "|401| 100| 100|\n" - + "|451| 100| 100|\n" - + "|501| 100| 100|\n" - + "|551| 50| 50|\n" - + "+---+---------------+---------------+\n" + + "+---+------------+----------+---------------+---------------+\n" + + "|key|window_start|window_end|count(us.d1.s1)|count(us.d1.s4)|\n" + + "+---+------------+----------+---------------+---------------+\n" + + "|251| 251| 350| 50| 50|\n" + + "|301| 301| 400| 100| 100|\n" + + "|351| 351| 450| 100| 100|\n" + + "|401| 401| 500| 100| 100|\n" + + "|451| 451| 550| 100| 100|\n" + + "|501| 501| 600| 100| 100|\n" + + "|551| 551| 650| 50| 50|\n" + + "+---+------------+----------+---------------+---------------+\n" + "Total line number = 7\n"); for (int i = 0; i < funcTypeList.size(); i++) { String type = funcTypeList.get(i); @@ -1934,6 +1945,98 @@ public void testRangeSlideWindowByTimeQuery() { } } + @Test + public void testRangeSlideWindowByTimeNoIntervalQuery() { + String statement = + "SELECT %s(s1), %s(s4) FROM us.d1 WHERE key > 300 AND s1 <= 600 OVER WINDOW (SIZE 100 SLIDE 50);"; + List funcTypeList = + Arrays.asList("MAX", "MIN", "FIRST_VALUE", "LAST_VALUE", "SUM", "AVG", "COUNT"); + List expectedList = + Arrays.asList( + "ResultSets:\n" + + "+---+------------+----------+-------------+-------------+\n" + + "|key|window_start|window_end|max(us.d1.s1)|max(us.d1.s4)|\n" + + "+---+------------+----------+-------------+-------------+\n" + + "|301| 301| 400| 400| 400.1|\n" + + "|351| 351| 450| 450| 450.1|\n" + + "|401| 401| 500| 500| 500.1|\n" + + "|451| 451| 550| 550| 550.1|\n" + + "|501| 501| 600| 600| 600.1|\n" + + "+---+------------+----------+-------------+-------------+\n" + + "Total line number = 5\n", + "ResultSets:\n" + + "+---+------------+----------+-------------+-------------+\n" + + "|key|window_start|window_end|min(us.d1.s1)|min(us.d1.s4)|\n" + + "+---+------------+----------+-------------+-------------+\n" + + "|301| 301| 400| 301| 301.1|\n" + + "|351| 351| 450| 351| 351.1|\n" + + "|401| 401| 500| 401| 401.1|\n" + + "|451| 451| 550| 451| 451.1|\n" + + "|501| 501| 600| 501| 501.1|\n" + + "+---+------------+----------+-------------+-------------+\n" + + "Total line number = 5\n", + "ResultSets:\n" + + "+---+------------+----------+---------------------+---------------------+\n" + + "|key|window_start|window_end|first_value(us.d1.s1)|first_value(us.d1.s4)|\n" + + "+---+------------+----------+---------------------+---------------------+\n" + + "|301| 301| 400| 301| 301.1|\n" + + "|351| 351| 450| 351| 351.1|\n" + + "|401| 401| 500| 401| 401.1|\n" + + "|451| 451| 550| 451| 451.1|\n" + + "|501| 501| 600| 501| 501.1|\n" + + "+---+------------+----------+---------------------+---------------------+\n" + + "Total line number = 5\n", + "ResultSets:\n" + + "+---+------------+----------+--------------------+--------------------+\n" + + "|key|window_start|window_end|last_value(us.d1.s1)|last_value(us.d1.s4)|\n" + + "+---+------------+----------+--------------------+--------------------+\n" + + "|301| 301| 400| 400| 400.1|\n" + + "|351| 351| 450| 450| 450.1|\n" + + "|401| 401| 500| 500| 500.1|\n" + + "|451| 451| 550| 550| 550.1|\n" + + "|501| 501| 600| 600| 600.1|\n" + + "+---+------------+----------+--------------------+--------------------+\n" + + "Total line number = 5\n", + "ResultSets:\n" + + "+---+------------+----------+-------------+-----------------+\n" + + "|key|window_start|window_end|sum(us.d1.s1)| sum(us.d1.s4)|\n" + + "+---+------------+----------+-------------+-----------------+\n" + + "|301| 301| 400| 35050|35059.99999999994|\n" + + "|351| 351| 450| 40050|40059.99999999993|\n" + + "|401| 401| 500| 45050|45059.99999999992|\n" + + "|451| 451| 550| 50050|50059.99999999992|\n" + + "|501| 501| 600| 55050|55059.99999999991|\n" + + "+---+------------+----------+-------------+-----------------+\n" + + "Total line number = 5\n", + "ResultSets:\n" + + "+---+------------+----------+-------------+-----------------+\n" + + "|key|window_start|window_end|avg(us.d1.s1)| avg(us.d1.s4)|\n" + + "+---+------------+----------+-------------+-----------------+\n" + + "|301| 301| 400| 350.5|350.5999999999994|\n" + + "|351| 351| 450| 400.5|400.5999999999993|\n" + + "|401| 401| 500| 450.5|450.5999999999992|\n" + + "|451| 451| 550| 500.5|500.5999999999992|\n" + + "|501| 501| 600| 550.5|550.5999999999991|\n" + + "+---+------------+----------+-------------+-----------------+\n" + + "Total line number = 5\n", + "ResultSets:\n" + + "+---+------------+----------+---------------+---------------+\n" + + "|key|window_start|window_end|count(us.d1.s1)|count(us.d1.s4)|\n" + + "+---+------------+----------+---------------+---------------+\n" + + "|301| 301| 400| 100| 100|\n" + + "|351| 351| 450| 100| 100|\n" + + "|401| 401| 500| 100| 100|\n" + + "|451| 451| 550| 100| 100|\n" + + "|501| 501| 600| 100| 100|\n" + + "+---+------------+----------+---------------+---------------+\n" + + "Total line number = 5\n"); + for (int i = 0; i < funcTypeList.size(); i++) { + String type = funcTypeList.get(i); + String expected = expectedList.get(i); + executor.executeAndCompare(String.format(statement, type, type), expected); + } + } + @Test public void testDelete() { if (!isAbleToDelete) { @@ -3139,12 +3242,27 @@ public void testAlias() { + "+----+-------------------------------+--------------------------+\n" + "Total line number = 10\n"; executor.executeAndCompare(statement, expected); + + // duplicate columns + statement = "SELECT s1 AS a, s1, s1 AS s1, s2 AS c, s2 FROM us.d1 WHERE s1 > 50 AND s1 < 55;"; + expected = + "ResultSets:\n" + + "+---+--+--------+--+--+--------+\n" + + "|key| a|us.d1.s1|s1| c|us.d1.s2|\n" + + "+---+--+--------+--+--+--------+\n" + + "| 51|51| 51|51|52| 52|\n" + + "| 52|52| 52|52|53| 53|\n" + + "| 53|53| 53|53|54| 54|\n" + + "| 54|54| 54|54|55| 55|\n" + + "+---+--+--------+--+--+--------+\n" + + "Total line number = 4\n"; + executor.executeAndCompare(statement, expected); } @Test public void testAggregateSubQuery() { String statement = - "SELECT %s_s1 FROM (SELECT %s(s1) AS %s_s1 FROM us.d1 OVER(RANGE 60 IN [1000, 1600)));"; + "SELECT %s_s1 FROM (SELECT %s(s1) AS %s_s1 FROM us.d1 OVER WINDOW(SIZE 60 IN [1000, 1600)));"; List funcTypeList = Arrays.asList("max", "min", "sum", "avg", "count", "first_value", "last_value"); @@ -3272,7 +3390,7 @@ public void testAggregateSubQuery() { @Test public void testSelectFromAggregate() { String statement = - "SELECT `%s(us.d1.s1)` FROM (SELECT %s(s1) FROM us.d1 OVER(RANGE 60 IN [1000, 1600)));"; + "SELECT `%s(us.d1.s1)` FROM (SELECT %s(s1) FROM us.d1 OVER WINDOW(SIZE 60 IN [1000, 1600)));"; List funcTypeList = Arrays.asList("max", "min", "sum", "avg", "count", "first_value", "last_value"); @@ -3421,7 +3539,7 @@ public void testValueFilterSubQuery() { executor.executeAndCompare(statement, expected); statement = - "SELECT avg_s1 FROM (SELECT AVG(s1) AS avg_s1 FROM us.d1 OVER (RANGE 100 IN [1000, 1600))) WHERE avg_s1 > 1200;"; + "SELECT avg_s1 FROM (SELECT AVG(s1) AS avg_s1 FROM us.d1 OVER WINDOW (size 100 IN [1000, 1600))) WHERE avg_s1 > 1200;"; expected = "ResultSets:\n" + "+----+------+\n" @@ -3436,7 +3554,7 @@ public void testValueFilterSubQuery() { executor.executeAndCompare(statement, expected); statement = - "SELECT avg_s1 FROM (SELECT AVG(s1) AS avg_s1 FROM us.d1 WHERE us.d1.s1 < 1500 OVER (RANGE 100 IN [1000, 1600))) WHERE avg_s1 > 1200;"; + "SELECT avg_s1 FROM (SELECT AVG(s1) AS avg_s1 FROM us.d1 WHERE us.d1.s1 < 1500 OVER WINDOW (size 100 IN [1000, 1600))) WHERE avg_s1 > 1200;"; expected = "ResultSets:\n" + "+----+------+\n" @@ -3453,30 +3571,30 @@ public void testValueFilterSubQuery() { @Test public void testMultiSubQuery() { String statement = - "SELECT AVG(s1) AS avg_s1, SUM(s2) AS sum_s2 FROM us.d1 OVER (RANGE 10 IN [1000, 1100));"; + "SELECT AVG(s1) AS avg_s1, SUM(s2) AS sum_s2 FROM us.d1 OVER WINDOW (size 10 IN [1000, 1100));"; String expected = "ResultSets:\n" - + "+----+------+------+\n" - + "| key|avg_s1|sum_s2|\n" - + "+----+------+------+\n" - + "|1000|1004.5| 10055|\n" - + "|1010|1014.5| 10155|\n" - + "|1020|1024.5| 10255|\n" - + "|1030|1034.5| 10355|\n" - + "|1040|1044.5| 10455|\n" - + "|1050|1054.5| 10555|\n" - + "|1060|1064.5| 10655|\n" - + "|1070|1074.5| 10755|\n" - + "|1080|1084.5| 10855|\n" - + "|1090|1094.5| 10955|\n" - + "+----+------+------+\n" + + "+----+------------+----------+------+------+\n" + + "| key|window_start|window_end|avg_s1|sum_s2|\n" + + "+----+------------+----------+------+------+\n" + + "|1000| 1000| 1009|1004.5| 10055|\n" + + "|1010| 1010| 1019|1014.5| 10155|\n" + + "|1020| 1020| 1029|1024.5| 10255|\n" + + "|1030| 1030| 1039|1034.5| 10355|\n" + + "|1040| 1040| 1049|1044.5| 10455|\n" + + "|1050| 1050| 1059|1054.5| 10555|\n" + + "|1060| 1060| 1069|1064.5| 10655|\n" + + "|1070| 1070| 1079|1074.5| 10755|\n" + + "|1080| 1080| 1089|1084.5| 10855|\n" + + "|1090| 1090| 1099|1094.5| 10955|\n" + + "+----+------------+----------+------+------+\n" + "Total line number = 10\n"; executor.executeAndCompare(statement, expected); statement = "SELECT avg_s1, sum_s2 " + "FROM (SELECT AVG(s1) AS avg_s1, SUM(s2) AS sum_s2 " - + "FROM us.d1 OVER (RANGE 10 IN [1000, 1100))) " + + "FROM us.d1 OVER WINDOW (size 10 IN [1000, 1100))) " + "WHERE avg_s1 > 1020 AND sum_s2 < 10800;"; expected = "ResultSets:\n" @@ -3497,7 +3615,7 @@ public void testMultiSubQuery() { "SELECT MAX(avg_s1), MIN(sum_s2) " + "FROM (SELECT avg_s1, sum_s2 " + "FROM (SELECT AVG(s1) AS avg_s1, SUM(s2) AS sum_s2 " - + "FROM us.d1 OVER (RANGE 10 IN [1000, 1100))) " + + "FROM us.d1 OVER WINDOW (size 10 IN [1000, 1100))) " + "WHERE avg_s1 > 1020 AND sum_s2 < 10800);"; expected = "ResultSets:\n" @@ -5282,7 +5400,9 @@ public void testInsertWithSubQuery() { executor.executeAndCompare(query, expected); insert = - "INSERT INTO us.d4(key, s1, s2) VALUES (SELECT AVG(s1) AS avg_s1, SUM(s2) AS sum_s2 FROM us.d1 OVER (RANGE 10 IN [1000, 1100)));"; + "INSERT INTO us.d4(key, s1, s2) VALUES " + + "(SELECT avg_s1, sum_s2 from " + + "(SELECT AVG(s1) AS avg_s1, SUM(s2) AS sum_s2 FROM us.d1 OVER WINDOW (size 10 IN [1000, 1100))));"; executor.execute(insert); query = "SELECT s1, s2 FROM us.d4;"; @@ -5308,7 +5428,7 @@ public void testInsertWithSubQuery() { insert = "INSERT INTO us.d5(key, s1, s2) VALUES (SELECT avg_s1, sum_s2 " + "FROM (SELECT AVG(s1) AS avg_s1, SUM(s2) AS sum_s2 " - + "FROM us.d1 OVER (RANGE 10 IN [1000, 1100))) " + + "FROM us.d1 OVER WINDOW (size 10 IN [1000, 1100))) " + "WHERE avg_s1 > 1020 AND sum_s2 < 10800);"; executor.execute(insert); @@ -5332,7 +5452,7 @@ public void testInsertWithSubQuery() { "INSERT INTO us.d6(key, s1, s2) VALUES (SELECT MAX(avg_s1), MIN(sum_s2) " + "FROM (SELECT avg_s1, sum_s2 " + "FROM (SELECT AVG(s1) AS avg_s1, SUM(s2) AS sum_s2 " - + "FROM us.d1 OVER (RANGE 10 IN [1000, 1100))) " + + "FROM us.d1 OVER WINDOW (size 10 IN [1000, 1100))) " + "WHERE avg_s1 > 1020 AND sum_s2 < 10800));"; executor.execute(insert); @@ -5672,7 +5792,7 @@ public void testErrorClause() { errClause = "DELETE FROM us.d1.s1 WHERE key != 105;"; executor.executeAndCompareErrMsg(errClause, "Not support [!=] in delete clause."); - errClause = "SELECT s1 FROM us.d1 OVER (RANGE 100 IN (0, 1000));"; + errClause = "SELECT s1 FROM us.d1 OVER WINDOW (size 100 IN (0, 1000));"; executor.executeAndCompareErrMsg( errClause, "Downsample clause cannot be used without aggregate function."); @@ -5680,7 +5800,7 @@ public void testErrorClause() { executor.executeAndCompareErrMsg( errClause, "SetToSet/SetToRow/RowToRow functions can not be mixed in aggregate query."); - errClause = "SELECT s1 FROM us.d1 OVER (RANGE 100 IN (100, 10));"; + errClause = "SELECT s1 FROM us.d1 OVER WINDOW (size 100 IN (100, 10));"; executor.executeAndCompareErrMsg( errClause, "start key should be smaller than end key in key interval."); @@ -6053,19 +6173,19 @@ public void testConcurrentQuery() { + "+---+--------+\n" + "Total line number = 10\n"), new Pair<>( - "SELECT max(s1), max(s4) FROM us.d1 WHERE key > 300 AND s1 <= 600 OVER (RANGE 100 IN (0, 1000) STEP 50);", + "SELECT max(s1), max(s4) FROM us.d1 WHERE key > 300 AND s1 <= 600 OVER WINDOW (size 100 IN (0, 1000) SLIDE 50);", "ResultSets:\n" - + "+---+-------------+-------------+\n" - + "|key|max(us.d1.s1)|max(us.d1.s4)|\n" - + "+---+-------------+-------------+\n" - + "|251| 350| 350.1|\n" - + "|301| 400| 400.1|\n" - + "|351| 450| 450.1|\n" - + "|401| 500| 500.1|\n" - + "|451| 550| 550.1|\n" - + "|501| 600| 600.1|\n" - + "|551| 600| 600.1|\n" - + "+---+-------------+-------------+\n" + + "+---+------------+----------+-------------+-------------+\n" + + "|key|window_start|window_end|max(us.d1.s1)|max(us.d1.s4)|\n" + + "+---+------------+----------+-------------+-------------+\n" + + "|251| 251| 350| 350| 350.1|\n" + + "|301| 301| 400| 400| 400.1|\n" + + "|351| 351| 450| 450| 450.1|\n" + + "|401| 401| 500| 500| 500.1|\n" + + "|451| 451| 550| 550| 550.1|\n" + + "|501| 501| 600| 600| 600.1|\n" + + "|551| 551| 650| 600| 600.1|\n" + + "+---+------------+----------+-------------+-------------+\n" + "Total line number = 7\n"), new Pair<>( "select avg(test1.a), test2.d from test1 join test2 on test1.a = test2.a group by test2.d;", @@ -6769,7 +6889,7 @@ public void testFilterFragmentOptimizer() { + "|Reorder | Reorder| Order: avg(bb)|\n" + "| +--GroupBy | GroupBy|GroupByCols: aa, FuncList(Name, FuncType): (avg, System), MappingType: SetMapping isDistinct: false|\n" + "| +--Select | Select| Filter: key > 2|\n" - + "| +--Rename | Rename| AliasMap: (us.d2.a, aa),(us.d2.b, bb)|\n" + + "| +--Rename | Rename| AliasList: (us.d2.a, aa),(us.d2.b, bb)|\n" + "| +--Reorder | Reorder| Order: us.d2.a,us.d2.b|\n" + "| +--Project | Project| Patterns: us.d2.a,us.d2.b|\n" + "| +--PathUnion| PathUnion| |\n" @@ -6785,9 +6905,9 @@ public void testFilterFragmentOptimizer() { "explain SELECT COUNT(*)\n" + "FROM (\n" + " SELECT AVG(s1) AS avg_s1, SUM(s2) AS sum_s2\n" - + " FROM us.d1 OVER (RANGE 10 IN [1000, 1100))\n" + + " FROM us.d1 OVER WINDOW (size 10 IN [1000, 1100))\n" + ")\n" - + "OVER (RANGE 20 IN [1000, 1100));", + + "OVER WINDOW (size 20 IN [1000, 1100));", "ResultSets:\n" + "+------------------------+-------------+----------------------------------------------------------------------------------------------------------------------------------------------+\n" + "| Logical Tree|Operator Type| Operator Info|\n" @@ -6795,7 +6915,7 @@ public void testFilterFragmentOptimizer() { + "|Reorder | Reorder| Order: count(*)|\n" + "| +--Downsample | Downsample| Precision: 20, SlideDistance: 20, TimeRange: [1000, 1100), FuncList(Name, FunctionType): (count, System), MappingType: SetMapping|\n" + "| +--Select | Select| Filter: (key >= 1000 && key < 1100)|\n" - + "| +--Rename | Rename| AliasMap: (avg(us.d1.s1), avg_s1),(sum(us.d1.s2), sum_s2)|\n" + + "| +--Rename | Rename| AliasList: (avg(us.d1.s1), avg_s1),(sum(us.d1.s2), sum_s2)|\n" + "| +--Reorder | Reorder| Order: avg(us.d1.s1),sum(us.d1.s2)|\n" + "| +--Downsample | Downsample|Precision: 10, SlideDistance: 10, TimeRange: [1000, 1100), FuncList(Name, FunctionType): (avg, System), (sum, System), MappingType: SetMapping|\n" + "| +--Select | Select| Filter: (key >= 1000 && key < 1100)|\n" @@ -6844,9 +6964,9 @@ public void testFilterFragmentOptimizer() { "explain SELECT COUNT(*)\n" + "FROM (\n" + " SELECT AVG(s1) AS avg_s1, SUM(s2) AS sum_s2\n" - + " FROM us.d1 OVER (RANGE 10 IN [1000, 1100))\n" + + " FROM us.d1 OVER WINDOW (size 10 IN [1000, 1100))\n" + ")\n" - + "OVER (RANGE 20 IN [1000, 1100));", + + "OVER WINDOW (size 20 IN [1000, 1100));", "ResultSets:\n" + "+--------------------------+-------------+----------------------------------------------------------------------------------------------------------------------------------------------+\n" + "| Logical Tree|Operator Type| Operator Info|\n" @@ -6854,7 +6974,7 @@ public void testFilterFragmentOptimizer() { + "|Reorder | Reorder| Order: count(*)|\n" + "| +--Downsample | Downsample| Precision: 20, SlideDistance: 20, TimeRange: [1000, 1100), FuncList(Name, FunctionType): (count, System), MappingType: SetMapping|\n" + "| +--Select | Select| Filter: (key >= 1000 && key < 1100)|\n" - + "| +--Rename | Rename| AliasMap: (avg(us.d1.s1), avg_s1),(sum(us.d1.s2), sum_s2)|\n" + + "| +--Rename | Rename| AliasList: (avg(us.d1.s1), avg_s1),(sum(us.d1.s2), sum_s2)|\n" + "| +--Reorder | Reorder| Order: avg(us.d1.s1),sum(us.d1.s2)|\n" + "| +--Downsample | Downsample|Precision: 10, SlideDistance: 10, TimeRange: [1000, 1100), FuncList(Name, FunctionType): (avg, System), (sum, System), MappingType: SetMapping|\n" + "| +--Select | Select| Filter: (key >= 1000 && key < 1100)|\n" @@ -7625,4 +7745,271 @@ public void testDistinctEliminate() { assertTrue(executor.execute("EXPLAIN " + statement).contains("isDistinct: true")); assertEquals(closeResult, executor.execute(statement)); } + + @Test + public void testJoinFactorizationRule() { + if (isFilterPushDown || isScaling) { + LOGGER.info( + "Skip SQLSessionIT.testJoinFactorizationRule because filter push down test or scaling test"); + return; + } + String openRule = "SET RULES JoinFactorizationRule=on;"; + String closeRule = "SET RULES JoinFactorizationRule=off;"; + + StringBuilder insert = new StringBuilder(); + insert.append("INSERT INTO t1 (key, c1, c2) VALUES "); + int rows = 50; + for (int i = 0; i < rows; i++) { + insert.append(String.format("(%d, %d, %d)", i, i % 10, i % 10)); + if (i != rows - 1) { + insert.append(","); + } + } + insert.append(";"); + executor.execute(insert.toString()); + + insert = new StringBuilder(); + insert.append("INSERT INTO t2 (key, c1, c2) VALUES "); + for (int i = 0; i < rows; i++) { + insert.append(String.format("(%d, %d, %d)", i, i % 20, i % 20)); + if (i != rows - 1) { + insert.append(","); + } + } + insert.append(";"); + executor.execute(insert.toString()); + + insert = new StringBuilder(); + insert.append("INSERT INTO t3 (key, c1, c2) VALUES "); + for (int i = 0; i < rows; i++) { + insert.append(String.format("(%d, %d, %d)", i, i % 30, i % 30)); + if (i != rows - 1) { + insert.append(","); + } + } + insert.append(";"); + executor.execute(insert.toString()); + + insert = new StringBuilder(); + insert.append("INSERT INTO t4 (key, c1, c2) VALUES "); + for (int i = 0; i < rows; i++) { + insert.append(String.format("(%d, %d, %d)", i, i % 40, i % 40)); + if (i != rows - 1) { + insert.append(","); + } + } + insert.append(";"); + executor.execute(insert.toString()); + + // 待优化的语句 + String statement = + "SELECT t1.c1, t2.c2\n" + + " FROM t1, t2, t3\n" + + " WHERE t1.c1 = t2.c1\n" + + " AND t1.c1 > 1\n" + + " AND t2.c2 = 2\n" + + " AND t2.c2 = t3.c2\n" + + " UNION ALL\n" + + " SELECT t1.c1, t2.c2\n" + + " FROM t1, t2, t4\n" + + " WHERE t1.c1 = t2.c1\n" + + " AND t1.c1 > 1\n" + + " AND t2.c1 = t4.c1;"; + // 优化后的查询计划与下面这个语句的基本相同,会有些order、filter顺序的不同,不影响结果 + // 例如project t1.c1 t2.c2和project t2.c2 t1.c1的区别 + String optimizing = + " SELECT t1.c1, t2.c2\n" + + " FROM t1, (SELECT t2.c1, t2.c2\n" + + " FROM t2, t3\n" + + " WHERE t2.c2 = t3.c2\n" + + " AND t2.c2 = 2\n" + + " UNION ALL\n" + + " SELECT t2.c1, t2.c2\n" + + " FROM t2, t4\n" + + " WHERE t2.c1 = t4.c1)\n" + + " WHERE t1.c1 = t2.c1\n" + + " AND t1.c1 > 1;"; + + executor.execute(openRule); + String result = executor.execute(statement); + executor.executeAndCompare(optimizing, result); + String expect = + "ResultSets:\n" + + "+----------------------------+-------------+------------------------------------------------------------------+\n" + + "| Logical Tree|Operator Type| Operator Info|\n" + + "+----------------------------+-------------+------------------------------------------------------------------+\n" + + "|Reorder | Reorder| Order: t1.c1,t2.c2|\n" + + "| +--Project | Project| Patterns: t1.c1,t2.c2|\n" + + "| +--Select | Select| Filter: (t1.c1 == t2.c1 && t1.c1 > 1)|\n" + + "| +--CrossJoin | CrossJoin| PrefixA: t1, PrefixB: null|\n" + + "| +--Project | Project| Patterns: t1.*, Target DU: unit0000000002|\n" + + "| +--Union | Union|LeftOrder: t2.c2,t2.c1, RightOrder: t2.c2,t2.c1, isDistinct: false|\n" + + "| +--Reorder | Reorder| Order: t2.c1,t2.c2|\n" + + "| +--Project | Project| Patterns: t2.c1,t2.c2|\n" + + "| +--Select | Select| Filter: (t2.c2 == 2 && t3.c2 == 2)|\n" + + "| +--CrossJoin| CrossJoin| PrefixA: t2, PrefixB: t3|\n" + + "| +--Project| Project| Patterns: t2.c1,t2.c2, Target DU: unit0000000002|\n" + + "| +--Project| Project| Patterns: t3.c2, Target DU: unit0000000002|\n" + + "| +--Reorder | Reorder| Order: t2.c1,t2.c2|\n" + + "| +--Project | Project| Patterns: t2.c1,t2.c2|\n" + + "| +--Select | Select| Filter: (t2.c1 == t4.c1)|\n" + + "| +--CrossJoin| CrossJoin| PrefixA: t2, PrefixB: t4|\n" + + "| +--Project| Project| Patterns: t2.c1,t2.c2, Target DU: unit0000000002|\n" + + "| +--Project| Project| Patterns: t4.c1, Target DU: unit0000000002|\n" + + "+----------------------------+-------------+------------------------------------------------------------------+\n" + + "Total line number = 18\n"; + assertEquals(expect, executor.execute("EXPLAIN " + statement)); + + executor.execute(closeRule); + assertTrue(TestUtils.compareTables(executor.execute(optimizing), result)); + expect = + "ResultSets:\n" + + "+----------------------+-------------+------------------------------------------------------------------+\n" + + "| Logical Tree|Operator Type| Operator Info|\n" + + "+----------------------+-------------+------------------------------------------------------------------+\n" + + "|Union | Union|LeftOrder: t1.c1,t2.c2, RightOrder: t1.c1,t2.c2, isDistinct: false|\n" + + "| +--Reorder | Reorder| Order: t1.c1,t2.c2|\n" + + "| +--Project | Project| Patterns: t2.c2,t1.c1|\n" + + "| +--Select | Select| Filter: (t1.c1 == t2.c1 && t1.c1 > 1 && t2.c2 == 2 && t3.c2 == 2)|\n" + + "| +--CrossJoin | CrossJoin| PrefixA: t2, PrefixB: t3|\n" + + "| +--CrossJoin| CrossJoin| PrefixA: t1, PrefixB: t2|\n" + + "| +--Project| Project| Patterns: t1.*, Target DU: unit0000000002|\n" + + "| +--Project| Project| Patterns: t2.*, Target DU: unit0000000002|\n" + + "| +--Project | Project| Patterns: t3.*, Target DU: unit0000000002|\n" + + "| +--Reorder | Reorder| Order: t1.c1,t2.c2|\n" + + "| +--Project | Project| Patterns: t2.c2,t1.c1|\n" + + "| +--Select | Select| Filter: (t1.c1 == t2.c1 && t1.c1 > 1 && t2.c1 == t4.c1)|\n" + + "| +--CrossJoin | CrossJoin| PrefixA: t2, PrefixB: t4|\n" + + "| +--CrossJoin| CrossJoin| PrefixA: t1, PrefixB: t2|\n" + + "| +--Project| Project| Patterns: t1.*, Target DU: unit0000000002|\n" + + "| +--Project| Project| Patterns: t2.*, Target DU: unit0000000002|\n" + + "| +--Project | Project| Patterns: t4.*, Target DU: unit0000000002|\n" + + "+----------------------+-------------+------------------------------------------------------------------+\n" + + "Total line number = 17\n"; + assertEquals(expect, executor.execute("EXPLAIN " + statement)); + + statement = + "SELECT t1.c2, t2.c2\n" + + "FROM t1, t2\n" + + "WHERE t1.c1 = t2.c1 \n" + + "AND t1.c1 = 1\n" + + "UNION ALL\n" + + "SELECT t1.c2, t2.c2\n" + + "FROM t1, t2\n" + + "WHERE t1.c1 = t2.c1 \n" + + "AND t1.c1 = 2;"; + optimizing = + "SELECT t1.c2, t2.c2\n" + + "FROM t2, (SELECT c1, c2\n" + + " FROM t1\n" + + " WHERE t1.c1 = 1\n" + + " UNION ALL\n" + + " SELECT c1, c2\n" + + " FROM t1\n" + + " WHERE t1.c1 = 2)\n" + + "WHERE t1.c1 = t2.c1;"; + + executor.execute(openRule); + result = executor.execute(statement); + executor.executeAndCompare(optimizing, result); + expect = + "ResultSets:\n" + + "+--------------------------+-------------+------------------------------------------------------------------+\n" + + "| Logical Tree|Operator Type| Operator Info|\n" + + "+--------------------------+-------------+------------------------------------------------------------------+\n" + + "|Reorder | Reorder| Order: t1.c2,t2.c2|\n" + + "| +--Project | Project| Patterns: t1.c2,t2.c2|\n" + + "| +--Select | Select| Filter: (t1.c1 == t2.c1)|\n" + + "| +--CrossJoin | CrossJoin| PrefixA: t2, PrefixB: null|\n" + + "| +--Project | Project| Patterns: t2.*, Target DU: unit0000000002|\n" + + "| +--Union | Union|LeftOrder: t1.c2,t1.c1, RightOrder: t1.c2,t1.c1, isDistinct: false|\n" + + "| +--Reorder | Reorder| Order: t1.c1,t1.c2|\n" + + "| +--Project | Project| Patterns: t1.c1,t1.c2|\n" + + "| +--Select | Select| Filter: (t1.c1 == 1)|\n" + + "| +--Project| Project| Patterns: t1.c1,t1.c2, Target DU: unit0000000002|\n" + + "| +--Reorder | Reorder| Order: t1.c1,t1.c2|\n" + + "| +--Project | Project| Patterns: t1.c1,t1.c2|\n" + + "| +--Select | Select| Filter: (t1.c1 == 2)|\n" + + "| +--Project| Project| Patterns: t1.c1,t1.c2, Target DU: unit0000000002|\n" + + "+--------------------------+-------------+------------------------------------------------------------------+\n" + + "Total line number = 14\n"; + assertEquals(expect, executor.execute("EXPLAIN " + statement)); + + executor.execute(closeRule); + assertTrue(TestUtils.compareTables(executor.execute(optimizing), result)); + expect = + "ResultSets:\n" + + "+--------------------+-------------+------------------------------------------------------------------+\n" + + "| Logical Tree|Operator Type| Operator Info|\n" + + "+--------------------+-------------+------------------------------------------------------------------+\n" + + "|Union | Union|LeftOrder: t1.c2,t2.c2, RightOrder: t1.c2,t2.c2, isDistinct: false|\n" + + "| +--Reorder | Reorder| Order: t1.c2,t2.c2|\n" + + "| +--Project | Project| Patterns: t2.c2,t1.c2|\n" + + "| +--Select | Select| Filter: (t2.c1 == 1 && t1.c1 == 1)|\n" + + "| +--CrossJoin| CrossJoin| PrefixA: t1, PrefixB: t2|\n" + + "| +--Project| Project| Patterns: t1.*, Target DU: unit0000000002|\n" + + "| +--Project| Project| Patterns: t2.*, Target DU: unit0000000002|\n" + + "| +--Reorder | Reorder| Order: t1.c2,t2.c2|\n" + + "| +--Project | Project| Patterns: t2.c2,t1.c2|\n" + + "| +--Select | Select| Filter: (t2.c1 == 2 && t1.c1 == 2)|\n" + + "| +--CrossJoin| CrossJoin| PrefixA: t1, PrefixB: t2|\n" + + "| +--Project| Project| Patterns: t1.*, Target DU: unit0000000002|\n" + + "| +--Project| Project| Patterns: t2.*, Target DU: unit0000000002|\n" + + "+--------------------+-------------+------------------------------------------------------------------+\n" + + "Total line number = 13\n"; + assertEquals(expect, executor.execute("EXPLAIN " + statement)); + } + + @Test + public void testOuterJoinEliminate() { + StringBuilder insert = new StringBuilder(); + insert.append("INSERT INTO us (key, d2.s1, d2.s2, d3.s1, d3.s2) VALUES "); + int rows = 15000; + for (int i = 0; i < rows; i++) { + insert.append(String.format("(%d, %d, %d, %d, %d)", i, i % 100, i % 150, i % 200, i % 250)); + if (i != rows - 1) { + insert.append(","); + } + } + insert.append(";"); + executor.execute(insert.toString()); + + String openRule = "SET RULES OuterJoinEliminateRule=on;"; + String closeRule = "SET RULES OuterJoinEliminateRule=off;"; + String openResult, openExplain, closeResult, closeExplain; + + String statement = + "SELECT distinct us.d1.s1, us.d1.s2 FROM us.d1 LEFT JOIN us.d2 ON us.d1.s1 = us.d2.s1;"; + executor.execute(openRule); + openResult = executor.execute(statement); + openExplain = executor.execute("EXPLAIN " + statement); + executor.execute(closeRule); + closeResult = executor.execute(statement); + closeExplain = executor.execute("EXPLAIN " + statement); + assertEquals(openResult, closeResult); + assertTrue(!openExplain.contains("OuterJoin") && closeExplain.contains("OuterJoin")); + + statement = + "SELECT * FROM us.d1 WHERE s1 IN (SELECT us.d2.s1 FROM us.d2 LEFT JOIN us.d3 ON us.d2.s1 = us.d3.s1);"; + executor.execute(openRule); + openResult = executor.execute(statement); + openExplain = executor.execute("EXPLAIN " + statement); + executor.execute(closeRule); + closeResult = executor.execute(statement); + closeExplain = executor.execute("EXPLAIN " + statement); + assertEquals(openResult, closeResult); + assertTrue(!openExplain.contains("OuterJoin") && closeExplain.contains("OuterJoin")); + + statement = + "SELECT * FROM us.d1 WHERE EXISTS " + + "(SELECT us.d2.s1 FROM us.d2 LEFT JOIN us.d3 ON us.d2.s1 = us.d3.s1);"; + executor.execute(openRule); + openResult = executor.execute(statement); + openExplain = executor.execute("EXPLAIN " + statement); + executor.execute(closeRule); + closeResult = executor.execute(statement); + closeExplain = executor.execute("EXPLAIN " + statement); + assertEquals(openResult, closeResult); + assertTrue(!openExplain.contains("OuterJoin") && closeExplain.contains("OuterJoin")); + } } diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/SQLSessionPoolIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/SQLSessionPoolIT.java index ce3652e2f4..02acf2e09e 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/SQLSessionPoolIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/SQLSessionPoolIT.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.func.sql; import static cn.edu.tsinghua.iginx.integration.controller.Controller.SUPPORT_KEY; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/tag/TagIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/tag/TagIT.java index 971b5bdd29..7457b0dbfa 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/tag/TagIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/tag/TagIT.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.func.tag; import static cn.edu.tsinghua.iginx.constant.GlobalConstant.CLEAR_DUMMY_DATA_CAUTION; @@ -917,7 +935,6 @@ public void testDeleteTSWithMultiTags() { String showColumnsData = "SELECT v FROM ah.* WITH t1=v1 AND t2=v2;"; expected = "ResultSets:\n" + "+---+\n" + "|key|\n" + "+---+\n" + "+---+\n" + "Empty set.\n"; - ; executeAndCompare(showColumnsData, expected); deleteTimeSeries = "DELETE COLUMNS * WITH t1=v1 AND t2=vv2 OR t1=vv1 AND t2=v2;"; @@ -945,7 +962,6 @@ public void testDeleteTSWithMultiTags() { showColumnsData = "SELECT * FROM * WITH t1=v1 AND t2=vv2 OR t1=vv1 AND t2=v2;"; expected = "ResultSets:\n" + "+---+\n" + "|key|\n" + "+---+\n" + "+---+\n" + "Empty set.\n"; - ; executeAndCompare(showColumnsData, expected); } @@ -1325,6 +1341,18 @@ public void testAlias() { + "Total line number = 2\n"; executeAndCompare(statement, expected); + statement = "SELECT s AS ts, s AS ss FROM ah.hr02;"; + expected = + "ResultSets:\n" + + "+---+----+---------+----+---------+\n" + + "|key| ts|ts{t1=v1}| ss|ss{t1=v1}|\n" + + "+---+----+---------+----+---------+\n" + + "|100|true| null|true| null|\n" + + "|400|null| false|null| false|\n" + + "+---+----+---------+----+---------+\n" + + "Total line number = 2\n"; + executeAndCompare(statement, expected); + statement = "SELECT s FROM ah.hr02 AS result_set;"; expected = "ResultSets:\n" diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/RemoteUDFIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/RemoteUDFIT.java index 1e4ebd6856..60539fd249 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/RemoteUDFIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/RemoteUDFIT.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.func.udf; import static cn.edu.tsinghua.iginx.integration.controller.Controller.clearAllData; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/TransformIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/TransformIT.java index 58078130ea..083819ad75 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/TransformIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/TransformIT.java @@ -1,25 +1,25 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.integration.func.udf; import static cn.edu.tsinghua.iginx.integration.controller.Controller.SUPPORT_KEY; import static cn.edu.tsinghua.iginx.integration.controller.Controller.clearAllData; +import static cn.edu.tsinghua.iginx.utils.FileUtils.appendFile; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; @@ -37,21 +37,16 @@ import cn.edu.tsinghua.iginx.utils.RpcUtils; import cn.edu.tsinghua.iginx.utils.YAMLReader; import cn.edu.tsinghua.iginx.utils.YAMLWriter; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.IOException; +import java.io.*; import java.nio.file.Files; import java.nio.file.Paths; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.commons.lang3.RandomStringUtils; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; +import org.junit.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -216,6 +211,22 @@ private void verifyJobState(long jobId) throws SessionException, InterruptedExce assertTrue(finishedJobIds.contains(jobId)); } + private void cancelJob(long jobID) { + try { + JobState jobState; + session.cancelTransformJob(jobID); + jobState = session.queryTransformJobStatus(jobID); + LOGGER.info("After cancellation, job {} state is {}", jobID, jobState.toString()); + assertEquals(JobState.JOB_CLOSED, jobState); + + List closedJobIds = session.showEligibleJob(JobState.JOB_CLOSED); + assertTrue(closedJobIds.contains(jobID)); + } catch (SessionException e) { + LOGGER.error("Failed to cancel job: {}", jobID, e); + fail(); + } + } + @Test public void exportFileWithoutPermissionTest() { LOGGER.info("exportFileWithoutPermissionTest"); @@ -273,6 +284,145 @@ public void commitSingleSqlStatementByYamlTest() { } } + @Test + public void commitScheduledYamlTestAfter10s() { + LOGGER.info("commitScheduledYamlTest(after 10s)"); + String outputFileName = OUTPUT_DIR_PREFIX + File.separator + "export_file_after_10_s.txt"; + try { + String yamlFileName = OUTPUT_DIR_PREFIX + File.separator + "TransformScheduledAfter10s.yaml"; + SessionExecuteSqlResult result = + session.executeSql(String.format(COMMIT_SQL_FORMATTER, yamlFileName)); + + Thread.sleep(3000L); // sleep 3s to delay insertion + String insertSQL = "insert into scheduleData(key, %s) values(1, 2);"; + // add col0 + session.executeSql(String.format(insertSQL, "col0")); + + long jobId = result.getJobId(); + verifyJobState(jobId); + // job will finish after 10 seconds + + // check whether new column is in job result + fileResultContains(outputFileName, "col0"); + } catch (SessionException | InterruptedException e) { + LOGGER.error("Transform: execute fail. Caused by:", e); + fail(); + } finally { + try { + assertTrue(Files.deleteIfExists(Paths.get(outputFileName))); + } catch (IOException e) { + LOGGER.error("Fail to delete result file: {}", outputFileName, e); + fail(); + } + } + } + + @Test + public void commitScheduledYamlTestEvery10sAndCancel() { + LOGGER.info("commitScheduledYamlTest(every 10s) and cancel"); + String outputFileName = OUTPUT_DIR_PREFIX + File.separator + "export_file_every_10_s.txt"; + try { + String insertSQL = "insert into scheduleData(key, %s) values(1, 2);"; + String yamlFileName = OUTPUT_DIR_PREFIX + File.separator + "TransformScheduledEvery10s.yaml"; + // add col0 + session.executeSql(String.format(insertSQL, "col0")); + SessionExecuteSqlResult result = + session.executeSql(String.format(COMMIT_SQL_FORMATTER, yamlFileName)); + long jobId = result.getJobId(); + try { + + Thread.sleep(3000L); // sleep 3s to make sure first try is triggered. + LOGGER.info("Verifying 0th try..."); + fileResultContains(outputFileName, "col0"); + + // add col1, col2 and verify res are changed + for (int i = 1; i < 3; i++) { + session.executeSql(String.format(insertSQL, "col" + i)); + Thread.sleep(10000L); // sleep 10s to make sure next try is triggered. + LOGGER.info("Verifying " + i + "th try..."); + fileResultContains(outputFileName, "col" + i); + } + } finally { + cancelJob(jobId); + assertTrue(Files.deleteIfExists(Paths.get(outputFileName))); + } + } catch (SessionException | InterruptedException e) { + LOGGER.error("Transform: execute fail. Caused by:", e); + fail(); + } catch (IOException e) { + LOGGER.error("Fail to delete result file: {}", outputFileName, e); + fail(); + } + } + + @Ignore // this test takes too much time(> 1 minute) because the smallest time unit in cron is + // minute. This test has been tested locally before committed + @Test + public void commitScheduledYamlTestByCronAndCancel() { + LOGGER.info("commitScheduledYamlTest(every 1 minute by cron) and cancel"); + String outputFileName = + OUTPUT_DIR_PREFIX + File.separator + "export_file_every_1_minute_cron.txt"; + try { + String insertSQL = "insert into scheduleData(key, %s) values(1, 2);"; + String yamlFileName = OUTPUT_DIR_PREFIX + File.separator + "TransformScheduledCron.yaml"; + SessionExecuteSqlResult result = + session.executeSql(String.format(COMMIT_SQL_FORMATTER, yamlFileName)); + long jobId = result.getJobId(); + try { + // add col0 + session.executeSql(String.format(insertSQL, "col0")); + Thread.sleep(62000L); // sleep 62s to make sure next try is triggered. + LOGGER.info("Verifying 0th try..."); + fileResultContains(outputFileName, "col0"); + } finally { + cancelJob(jobId); + assertTrue(Files.deleteIfExists(Paths.get(outputFileName))); + } + } catch (SessionException | InterruptedException e) { + LOGGER.error("Transform: execute fail. Caused by:", e); + fail(); + } catch (IOException e) { + LOGGER.error("Fail to delete result file: {}", outputFileName, e); + fail(); + } + } + + @Ignore // the time system on github action is somehow bugged, thus it cannot be tested in action + // It has passed local test + @Test + public void commitScheduledYamlTestAt10sFromNow() { + LOGGER.info("commitScheduledYamlTest(at 10s from now)"); + String outputFileName = OUTPUT_DIR_PREFIX + File.separator + "export_file_at_10_s.txt"; + try { + String insertSQL = "insert into scheduleData(key, %s) values(1, 2);"; + String yamlFileName = + OUTPUT_DIR_PREFIX + File.separator + "TransformScheduledAt10sFromNow.yaml"; + + LocalDateTime tenSecondsLater = LocalDateTime.now().plusSeconds(10); + String formattedDateTime = + tenSecondsLater.format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")); + appendFile(new File(yamlFileName), "\nschedule: \"at '" + formattedDateTime + "'\""); + + SessionExecuteSqlResult result = + session.executeSql(String.format(COMMIT_SQL_FORMATTER, yamlFileName)); + long jobId = result.getJobId(); + Thread.sleep(2000L); // add new data after 2s, before job is triggered. + // add col0 + session.executeSql(String.format(insertSQL, "col0")); + + Thread.sleep(8000L); // verify result after 10s + fileResultContains(outputFileName, "col0"); + + verifyJobState(jobId); + } catch (SessionException | InterruptedException e) { + LOGGER.error("Transform: execute fail. Caused by:", e); + fail(); + } catch (IOException e) { + LOGGER.error("Fail to add schedule line in yaml", e); + fail(); + } + } + @Test public void commitMultipleSqlStatementsTest() { LOGGER.info("commitMultipleSqlStatementsTest"); @@ -739,4 +889,23 @@ public void cancelJobTest() { fail(); } } + + // file contains + private void fileResultContains(String filename, String content) { + boolean contains = false; + try (BufferedReader reader = new BufferedReader(new FileReader(filename))) { + String line = reader.readLine(); + while (line != null) { + if (line.contains(content)) { + contains = true; + break; + } + line = reader.readLine(); + } + } catch (IOException e) { + LOGGER.error("Verify file export result failed.", e); + fail(); + } + assertTrue(contains); + } } diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/UDFIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/UDFIT.java index 428d1de1a9..fe3808532e 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/UDFIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/UDFIT.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.integration.func.udf; @@ -185,7 +184,7 @@ public void dropTasks() { @Test public void baseTests() { String udtfSQLFormat = "SELECT %s(s1) FROM us.d1 WHERE key < 200;"; - String udafSQLFormat = "SELECT %s(s1) FROM us.d1 OVER (RANGE 50 IN [0, 200));"; + String udafSQLFormat = "SELECT %s(s1) FROM us.d1 OVER WINDOW (size 50 IN [0, 200));"; String udsfSQLFormat = "SELECT %s(s1) FROM us.d1 WHERE key < 50;"; SessionExecuteSqlResult ret = tool.execute(SHOW_FUNCTION_SQL); diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/UDFTestTools.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/UDFTestTools.java index 7964b8736c..3b813e3b01 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/UDFTestTools.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/UDFTestTools.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.func.udf; import static org.junit.Assert.assertEquals; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/mds/ETCDSyncProtocolTest.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/mds/ETCDSyncProtocolTest.java index 1f5a961ae1..e799d78a63 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/mds/ETCDSyncProtocolTest.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/mds/ETCDSyncProtocolTest.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.mds; import cn.edu.tsinghua.iginx.metadata.sync.protocol.SyncProtocol; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/mds/IMetaManagerTest.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/mds/IMetaManagerTest.java index bce3b41008..03ba693280 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/mds/IMetaManagerTest.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/mds/IMetaManagerTest.java @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package cn.edu.tsinghua.iginx.integration.mds; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/mds/RandomUtils.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/mds/RandomUtils.java index f10fce8b63..0d10627071 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/mds/RandomUtils.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/mds/RandomUtils.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.mds; import java.util.Random; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/mds/SyncProtocolTest.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/mds/SyncProtocolTest.java index 1dfaca0716..408ce61d45 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/mds/SyncProtocolTest.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/mds/SyncProtocolTest.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.mds; import static org.junit.Assert.*; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/mds/ZooKeeperSyncProtocolTest.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/mds/ZooKeeperSyncProtocolTest.java index 77ea5e612d..14e2f15cf6 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/mds/ZooKeeperSyncProtocolTest.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/mds/ZooKeeperSyncProtocolTest.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.mds; import cn.edu.tsinghua.iginx.metadata.sync.protocol.NetworkException; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/other/FileLoaderTest.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/other/FileLoaderTest.java index e950b8f790..11fa789805 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/other/FileLoaderTest.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/other/FileLoaderTest.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.other; import static org.junit.Assert.fail; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/other/TimePrecisionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/other/TimePrecisionIT.java index 073f3076c9..af3a78e240 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/other/TimePrecisionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/other/TimePrecisionIT.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.other; import static org.junit.Assert.fail; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/other/UDFPathIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/other/UDFPathIT.java index c4b8d54254..6987222deb 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/other/UDFPathIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/other/UDFPathIT.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.other; import cn.edu.tsinghua.iginx.exception.SessionException; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/regression/MixClusterShowColumnsRegressionTest.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/regression/MixClusterShowColumnsRegressionTest.java index a7dd882c26..c22d1ff3b4 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/regression/MixClusterShowColumnsRegressionTest.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/regression/MixClusterShowColumnsRegressionTest.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.regression; import static org.junit.Assert.assertEquals; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/CombinedInsertTests.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/CombinedInsertTests.java index 753dda308e..74e5507e39 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/CombinedInsertTests.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/CombinedInsertTests.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.tool; import cn.edu.tsinghua.iginx.exception.SessionException; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/ConfLoader.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/ConfLoader.java index 618b6a18e4..b0553a2f6e 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/ConfLoader.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/ConfLoader.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.tool; import cn.edu.tsinghua.iginx.integration.expansion.constant.Constant; @@ -108,6 +126,24 @@ public String getDBCETestWay() { return getTestProperty(DBCE_TEST_WAY, DEFAULT_DBCE_TEST_WAY); } + public List getQueryIds() { + String input = properties.getProperty("query_ids"); + String[] split = input.split(","); + List list = new ArrayList<>(); + for (String s : split) { + list.add(Integer.parseInt(s)); + } + return list; + } + + public int getMaxRepetitionsNum() { + return Integer.parseInt(properties.getProperty("max_repetitions_num")); + } + + public double getRegressionThreshold() { + return Double.parseDouble(properties.getProperty("regression_threshold")); + } + public ConfLoader(String confPath) { this.confPath = confPath; try { diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/DBConf.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/DBConf.java index 30d7fbacaf..e1cccbaf10 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/DBConf.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/DBConf.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.tool; import java.util.HashMap; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/MultiConnection.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/MultiConnection.java index 0c07df7f6f..631f9fd436 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/MultiConnection.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/MultiConnection.java @@ -1,6 +1,24 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.tool; -import static cn.edu.tsinghua.iginx.constant.GlobalConstant.CLEAR_DUMMY_DATA_CAUTION; +import static cn.edu.tsinghua.iginx.constant.GlobalConstant.*; import static cn.edu.tsinghua.iginx.integration.controller.Controller.*; import static org.junit.Assert.fail; @@ -162,6 +180,13 @@ public SessionAggregateQueryDataSet aggregateQuery( return null; } + // downsample query with time interval + public SessionQueryDataSet downsampleQuery( + List paths, AggregateType aggregateType, long precision) throws SessionException { + return downsampleQuery(paths, KEY_MIN_VAL, KEY_MAX_VAL, aggregateType, precision); + } + + // downsample query without time interval public SessionQueryDataSet downsampleQuery( List paths, long startKey, long endKey, AggregateType aggregateType, long precision) throws SessionException { diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/SQLExecutor.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/SQLExecutor.java index 69eb1fb334..256b438748 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/SQLExecutor.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/SQLExecutor.java @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.tool; import static cn.edu.tsinghua.iginx.constant.GlobalConstant.CLEAR_DUMMY_DATA_CAUTION; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/TestUtils.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/TestUtils.java new file mode 100644 index 0000000000..0968c622ae --- /dev/null +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/TestUtils.java @@ -0,0 +1,84 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.integration.tool; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; + +public class TestUtils { + /** + * 比较ResultSet的表格内容是否相等,不比较顺序。 + * + * @param table1 表格1 + * @param table2 表格2 + * @return 是否相等 + */ + public static boolean compareTables(String table1, String table2) { + List> list1 = parseTable(table1); + List> list2 = parseTable(table2); + + list1.sort(new ListComparator()); + list2.sort(new ListComparator()); + + return list1.equals(list2); + } + + /** + * 将ResultSet的表格内容解析为二维字符串列表。 + * + * @param table 表格字符串 + * @return 二维字符串列表 + */ + private static List> parseTable(String table) { + List> list = new ArrayList<>(); + String[] lines = table.split("\n"); + + for (String line : lines) { + if (line.startsWith("|") && line.endsWith("|")) { + String[] values = line.split("\\|"); + List row = new ArrayList<>(); + for (String value : values) { + value = value.trim(); + if (!value.isEmpty()) { + row.add(value); + } + } + if (!row.isEmpty()) { + list.add(row); + } + } + } + + return list; + } + + static class ListComparator implements Comparator> { + @Override + public int compare(List o1, List o2) { + for (int i = 0; i < o1.size(); i++) { + int comparison = o1.get(i).compareTo(o2.get(i)); + if (comparison != 0) { + return comparison; + } + } + return 0; + } + } +} diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/tpch/TPCHDataGeneratorIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/tpch/TPCHDataGeneratorIT.java new file mode 100644 index 0000000000..1472236bde --- /dev/null +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/tpch/TPCHDataGeneratorIT.java @@ -0,0 +1,334 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cn.edu.tsinghua.iginx.integration.tpch; + +import static cn.edu.tsinghua.iginx.integration.tpch.TPCHDataGeneratorIT.FieldType.DATE; +import static cn.edu.tsinghua.iginx.integration.tpch.TPCHDataGeneratorIT.FieldType.NUM; +import static cn.edu.tsinghua.iginx.integration.tpch.TPCHDataGeneratorIT.FieldType.STR; +import static org.junit.Assert.fail; + +import cn.edu.tsinghua.iginx.exception.SessionException; +import cn.edu.tsinghua.iginx.integration.controller.Controller; +import cn.edu.tsinghua.iginx.integration.tool.ConfLoader; +import cn.edu.tsinghua.iginx.session.Session; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.TimeZone; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TPCHDataGeneratorIT { + + private static final Logger LOGGER = LoggerFactory.getLogger(TPCHDataGeneratorIT.class); + + // host info + protected static String defaultTestHost = "127.0.0.1"; + protected static int defaultTestPort = 6888; + protected static String defaultTestUser = "root"; + protected static String defaultTestPass = "root"; + + // .tbl文件所在目录 + static final String DATA_DIR = System.getProperty("user.dir") + "/../tpc/TPC-H V3.0.1/data"; + + // udf文件所在目录 + static final String UDF_DIR = "src/test/resources/tpch/udf/"; + + static final String SHOW_FUNCTION = "SHOW FUNCTIONS;"; + + static final String SINGLE_UDF_REGISTER_SQL = "CREATE FUNCTION %s \"%s\" FROM \"%s\" IN \"%s\";"; + + protected static Session session; + + List queryIds; + + enum FieldType { + NUM, + STR, + DATE + } + + public TPCHDataGeneratorIT() { + ConfLoader conf = new ConfLoader(Controller.CONFIG_FILE); + queryIds = conf.getQueryIds(); + } + + @BeforeClass + public static void setUp() throws SessionException { + session = new Session(defaultTestHost, defaultTestPort, defaultTestUser, defaultTestPass); + session.openSession(); + } + + @AfterClass + public static void close() throws SessionException { + session.closeSession(); + } + + @Before + public void prepare() { + List tableList = + Arrays.asList( + "region", "nation", "supplier", "part", "partsupp", "customer", "orders", "lineitem"); + + List> fieldsList = new ArrayList<>(); + List> typesList = new ArrayList<>(); + + // region表 + fieldsList.add(Arrays.asList("r_regionkey", "r_name", "r_comment")); + typesList.add(Arrays.asList(NUM, STR, STR)); + + // nation表 + fieldsList.add(Arrays.asList("n_nationkey", "n_name", "n_regionkey", "n_comment")); + typesList.add(Arrays.asList(NUM, STR, NUM, STR)); + + // supplier表 + fieldsList.add( + Arrays.asList( + "s_suppkey", + "s_name", + "s_address", + "s_nationkey", + "s_phone", + "s_acctbal", + "s_comment")); + typesList.add(Arrays.asList(NUM, STR, STR, NUM, STR, NUM, STR)); + + // part表 + fieldsList.add( + Arrays.asList( + "p_partkey", + "p_name", + "p_mfgr", + "p_brand", + "p_type", + "p_size", + "p_container", + "p_retailprice", + "p_comment")); + typesList.add(Arrays.asList(NUM, STR, STR, STR, STR, NUM, STR, NUM, STR)); + + // partsupp表 + fieldsList.add( + Arrays.asList("ps_partkey", "ps_suppkey", "ps_availqty", "ps_supplycost", "ps_comment")); + typesList.add(Arrays.asList(NUM, NUM, NUM, NUM, STR)); + + // customer表 + fieldsList.add( + Arrays.asList( + "c_custkey", + "c_name", + "c_address", + "c_nationkey", + "c_phone", + "c_acctbal", + "c_mktsegment", + "c_comment")); + typesList.add(Arrays.asList(NUM, STR, STR, NUM, STR, NUM, STR, STR)); + + // orders表 + fieldsList.add( + Arrays.asList( + "o_orderkey", + "o_custkey", + "o_orderstatus", + "o_totalprice", + "o_orderdate", + "o_orderpriority", + "o_clerk", + "o_shippriority", + "o_comment")); + typesList.add(Arrays.asList(NUM, NUM, STR, NUM, DATE, STR, STR, NUM, STR)); + + // lineitem表 + fieldsList.add( + Arrays.asList( + "l_orderkey", + "l_partkey", + "l_suppkey", + "l_linenumber", + "l_quantity", + "l_extendedprice", + "l_discount", + "l_tax", + "l_returnflag", + "l_linestatus", + "l_shipdate", + "l_commitdate", + "l_receiptdate", + "l_shipinstruct", + "l_shipmode", + "l_comment")); + typesList.add( + Arrays.asList( + NUM, NUM, NUM, NUM, NUM, NUM, NUM, NUM, STR, STR, DATE, DATE, DATE, STR, STR, STR)); + + // 插入数据 + for (int i = 0; i < 8; i++) { + insertTable(tableList.get(i), fieldsList.get(i), typesList.get(i)); + } + + List> UDFInfos = new ArrayList<>(); + UDFInfos.add(Arrays.asList("UDTF", "extractYear", "UDFExtractYear", "udtf_extract_year.py")); + // 注册UDF函数 + for (List UDFInfo : UDFInfos) { + registerUDF(UDFInfo); + } + } + + private void insertTable(String table, List fields, List types) { + StringBuilder builder = new StringBuilder("INSERT INTO "); + builder.append(table); + builder.append("(key, "); + for (String field : fields) { + builder.append(field); + builder.append(", "); + } + builder.setLength(builder.length() - 2); + builder.append(") VALUES "); + String insertPrefix = builder.toString(); + + long count = 0; + try (BufferedReader br = + new BufferedReader(new FileReader(String.format("%s/%s.tbl", DATA_DIR, table)))) { + StringBuilder sb = new StringBuilder(insertPrefix); + String line; + while ((line = br.readLine()) != null) { + String[] items = line.split("\\|"); + sb.append("("); + sb.append(count); // 插入自增key列 + count++; + sb.append(", "); + assert fields.size() == items.length; + for (int i = 0; i < items.length; i++) { + switch (types.get(i)) { + case NUM: + sb.append(items[i]); + sb.append(", "); + break; + case STR: // 字符串类型在外面需要包一层引号 + sb.append("\""); + sb.append(items[i]); + sb.append("\", "); + break; + case DATE: // 日期类型需要转为时间戳 + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + dateFormat.setTimeZone(TimeZone.getTimeZone("GMT")); + long time = dateFormat.parse(items[i]).getTime(); + sb.append(time); + sb.append(", "); + break; + default: + break; + } + } + sb.setLength(sb.length() - 2); + sb.append("), "); + + // 每次最多插入10000条数据 + if (count % 10000 == 0) { + sb.setLength(sb.length() - 2); + sb.append(";"); + session.executeSql(sb.toString()); + sb = new StringBuilder(insertPrefix); + } + } + // 插入剩余数据 + if (sb.length() != insertPrefix.length()) { + sb.setLength(sb.length() - 2); + sb.append(";"); + session.executeSql(sb.toString()); + } + LOGGER.info("Insert {} records into table [{}].", count, table); + } catch (IOException | ParseException | SessionException e) { + LOGGER.error("Insert into table {} fail. Caused by:", table, e); + fail(); + } + } + + private void registerUDF(List UDFInfo) { + String result = ""; + try { + result = session.executeSql(SHOW_FUNCTION).getResultInString(false, ""); + } catch (SessionException e) { + LOGGER.error("Statement: \"{}\" execute fail. Caused by:", SHOW_FUNCTION, e); + fail(); + } + // UDF已注册 + if (result.contains(UDFInfo.get(1))) { + return; + } + File udfFile = new File(UDF_DIR + UDFInfo.get(3)); + String register = + String.format( + SINGLE_UDF_REGISTER_SQL, + UDFInfo.get(0), + UDFInfo.get(1), + UDFInfo.get(2), + udfFile.getAbsolutePath()); + try { + LOGGER.info("Execute register UDF statement: {}", register); + session.executeRegisterTask(register, false); + } catch (SessionException e) { + LOGGER.error("Statement: \"{}\" execute fail. Caused by:", register, e); + fail(); + } + } + + // 插入TPC-H测试中的临时表 + @Test + public void insertTmpTable() { + for (int queryId : queryIds) { + String sqlString = null; + try { + sqlString = + TPCHUtils.readSqlFileAsString("src/test/resources/tpch/queries/q" + queryId + ".sql"); + } catch (IOException e) { + LOGGER.error("Fail to read sql file: q{}.sql. Caused by: ", queryId, e); + fail(); + } + String[] sqls = sqlString.split(";"); + if (sqls.length < 2) { + LOGGER.error("q{}.sql has no ';' in the end. Caused by: ", queryId); + fail(); + } else if (sqls.length == 2) { + // 只有一条查询语句,无需插入临时表 + continue; + } + // 处理插入临时表语句 + for (int i = 0; i < sqls.length - 2; i++) { + String sql = sqls[i] + ";"; + try { + session.executeSql(sql); + } catch (SessionException e) { + LOGGER.error("Statement: \"{}\" execute fail. Caused by:", sql, e); + fail(); + } + } + } + } +} diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/tpch/TPCHRegressionMainIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/tpch/TPCHRegressionMainIT.java new file mode 100644 index 0000000000..57b695368c --- /dev/null +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/tpch/TPCHRegressionMainIT.java @@ -0,0 +1,117 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package cn.edu.tsinghua.iginx.integration.tpch; + +import cn.edu.tsinghua.iginx.exception.SessionException; +import cn.edu.tsinghua.iginx.integration.controller.Controller; +import cn.edu.tsinghua.iginx.integration.tool.ConfLoader; +import cn.edu.tsinghua.iginx.session.Session; +import java.util.ArrayList; +import java.util.List; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TPCHRegressionMainIT { + + private static final Logger LOGGER = LoggerFactory.getLogger(TPCHRegressionMainIT.class); + + // host info + protected static String defaultTestHost = "127.0.0.1"; + protected static int defaultTestPort = 6888; + protected static String defaultTestUser = "root"; + protected static String defaultTestPass = "root"; + + protected static Session session; + + static final String FAILED_QUERY_ID_PATH = + "src/test/resources/tpch/runtimeInfo/failedQueryIds.txt"; + + static final String ITERATION_TIMES_PATH = + "src/test/resources/tpch/runtimeInfo/iterationTimes.txt"; + + static final String MAIN_TIME_COSTS_PATH = "src/test/resources/tpch/runtimeInfo/oldTimeCosts.txt"; + + // 最大重复测试次数 + int MAX_REPETITIONS_NUM; + + List queryIds; + + // 当前查询次数 + int iterationTimes; + + // 是否需要验证正确性 + boolean needValidate; + + public TPCHRegressionMainIT() { + ConfLoader conf = new ConfLoader(Controller.CONFIG_FILE); + List lines = TPCHUtils.getLinesFromFile(ITERATION_TIMES_PATH); + iterationTimes = Integer.parseInt(lines.get(0)); + if (iterationTimes == 1) { + queryIds = conf.getQueryIds(); + } else { + lines = TPCHUtils.getLinesFromFile(FAILED_QUERY_ID_PATH); + queryIds = new ArrayList<>(); + for (String line : lines) { + queryIds.add(Integer.parseInt(line)); + } + } + // 第一次查询需要验证查询结果正确性 + needValidate = iterationTimes == 1; + MAX_REPETITIONS_NUM = conf.getMaxRepetitionsNum(); + } + + @BeforeClass + public static void setUp() throws SessionException { + session = new Session(defaultTestHost, defaultTestPort, defaultTestUser, defaultTestPass); + session.openSession(); + } + + @AfterClass + public static void tearDown() throws SessionException { + session.closeSession(); + } + + @Test + public void testMainBranch() { + if (queryIds.isEmpty()) { + LOGGER.info("No query remain, skip test main branch."); + return; + } + LOGGER.info("QueryIds remain: {}", queryIds); + if (iterationTimes > MAX_REPETITIONS_NUM) { + LOGGER.error( + "Repeatedly executed query more than {} times, test failed.", MAX_REPETITIONS_NUM); + Assert.fail(); + } + + List> timeCosts = TPCHUtils.readTimeCostsFromFile(MAIN_TIME_COSTS_PATH); + for (int queryId : queryIds) { + long timeCost = TPCHUtils.executeTPCHQuery(session, queryId, needValidate); + timeCosts.get(queryId - 1).add(timeCost); + System.out.printf( + "Successfully execute TPC-H query %d in main branch in iteration %d, time cost: %dms%n", + queryId, iterationTimes, timeCost); + } + TPCHUtils.clearAndRewriteTimeCostsToFile(timeCosts, MAIN_TIME_COSTS_PATH); + } +} diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/tpch/TPCHRegressionNewIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/tpch/TPCHRegressionNewIT.java new file mode 100644 index 0000000000..9c2bc79f13 --- /dev/null +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/tpch/TPCHRegressionNewIT.java @@ -0,0 +1,239 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cn.edu.tsinghua.iginx.integration.tpch; + +import cn.edu.tsinghua.iginx.exception.SessionException; +import cn.edu.tsinghua.iginx.integration.controller.Controller; +import cn.edu.tsinghua.iginx.integration.tool.ConfLoader; +import cn.edu.tsinghua.iginx.session.Session; +import java.io.BufferedWriter; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TPCHRegressionNewIT { + + private static final Logger LOGGER = LoggerFactory.getLogger(TPCHRegressionNewIT.class); + + // host info + protected static String defaultTestHost = "127.0.0.1"; + protected static int defaultTestPort = 6888; + protected static String defaultTestUser = "root"; + protected static String defaultTestPass = "root"; + + protected static Session session; + + static final String FAILED_QUERY_ID_PATH = + "src/test/resources/tpch/runtimeInfo/failedQueryIds.txt"; + + static final String ITERATION_TIMES_PATH = + "src/test/resources/tpch/runtimeInfo/iterationTimes.txt"; + + static final String MAIN_TIME_COSTS_PATH = "src/test/resources/tpch/runtimeInfo/oldTimeCosts.txt"; + + static final String NEW_TIME_COSTS_PATH = "src/test/resources/tpch/runtimeInfo/newTimeCosts.txt"; + + static final String STATUS_PATH = "src/test/resources/tpch/runtimeInfo/status.txt"; + + // 最大重复测试次数 + int MAX_REPETITIONS_NUM; + + // 回归阈值 + double REGRESSION_THRESHOLD; + + List queryIds; + + // 当前查询次数 + int iterationTimes; + + // 是否需要验证正确性 + boolean needValidate; + + public TPCHRegressionNewIT() { + ConfLoader conf = new ConfLoader(Controller.CONFIG_FILE); + List lines = TPCHUtils.getLinesFromFile(ITERATION_TIMES_PATH); + iterationTimes = Integer.parseInt(lines.get(0)); + if (iterationTimes == 1) { + queryIds = conf.getQueryIds(); + } else { + lines = TPCHUtils.getLinesFromFile(FAILED_QUERY_ID_PATH); + queryIds = new ArrayList<>(); + for (String line : lines) { + queryIds.add(Integer.parseInt(line)); + } + } + // 第一次查询需要验证查询结果正确性 + needValidate = iterationTimes == 1; + MAX_REPETITIONS_NUM = conf.getMaxRepetitionsNum(); + REGRESSION_THRESHOLD = conf.getRegressionThreshold(); + } + + @BeforeClass + public static void setUp() throws SessionException { + session = new Session(defaultTestHost, defaultTestPort, defaultTestUser, defaultTestPass); + session.openSession(); + } + + @AfterClass + public static void tearDown() throws SessionException { + session.closeSession(); + } + + @Test + public void testNewBranch() { + if (queryIds.isEmpty()) { + LOGGER.info("No query remain, skip test new branch."); + return; + } + LOGGER.info("QueryIds remain: {}", queryIds); + if (iterationTimes > MAX_REPETITIONS_NUM) { + LOGGER.error( + "Repeatedly executed query more than {} times, test failed.", MAX_REPETITIONS_NUM); + Assert.fail(); + } + + List failedQueryIds = new ArrayList<>(); + List> oldTimeCosts = TPCHUtils.readTimeCostsFromFile(MAIN_TIME_COSTS_PATH); + List> newTimeCosts = TPCHUtils.readTimeCostsFromFile(NEW_TIME_COSTS_PATH); + double ratio = 1 + REGRESSION_THRESHOLD; + for (int queryId : queryIds) { + long timeCost = TPCHUtils.executeTPCHQuery(session, queryId, needValidate); + newTimeCosts.get(queryId - 1).add(timeCost); + LOGGER.info( + "Successfully execute TPC-H query {} in new branch in iteration {}, time cost: {}ms", + queryId, + iterationTimes, + timeCost); + + // 新旧分支查询次数不相同 + if (oldTimeCosts.get(queryId - 1).size() != newTimeCosts.get(queryId - 1).size()) { + LOGGER.error( + "Query {} run {} times in main branch, but {} times in new branch, please check.", + queryId, + oldTimeCosts.get(queryId - 1).size(), + newTimeCosts.get(queryId - 1).size()); + Assert.fail(); + } + + // 与主分支运行结果进行比较 + long oldTimeCostMedian = getMedian(oldTimeCosts.get(queryId - 1)); + long newTimeCostMedian = getMedian(newTimeCosts.get(queryId - 1)); + if (oldTimeCostMedian * ratio < newTimeCostMedian) { + if (iterationTimes >= MAX_REPETITIONS_NUM) { + LOGGER.error( + "Repeatedly executed query {} more than {} times, old time costs' median: {}ms, new time costs' median: {}ms, test failed.", + queryId, + MAX_REPETITIONS_NUM, + oldTimeCostMedian, + newTimeCostMedian); + Assert.fail(); + } + failedQueryIds.add(queryId); + continue; + } + LOGGER.info( + "Query {} passed after {} times' iterations, old time costs' median: {}ms, new time costs' median: {}ms.", + queryId, + iterationTimes, + oldTimeCostMedian, + newTimeCostMedian); + } + + if (failedQueryIds.isEmpty()) { + writeOKtoFile(); + } + + TPCHUtils.clearAndRewriteTimeCostsToFile(newTimeCosts, NEW_TIME_COSTS_PATH); + clearAndRewriteFailedQueryIdsToFile(failedQueryIds); + updateIterationTimes(); + } + + private long getMedian(List array) { + Collections.sort(array); + int middle = array.size() / 2; + return array.size() % 2 == 0 + ? (long) ((array.get(middle - 1) + array.get(middle)) / 2.0) + : array.get(middle); + } + + private void writeOKtoFile() { + try (FileWriter fileWriter = new FileWriter(STATUS_PATH); + BufferedWriter bufferedWriter = new BufferedWriter(fileWriter)) { + // 清空文件内容 + fileWriter.write(""); + fileWriter.flush(); + // 重新写入内容 + bufferedWriter.write("ok"); + } catch (IOException e) { + LOGGER.error("Write to file {} fail. Caused by:", STATUS_PATH, e); + Assert.fail(); + } + } + + private void clearAndRewriteFailedQueryIdsToFile(List failedQueryIds) { + Path path = Paths.get(FAILED_QUERY_ID_PATH); + if (!Files.exists(path)) { + try { + Files.createFile(path); + } catch (IOException e) { + LOGGER.error("Failed to create file {}. Caused by: ", FAILED_QUERY_ID_PATH, e); + Assert.fail(); + } + } + try (FileWriter fileWriter = new FileWriter(String.valueOf(path)); + BufferedWriter bufferedWriter = new BufferedWriter(fileWriter)) { + // 清空文件内容 + fileWriter.write(""); + fileWriter.flush(); + // 重新写入内容 + for (int failedQueryId : failedQueryIds) { + bufferedWriter.write(String.valueOf(failedQueryId)); + bufferedWriter.newLine(); + } + } catch (IOException e) { + LOGGER.error("Write to file {} fail. Caused by:", path.getFileName(), e); + Assert.fail(); + } + } + + private void updateIterationTimes() { + iterationTimes++; + try (FileWriter fileWriter = new FileWriter(ITERATION_TIMES_PATH); + BufferedWriter bufferedWriter = new BufferedWriter(fileWriter)) { + // 清空文件内容 + fileWriter.write(""); + fileWriter.flush(); + // 重新写入内容 + bufferedWriter.write(String.valueOf(iterationTimes)); + } catch (IOException e) { + LOGGER.error("Write to file {} fail. Caused by:", ITERATION_TIMES_PATH, e); + Assert.fail(); + } + } +} diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/tpch/TPCHUtils.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/tpch/TPCHUtils.java new file mode 100644 index 0000000000..d22b5a0e9a --- /dev/null +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/tpch/TPCHUtils.java @@ -0,0 +1,188 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cn.edu.tsinghua.iginx.integration.tpch; + +import cn.edu.tsinghua.iginx.exception.SessionException; +import cn.edu.tsinghua.iginx.session.Session; +import cn.edu.tsinghua.iginx.session.SessionExecuteSqlResult; +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Scanner; +import java.util.stream.Collectors; +import org.junit.Assert; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TPCHUtils { + + private static final Logger LOGGER = LoggerFactory.getLogger(TPCHUtils.class); + + public static String readSqlFileAsString(String filePath) throws IOException { + StringBuilder contentBuilder = new StringBuilder(); + try (BufferedReader br = new BufferedReader(new FileReader(filePath))) { + String line; + while ((line = br.readLine()) != null) { + contentBuilder.append(line).append("\n"); + } + } + return contentBuilder.toString(); + } + + public static List> readTimeCostsFromFile(String filename) { + List lines = getLinesFromFile(filename); + assert lines.size() == 22; + + // TPC-H一共有22条查询 + List> timeCosts = new ArrayList<>(22); + for (int i = 0; i < 22; i++) { + List tmp = new ArrayList<>(); + String[] lineArray = lines.get(i).split(","); + for (String str : lineArray) { + if (!str.isEmpty()) { + tmp.add(Long.parseLong(str)); + } + } + timeCosts.add(tmp); + } + return timeCosts; + } + + public static void clearAndRewriteTimeCostsToFile(List> timeCosts, String filename) { + try (FileWriter fileWriter = new FileWriter(filename); + BufferedWriter bufferedWriter = new BufferedWriter(fileWriter)) { + // 清空文件内容 + fileWriter.write(""); + fileWriter.flush(); + // 重新写入内容 + for (List timeCost : timeCosts) { + bufferedWriter.write( + timeCost.stream().map(String::valueOf).collect(Collectors.joining(","))); + bufferedWriter.newLine(); + } + } catch (IOException e) { + LOGGER.error("Write to file {} fail. Caused by:", filename, e); + Assert.fail(); + } + } + + public static List getLinesFromFile(String fileName) { + Path filePath = Paths.get(fileName); + List lines = null; + try { + lines = Files.readAllLines(filePath); + } catch (IOException e) { + LOGGER.error("Read file {} fail. Caused by:", filePath, e); + Assert.fail(); + } + return lines; + } + + public static long executeTPCHQuery(Session session, int queryId, boolean needValidate) { + String sqlString = null; + try { + sqlString = + TPCHUtils.readSqlFileAsString("src/test/resources/tpch/queries/q" + queryId + ".sql"); + } catch (IOException e) { + LOGGER.error("Fail to read sql file: q{}.sql. Caused by: ", queryId, e); + Assert.fail(); + } + String[] sqls = sqlString.split(";"); + String sql = sqls[sqls.length - 2] + ";"; + SessionExecuteSqlResult result = null; + long startTime = System.currentTimeMillis(); + try { + result = session.executeSql(sql); + } catch (SessionException e) { + LOGGER.error("Statement: \"{}\" execute fail. Caused by:", sql, e); + Assert.fail(); + } + long cost = System.currentTimeMillis() - startTime; + if (needValidate) { + validate(result, queryId); + } + return cost; + } + + private static void validate(SessionExecuteSqlResult result, int queryId) { + List> values = result.getValues(); + List> answers = csvReader("src/test/resources/tpch/sf0.1/q" + queryId + ".csv"); + if (values.size() != answers.size()) { + System.out.println("values.size() = " + values.size()); + System.out.println("answers.size() = " + answers.size()); + throw new RuntimeException("size not equal"); + } + for (int i = 0; i < values.size(); i++) { + for (int j = 0; j < values.get(i).size(); j++) { + if (result.getPaths().get(j).contains("address") + || result.getPaths().get(j).contains("comment") + || result.getPaths().get(j).contains("orderdate")) { + // TODO change unix time to date + continue; + } + // if only contains number and dot, then parse to double + if (values.get(i).get(j).toString().matches("-?[0-9]+.*[0-9]*")) { + double number = Double.parseDouble(values.get(i).get(j).toString()); + double answerNumber = Double.parseDouble(answers.get(i).get(j)); + if (answerNumber - number >= 1e-3 || number - answerNumber >= 1e-3) { + System.out.println("Number: " + number); + System.out.println("Answer number: " + answerNumber); + } + assert answerNumber - number < 1e-3 && number - answerNumber < 1e-3; + } else { + String resultString = new String((byte[]) values.get(i).get(j), StandardCharsets.UTF_8); + String answerString = answers.get(i).get(j); + if (!resultString.equals(answerString)) { + System.out.println("Result string: " + resultString); + System.out.println("Answer string: " + answerString); + } + assert resultString.equals(answerString); + } + } + } + } + + private static List> csvReader(String filePath) { + List> data = new ArrayList<>(); + boolean skipHeader = true; + try (Scanner scanner = new Scanner(Paths.get(filePath))) { + while (scanner.hasNextLine()) { + String line = scanner.nextLine(); + if (skipHeader) { + skipHeader = false; + continue; + } + List row = Arrays.asList(line.split("\\|")); + data.add(row); + } + } catch (IOException e) { + LOGGER.error("Read file {} fail. Caused by:", filePath, e); + Assert.fail(); + } + return data; + } +} diff --git a/test/src/test/resources/pySessionIT/tests.py b/test/src/test/resources/pySessionIT/tests.py index 983faea032..535ac3e75c 100644 --- a/test/src/test/resources/pySessionIT/tests.py +++ b/test/src/test/resources/pySessionIT/tests.py @@ -1,21 +1,20 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . # - # 无法单独执行,用来测试PySessionIT import sys, traceback sys.path.append('../session_py/') # 将上一级目录添加到Python模块搜索路径中 @@ -265,9 +264,12 @@ def deleteRow(self): return retStr def downsampleQuery(self): + import pandas as pd try: dataset = self.session.downsample_query(["test.*"], start_time=0, end_time=10, type=AggregateType.COUNT, precision=3) + pd.set_option('display.max_columns', None) + pd.set_option('display.max_rows', None) retStr = str(dataset.to_df()) + "\n" except Exception as e: print(e) @@ -276,6 +278,19 @@ def downsampleQuery(self): return retStr + def downsampleQueryNoInterval(self): + import pandas as pd + try: + dataset = self.session.downsample_query_no_interval(["test.*"], type=AggregateType.COUNT, + precision=3) + pd.set_option('display.max_columns', None) + pd.set_option('display.max_rows', None) + retStr = str(dataset.to_df()) + "\n" + except Exception as e: + print(e) + exit(1) + return retStr + def exportToFile(self): try: # 将数据存入csv @@ -379,6 +394,35 @@ def insertBaseDataset(self): return "" + def insertDF(self): + try: + import pandas as pd + data = { + 'key': list(range(10, 20)), + 'value1': ['A']*10, + 'value2': [1.1]*10 + } + + df = pd.DataFrame(data) + self.session.insert_df(df, "dftestdata") + data = { + 'key': list(range(10, 20)), + 'dftestdata.value3': ['B']*10, + 'dftestdata.value4': [2.2]*10 + } + + df = pd.DataFrame(data) + self.session.insert_df(df) + + dataset = self.session.query(["dftestdata.*"], 0, 1000) + pd.set_option('display.max_columns', None) + pd.set_option('display.max_rows', None) + retStr = dataset.to_df().to_string(index=False) + "\n" + return retStr + except Exception as e: + print(e) + exit(1) + def lastQuery(self): retStr = "" # 获取部分序列的最后一个数据点 diff --git a/test/src/test/resources/testConfig.properties b/test/src/test/resources/testConfig.properties index b6ac7558ae..6d68075533 100644 --- a/test/src/test/resources/testConfig.properties +++ b/test/src/test/resources/testConfig.properties @@ -33,7 +33,7 @@ PostgreSQL_mock=127.0.0.1#5432#relational#engine=postgresql#username=postgres#pa MongoDB_mock=127.0.0.1#27017#MongoDB#has_data=false Redis_mock=127.0.0.1#6379#Redis#has_data=false#is_read_only=false#timeout=5000 FileSystem_mock=127.0.0.1#6667#FileSystem#dir=test/iginx_mn#dummy_dir=/path/to/your/data#iginx_port=6888#chunk_size_in_bytes=1048576#memory_pool_size=100#has_data=false#is_read_only=false -MySQL_mock=127.0.0.1#3306#relational#engine=mysql#has_data=false#username=root#meta_properties_path=../dataSources/relational/src/main/resources/mysql-meta-template.properties +MySQL_mock=127.0.0.1#3306#relational#engine=mysql#has_data=false#username=root#meta_properties_path=../dataSource/relational/src/main/resources/mysql-meta-template.properties # class name for each DB IoTDB12_class=cn.edu.tsinghua.iginx.iotdb.IoTDBStorage @@ -71,3 +71,8 @@ stand_alone_DB=IoTDB12 # Local test DB-CE is_scaling=false DBCE_test_way=oriHasDataExpHasData + +# TPC-H test +query_ids=1,2,3,5,6,9,10,16,17,18,19,20 +max_repetitions_num=10 +regression_threshold=0.3 diff --git a/test/src/test/resources/tpch/queries/q1.sql b/test/src/test/resources/tpch/queries/q1.sql new file mode 100644 index 0000000000..c77305f28d --- /dev/null +++ b/test/src/test/resources/tpch/queries/q1.sql @@ -0,0 +1,31 @@ +select + lineitem.l_returnflag as l_returnflag, + lineitem.l_linestatus as l_linestatus, + sum(lineitem.l_quantity) as sum_qty, + sum(lineitem.l_extendedprice) as sum_base_price, + sum(tmp1) as sum_disc_price, + sum(tmp2) as sum_charge, + avg(lineitem.l_quantity) as avg_qty, + avg(lineitem.l_extendedprice) as avg_price, + avg(lineitem.l_discount) as avg_disc, + count(lineitem.l_returnflag) as count_order +from ( + select + l_returnflag, + l_linestatus, + l_quantity, + l_extendedprice, + l_discount, + l_extendedprice * (1 - l_discount) as tmp1, + l_extendedprice * (1 - l_discount) * (1 + l_tax) as tmp2 + from + lineitem + where + lineitem.l_shipdate <= 904694400000 + ) +group by + lineitem.l_returnflag, + lineitem.l_linestatus +order by + lineitem.l_returnflag, + lineitem.l_linestatus; \ No newline at end of file diff --git a/test/src/test/resources/tpch/queries/q10.sql b/test/src/test/resources/tpch/queries/q10.sql new file mode 100644 index 0000000000..2fdd79af91 --- /dev/null +++ b/test/src/test/resources/tpch/queries/q10.sql @@ -0,0 +1,51 @@ +select + customer.c_custkey, + customer.c_name, + revenue, + customer.c_acctbal, + nation.n_name, + customer.c_address, + customer.c_phone, + customer.c_comment +from ( + select + customer.c_custkey, + customer.c_name, + sum(tmp) as revenue, + customer.c_acctbal, + nation.n_name, + customer.c_address, + customer.c_phone, + customer.c_comment + from ( + select + customer.c_custkey, + customer.c_name, + lineitem.l_extendedprice * (1 - lineitem.l_discount) as tmp, + customer.c_acctbal, + nation.n_name, + customer.c_address, + customer.c_phone, + customer.c_comment + from + customer + join orders on customer.c_custkey = orders.o_custkey + join lineitem on lineitem.l_orderkey = orders.o_orderkey + join nation on customer.c_nationkey = nation.n_nationkey + where + orders.o_orderdate >= 749404800000 + and orders.o_orderdate < 757353600000 + and lineitem.l_returnflag = 'R' + ) + group by + customer.c_custkey, + customer.c_name, + customer.c_acctbal, + customer.c_phone, + nation.n_name, + customer.c_address, + customer.c_comment + ) +order by + revenue desc +limit 20; \ No newline at end of file diff --git a/test/src/test/resources/tpch/queries/q13.sql b/test/src/test/resources/tpch/queries/q13.sql new file mode 100644 index 0000000000..c40d34162c --- /dev/null +++ b/test/src/test/resources/tpch/queries/q13.sql @@ -0,0 +1,25 @@ +select + c_count, + custdist +from ( + select + c_count, + count(c_custkey) as custdist + from ( + select + customer.c_custkey as c_custkey, + count(orders.o_orderkey) as c_count + from + customer + left outer join orders + on customer.c_custkey = orders.o_custkey + and !(orders.o_comment like '.*pending.*') + group by + customer.c_custkey + ) + group by + c_count + ) +order by + custdist, + c_count desc; \ No newline at end of file diff --git a/test/src/test/resources/tpch/queries/q16.sql b/test/src/test/resources/tpch/queries/q16.sql new file mode 100644 index 0000000000..a19922c7c9 --- /dev/null +++ b/test/src/test/resources/tpch/queries/q16.sql @@ -0,0 +1,49 @@ +select + p_brand, + p_type, + p_size, + supplier_cnt +from ( + select + part.p_brand as p_brand, + part.p_type as p_type, + part.p_size as p_size, + count(distinct partsupp.ps_suppkey) as supplier_cnt + from + partsupp + join part on part.p_partkey = partsupp.ps_partkey + where + part.p_brand <> 'Brand#45' + and part.p_partkey not in( + select p_partkey from part + where part.p_type like '.*MEDIUM POLISHED.*' + ) + and ( + part.p_size = 3 + or part.p_size = 9 + or part.p_size = 14 + or part.p_size = 19 + or part.p_size = 23 + or part.p_size = 36 + or part.p_size = 45 + or part.p_size = 49 + ) + and partsupp.ps_suppkey not in ( + select + s_suppkey + from + supplier + where + supplier.s_comment like '.*Customer.*Complaints.*' + ) + group by + part.p_brand, + part.p_type, + part.p_size + order by + part.p_brand, + part.p_type, + part.p_size + ) +order by + supplier_cnt desc; \ No newline at end of file diff --git a/test/src/test/resources/tpch/queries/q17.sql b/test/src/test/resources/tpch/queries/q17.sql new file mode 100644 index 0000000000..c573ec0e2d --- /dev/null +++ b/test/src/test/resources/tpch/queries/q17.sql @@ -0,0 +1,29 @@ +insert into tmpTableB(key, p_partkey, val) values ( + select + part.p_partkey, + 0.2 * tmp + from ( + select + part.p_partkey, + avg(lineitem.l_quantity) as tmp + from + lineitem + join part on lineitem.l_partkey = part.p_partkey + group by part.p_partkey + ) +); + +select + tmp2 / 7 as avg_yearly +from ( + select + sum(lineitem.l_extendedprice) as tmp2 + from + lineitem + join part on part.p_partkey = lineitem.l_partkey + join tmpTableB on tmpTableB.p_partkey = lineitem.l_partkey + where + part.p_brand = 'Brand#23' + and part.p_container = 'MED BOX' + and lineitem.l_quantity < tmpTableB.val + ); \ No newline at end of file diff --git a/test/src/test/resources/tpch/queries/q18.sql b/test/src/test/resources/tpch/queries/q18.sql new file mode 100644 index 0000000000..aaa10ae7b3 --- /dev/null +++ b/test/src/test/resources/tpch/queries/q18.sql @@ -0,0 +1,35 @@ +select + customer.c_name, + customer.c_custkey, + orders.o_orderkey, + orders.o_orderdate, + orders.o_totalprice, + sum(lineitem.l_quantity) +from + customer + join orders on customer.c_custkey = orders.o_custkey + join lineitem on orders.o_orderkey = lineitem.l_orderkey +where + orders.o_orderkey in ( + select + lineitem.l_orderkey + from ( + select + l_orderkey, + sum(l_quantity) + from + lineitem + group by + l_orderkey + having + sum(lineitem.l_quantity) > 300 + ) + ) +group by + customer.c_name, + customer.c_custkey, + orders.o_orderkey, + orders.o_orderdate, + orders.o_totalprice +order by + orders.o_totalprice desc; \ No newline at end of file diff --git a/test/src/test/resources/tpch/queries/q19.sql b/test/src/test/resources/tpch/queries/q19.sql new file mode 100644 index 0000000000..0af851b6a0 --- /dev/null +++ b/test/src/test/resources/tpch/queries/q19.sql @@ -0,0 +1,48 @@ +select + sum(tmp) as revenue +from ( + select + lineitem.l_extendedprice * (1 - lineitem.l_discount) as tmp + from + lineitem + join part on part.p_partkey = lineitem.l_partkey + where ( + part.p_brand = 'Brand#12' + and ( + part.p_container = 'SM CASE' + or part.p_container = 'SM BOX' + or part.p_container = 'SM PACK' + or part.p_container = 'SM PKG' + ) + and lineitem.l_quantity >= 1 and lineitem.l_quantity <= 11 + and part.p_size >= 1 and part.p_size <= 5 + and (lineitem.l_shipmode = 'AIR' or lineitem.l_shipmode = 'AIR REG') + and lineitem.l_shipinstruct = 'DELIVER IN PERSON' + ) + or ( + part.p_brand = 'Brand#23' + and ( + part.p_container = 'MED PKG' + or part.p_container = 'MED BOX' + or part.p_container = 'MED BAG' + or part.p_container = 'MED PACK' + ) + and lineitem.l_quantity >= 10 and lineitem.l_quantity <= 20 + and part.p_size >= 1 and part.p_size <= 10 + and (lineitem.l_shipmode = 'AIR' or lineitem.l_shipmode = 'AIR REG') + and lineitem.l_shipinstruct = 'DELIVER IN PERSON' + ) + or ( + part.p_brand = 'Brand#34' + and ( + part.p_container = 'LG PACK' + or part.p_container = 'LG BOX' + or part.p_container = 'LG CASE' + or part.p_container = 'LG PKG' + ) + and lineitem.l_quantity >= 20 and lineitem.l_quantity <= 30 + and part.p_size >= 1 and part.p_size <= 15 + and (lineitem.l_shipmode = 'AIR' or lineitem.l_shipmode = 'AIR REG') + and lineitem.l_shipinstruct = 'DELIVER IN PERSON' + ) + ); \ No newline at end of file diff --git a/test/src/test/resources/tpch/queries/q2.sql b/test/src/test/resources/tpch/queries/q2.sql new file mode 100644 index 0000000000..a5b56e3267 --- /dev/null +++ b/test/src/test/resources/tpch/queries/q2.sql @@ -0,0 +1,44 @@ +insert into tmpTable(key, p_key, minCost) values ( +select + part.p_partkey as p_key, + min(partsupp.ps_supplycost) as minCost +from + partsupp + join supplier on supplier.s_suppkey = partsupp.ps_suppkey + join nation on supplier.s_nationkey = nation.n_nationkey + join region on nation.n_regionkey = region.r_regionkey + join part on part.p_partkey = partsupp.ps_partkey +where + region.r_name = 'EUROPE' + and part.p_size = 15 + and part.p_type like "^.*BRASS" +group by part.p_partkey +); + +select s_acctbal, s_name, n_name, p_partkey, p_mfgr, s_address, s_phone, s_comment from + (select + supplier.s_acctbal as s_acctbal, + supplier.s_name as s_name, + nation.n_name as n_name, + part.p_partkey as p_partkey, + part.p_mfgr as p_mfgr, + supplier.s_address as s_address, + supplier.s_phone as s_phone, + supplier.s_comment as s_comment + from + part + join partsupp on part.p_partkey = partsupp.ps_partkey + join supplier on supplier.s_suppkey = partsupp.ps_suppkey + join nation on supplier.s_nationkey = nation.n_nationkey + join region on nation.n_regionkey = region.r_regionkey + join tmpTable on tmpTable.p_key = part.p_partkey and partsupp.ps_supplycost = tmpTable.minCost + where + part.p_size = 15 + and region.r_name = 'EUROPE' + and part.p_type like "^.*BRASS" + order by + nation.n_name, + supplier.s_name, + part.p_partkey + ) +order by s_acctbal desc; \ No newline at end of file diff --git a/test/src/test/resources/tpch/queries/q20.sql b/test/src/test/resources/tpch/queries/q20.sql new file mode 100644 index 0000000000..4671b79f8a --- /dev/null +++ b/test/src/test/resources/tpch/queries/q20.sql @@ -0,0 +1,45 @@ +insert into tmpTableA(key, partkey, suppkey, val) values ( + select + partkey, + suppkey, + 0.5 * tmp from( + select + l_partkey as partkey, + l_suppkey as suppkey, + sum(l_quantity) as tmp + from + lineitem + where + lineitem.l_shipdate >= 757353600000 + and lineitem.l_shipdate < 788889600000 + group by l_partkey, l_suppkey + ) +); + +select + supplier.s_name, + supplier.s_address +from + supplier + join nation on supplier.s_nationkey = nation.n_nationkey +where + supplier.s_suppkey in ( + select + partsupp.ps_suppkey + from + partsupp + join tmpTableA on tmpTableA.suppkey = partsupp.ps_suppkey and tmpTableA.partkey = partsupp.ps_partkey + where + partsupp.ps_partkey in ( + select + p_partkey + from + part + where + part.p_name like 'forest.*' + ) + and partsupp.ps_availqty > tmpTableA.val + ) + and nation.n_name = 'CANADA' +order by + supplier.s_name; \ No newline at end of file diff --git a/test/src/test/resources/tpch/queries/q3.sql b/test/src/test/resources/tpch/queries/q3.sql new file mode 100644 index 0000000000..81a232a712 --- /dev/null +++ b/test/src/test/resources/tpch/queries/q3.sql @@ -0,0 +1,31 @@ +select l_orderkey, + revenue, + o_orderdate, + o_shippriority from + (select + l_orderkey, + o_orderdate, + o_shippriority, + sum(tmp) as revenue + from( + select + lineitem.l_extendedprice * (1 - lineitem.l_discount) as tmp, + lineitem.l_orderkey as l_orderkey, + orders.o_orderdate as o_orderdate, + orders.o_shippriority as o_shippriority + from + customer + join orders on customer.c_custkey = orders.o_custkey + join lineitem on lineitem.l_orderkey = orders.o_orderkey + where + customer.c_mktsegment = 'BUILDING' + and orders.o_orderdate < 795196800000 + and lineitem.l_shipdate > 795225600000 + ) + group by + l_orderkey, + o_orderdate, + o_shippriority + ) +order by revenue desc + limit 10; \ No newline at end of file diff --git a/test/src/test/resources/tpch/queries/q5.sql b/test/src/test/resources/tpch/queries/q5.sql new file mode 100644 index 0000000000..197cced143 --- /dev/null +++ b/test/src/test/resources/tpch/queries/q5.sql @@ -0,0 +1,28 @@ +select + nation.n_name, + revenue +from ( + select + nation.n_name, + sum(tmp) as revenue + from ( + select + nation.n_name, + lineitem.l_extendedprice * (1 - lineitem.l_discount) as tmp + from + customer + join orders on customer.c_custkey = orders.o_custkey + join lineitem on lineitem.l_orderkey = orders.o_orderkey + join supplier on lineitem.l_suppkey = supplier.s_suppkey and customer.c_nationkey = supplier.s_nationkey + join nation on supplier.s_nationkey = nation.n_nationkey + join region on nation.n_regionkey = region.r_regionkey + where + region.r_name = "ASIA" + and orders.o_orderdate >= 757353600000 + and orders.o_orderdate < 788889600000 + ) + group by + nation.n_name + ) +order by + revenue desc; \ No newline at end of file diff --git a/test/src/test/resources/tpch/queries/q6.sql b/test/src/test/resources/tpch/queries/q6.sql new file mode 100644 index 0000000000..3e9c5a4b65 --- /dev/null +++ b/test/src/test/resources/tpch/queries/q6.sql @@ -0,0 +1,12 @@ +select sum(tmp) from ( + select + l_extendedprice * l_discount as tmp + from + lineitem + where + lineitem.l_shipdate >= 757353600000 + and lineitem.l_shipdate < 788889600000 + and lineitem.l_discount >= 0.05 + and lineitem.l_discount <= 0.07 + and lineitem.l_quantity < 24 + ); diff --git a/test/src/test/resources/tpch/queries/q9.sql b/test/src/test/resources/tpch/queries/q9.sql new file mode 100644 index 0000000000..311fb02ad7 --- /dev/null +++ b/test/src/test/resources/tpch/queries/q9.sql @@ -0,0 +1,32 @@ +insert into tmpTableC(key, orderkey, year) values ( + select o_orderkey, extractYear(o_orderdate) from orders +); + +select * from ( + select + nation, + o_year, + sum(amount) as sum_profit + from ( + select + nation.n_name as nation, + tmpTableC.year as o_year, + lineitem.l_extendedprice * (1 - lineitem.l_discount) - partsupp.ps_supplycost * lineitem.l_quantity as amount + from + part + join lineitem on part.p_partkey = lineitem.l_partkey + join supplier on supplier.s_suppkey = lineitem.l_suppkey + join partsupp on partsupp.ps_suppkey = lineitem.l_suppkey and partsupp.ps_partkey = lineitem.l_partkey + join orders on orders.o_orderkey = lineitem.l_orderkey + join nation on supplier.s_nationkey = nation.n_nationkey + join tmpTableC on orders.o_orderkey = tmpTableC.orderkey + where + part.p_name like '.*green.*' + ) + group by + o_year, + nation + order by + o_year desc +) +order by nation; \ No newline at end of file diff --git a/test/src/test/resources/tpch/runtimeInfo/failedQueryIds.txt b/test/src/test/resources/tpch/runtimeInfo/failedQueryIds.txt new file mode 100644 index 0000000000..00eb5f812a --- /dev/null +++ b/test/src/test/resources/tpch/runtimeInfo/failedQueryIds.txt @@ -0,0 +1,12 @@ +1 +2 +3 +5 +6 +9 +10 +16 +17 +18 +19 +20 \ No newline at end of file diff --git a/test/src/test/resources/tpch/runtimeInfo/iterationTimes.txt b/test/src/test/resources/tpch/runtimeInfo/iterationTimes.txt new file mode 100644 index 0000000000..56a6051ca2 --- /dev/null +++ b/test/src/test/resources/tpch/runtimeInfo/iterationTimes.txt @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/src/test/resources/tpch/runtimeInfo/newTimeCosts.txt b/test/src/test/resources/tpch/runtimeInfo/newTimeCosts.txt new file mode 100644 index 0000000000..901c11fe8f --- /dev/null +++ b/test/src/test/resources/tpch/runtimeInfo/newTimeCosts.txt @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/src/test/resources/tpch/runtimeInfo/oldTimeCosts.txt b/test/src/test/resources/tpch/runtimeInfo/oldTimeCosts.txt new file mode 100644 index 0000000000..901c11fe8f --- /dev/null +++ b/test/src/test/resources/tpch/runtimeInfo/oldTimeCosts.txt @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/src/test/resources/tpch/runtimeInfo/status.txt b/test/src/test/resources/tpch/runtimeInfo/status.txt new file mode 100644 index 0000000000..c852a940ea --- /dev/null +++ b/test/src/test/resources/tpch/runtimeInfo/status.txt @@ -0,0 +1 @@ +unfinished \ No newline at end of file diff --git a/test/src/test/resources/tpch/sf0.1/q1.csv b/test/src/test/resources/tpch/sf0.1/q1.csv new file mode 100644 index 0000000000..ac95c90db3 --- /dev/null +++ b/test/src/test/resources/tpch/sf0.1/q1.csv @@ -0,0 +1,5 @@ +l_returnflag|l_linestatus|sum_qty|sum_base_price|sum_disc_price|sum_charge|avg_qty|avg_price|avg_disc|count_order +A|F|3774200|5320753880.69|5054096266.6828|5256751331.449234|25.537587116854997|36002.12382901414|0.05014459706340077|147790 +N|F|95257|133737795.84|127132372.6512|132286291.229445|25.30066401062417|35521.32691633466|0.04939442231075697|3765 +N|O|7459297|10512270008.90|9986238338.3847|10385578376.585467|25.545537671232875|36000.9246880137|0.05009595890410959|292000 +R|F|3785523|5337950526.47|5071818532.9420|5274405503.049367|25.5259438574251|35994.029214030925|0.04998927856184382|148301 diff --git a/test/src/test/resources/tpch/sf0.1/q10.csv b/test/src/test/resources/tpch/sf0.1/q10.csv new file mode 100644 index 0000000000..79f006ddb9 --- /dev/null +++ b/test/src/test/resources/tpch/sf0.1/q10.csv @@ -0,0 +1,21 @@ +c_custkey|c_name|revenue|c_acctbal|n_name|c_address|c_phone|c_comment +8242|Customer#000008242|622786.7297|6322.09|ETHIOPIA|cYDWDiJt06B8CYzXX2L8x2hn1VFG|15-792-676-1184| regular theodolites affix. carefully ironic packages cajole deposits; slyly ironic packages wake quickly. regular, +7714|Customer#000007714|557400.3053|9799.98|IRAN|9DDikq08GEE4B3X|20-922-418-6024|even accounts should cajole. regular, regular +11032|Customer#000011032|512500.9641|8496.93|UNITED KINGDOM|5igjoUgXoDUZVUIectL5lXO1T3AGKza0ft|33-102-772-3533|uests. ironic accounts after the fluffily fi +2455|Customer#000002455|481592.4053|2070.99|GERMANY|a5DZ199yfAcFhfi2uwBE PKo,Z|17-946-225-9977|pinto beans alongside of the furiously ironic asymptotes are quickly even platelets: express +12106|Customer#000012106|479414.2133|5342.11|UNITED STATES|wyJXywcExUxt|34-905-346-4472|blithely blithely final attainments? carefully special pinto beans around the quickly even asymptote +8530|Customer#000008530|457855.9467|9734.95|MOROCCO|leatyNRWCnfTMnTNuDGHsWJjRuAX|25-736-932-5850| the carefully pending packages. carefully +13984|Customer#000013984|446316.5104|3482.28|IRAN|B13vxRBojwvP3|20-981-264-2952|egular, ironic accounts integrate sly +1966|Customer#000001966|444059.0382|1937.72|ALGERIA|IbwZr7j QVifqf9WizOIWx,UXV9CqxUyrwj|10-973-269-8886|odolites across the unusual accounts hang carefully furiously bold excuses. regular pi +11026|Customer#000011026|417913.4142|7738.76|ALGERIA|4C iGzChcqnhGBdeeu|10-184-163-4632|eposits cajole according to the furiously bold instructions. regular, regular dependencies wake carefully eve +8501|Customer#000008501|412797.5100|6906.70|ARGENTINA|UTUQLX cQrF1UUJPsz|11-317-552-5840| packages. pending Tiresias after the regularly express forges haggle fina +1565|Customer#000001565|412506.0062|1820.03|BRAZIL|n4acVpG0Deyj5aIFAfSNg Iu9cUagwN3OsRbKC 4|12-402-178-2007|deposits; unusual, bold deposits around the f +14398|Customer#000014398|408575.3600|-602.24|UNITED STATES|l49oKjbjQHz6YZwjo5wPihM lyYO6G|34-814-111-5424|es haggle fluffily blithely fluffy requests; slyly express req +1465|Customer#000001465|405055.3457|9365.93|INDIA|zn9Q7pT6KlQp3T5mUO533aq,|18-807-487-1074|ress ideas cajole. slyly unusual theodolites cajole thin foxes. account +12595|Customer#000012595|401402.2391|-6.92|INDIA|gEMQ3WO90vSdAgxLFrt9FRS|18-186-132-3352| slyly dogged excuses. blithely blithe packages cajole +961|Customer#000000961|401198.1737|6963.68|JAPAN|W0SZ2oflx9aWTggtwSk3OEIXsubXTbGbD|22-989-463-6089|use furiously across the final deposits. quickly +14299|Customer#000014299|400968.3751|6595.97|RUSSIA|UFlOs8tQ,IfZPJm57|32-156-618-1224|slyly. ironic, bold deposits sleep blithely ironic, pending attainm +623|Customer#000000623|399883.4257|7887.60|INDONESIA|k3IlPSC4FKB13 hc6omhVs1ibvqeWEV|19-113-202-7085|se around the ideas. accounts cajole blithely slyly ironic requests. b +9151|Customer#000009151|396562.0295|5691.95|IRAQ|UKiN9OQupR,m5NtvSntbI8JBeo|21-834-147-4906|the deposits. pending, ironic foxes haggle along the regular, bold req +14819|Customer#000014819|396271.1036|7308.39|FRANCE|wS8yiQtE63FfoO6RKUzuVf6iBTmXBq16u|16-769-398-7926|ending asymptotes use fluffily quickly bold instructions. slyly bold dependencies sleep carefully pending a +13478|Customer#000013478|395513.1358|-778.11|KENYA|S5izwjM1 hCoUccO2JMepYwNyBSqI,ay|24-983-202-8240| requests boost quickly according to the express sheaves. blithely unusual packages sleep diff --git a/test/src/test/resources/tpch/sf0.1/q13.csv b/test/src/test/resources/tpch/sf0.1/q13.csv new file mode 100644 index 0000000000..d3f05df8a6 --- /dev/null +++ b/test/src/test/resources/tpch/sf0.1/q13.csv @@ -0,0 +1,38 @@ +c_count|custdist +0|5000 +9|659 +10|658 +11|643 +8|555 +12|542 +13|508 +7|494 +19|471 +20|464 +14|451 +17|449 +18|448 +15|446 +16|425 +21|406 +22|351 +6|334 +23|331 +24|278 +5|197 +25|184 +26|175 +27|136 +4|90 +28|86 +29|63 +3|45 +30|36 +31|26 +32|13 +2|12 +33|11 +34|5 +35|4 +36|2 +1|2 diff --git a/test/src/test/resources/tpch/sf0.1/q16.csv b/test/src/test/resources/tpch/sf0.1/q16.csv new file mode 100644 index 0000000000..23f2fa9b13 --- /dev/null +++ b/test/src/test/resources/tpch/sf0.1/q16.csv @@ -0,0 +1,2763 @@ +p_brand|p_type|p_size|supplier_cnt +Brand#14|SMALL ANODIZED NICKEL|45|12 +Brand#22|SMALL BURNISHED BRASS|19|12 +Brand#25|PROMO POLISHED COPPER|14|12 +Brand#35|LARGE ANODIZED STEEL|45|12 +Brand#35|PROMO BRUSHED COPPER|9|12 +Brand#51|ECONOMY ANODIZED STEEL|9|12 +Brand#53|LARGE BRUSHED NICKEL|45|12 +Brand#11|ECONOMY POLISHED COPPER|14|8 +Brand#11|LARGE PLATED STEEL|23|8 +Brand#11|PROMO POLISHED STEEL|23|8 +Brand#11|STANDARD ANODIZED COPPER|9|8 +Brand#12|ECONOMY BURNISHED BRASS|9|8 +Brand#12|LARGE ANODIZED BRASS|14|8 +Brand#12|SMALL ANODIZED TIN|23|8 +Brand#12|SMALL BRUSHED NICKEL|23|8 +Brand#12|STANDARD ANODIZED BRASS|3|8 +Brand#12|STANDARD BURNISHED TIN|23|8 +Brand#13|ECONOMY POLISHED BRASS|9|8 +Brand#13|LARGE BURNISHED COPPER|45|8 +Brand#13|MEDIUM ANODIZED STEEL|23|8 +Brand#13|MEDIUM PLATED NICKEL|3|8 +Brand#13|PROMO BURNISHED BRASS|9|8 +Brand#13|PROMO POLISHED BRASS|3|8 +Brand#13|PROMO POLISHED TIN|36|8 +Brand#13|SMALL BURNISHED STEEL|23|8 +Brand#13|STANDARD BRUSHED STEEL|9|8 +Brand#14|ECONOMY BRUSHED TIN|3|8 +Brand#14|ECONOMY BURNISHED TIN|23|8 +Brand#14|PROMO BRUSHED STEEL|9|8 +Brand#14|PROMO PLATED TIN|45|8 +Brand#15|ECONOMY PLATED TIN|9|8 +Brand#15|STANDARD BRUSHED COPPER|14|8 +Brand#15|STANDARD PLATED TIN|3|8 +Brand#21|ECONOMY POLISHED TIN|3|8 +Brand#21|PROMO POLISHED COPPER|9|8 +Brand#21|PROMO POLISHED TIN|49|8 +Brand#21|SMALL POLISHED STEEL|3|8 +Brand#21|STANDARD PLATED BRASS|49|8 +Brand#21|STANDARD PLATED NICKEL|49|8 +Brand#22|ECONOMY ANODIZED TIN|49|8 +Brand#22|ECONOMY BRUSHED BRASS|14|8 +Brand#22|LARGE BURNISHED TIN|36|8 +Brand#22|MEDIUM ANODIZED STEEL|36|8 +Brand#22|MEDIUM PLATED STEEL|9|8 +Brand#22|PROMO POLISHED NICKEL|9|8 +Brand#22|SMALL ANODIZED STEEL|19|8 +Brand#22|STANDARD ANODIZED COPPER|23|8 +Brand#23|ECONOMY BRUSHED NICKEL|23|8 +Brand#23|LARGE ANODIZED BRASS|9|8 +Brand#23|LARGE ANODIZED STEEL|23|8 +Brand#23|SMALL BRUSHED COPPER|23|8 +Brand#23|STANDARD BRUSHED TIN|3|8 +Brand#23|STANDARD BURNISHED NICKEL|49|8 +Brand#23|STANDARD PLATED NICKEL|36|8 +Brand#24|ECONOMY ANODIZED BRASS|19|8 +Brand#24|ECONOMY POLISHED BRASS|36|8 +Brand#24|LARGE BURNISHED STEEL|14|8 +Brand#24|MEDIUM PLATED NICKEL|36|8 +Brand#25|ECONOMY BRUSHED STEEL|49|8 +Brand#25|MEDIUM BURNISHED TIN|3|8 +Brand#25|PROMO ANODIZED TIN|36|8 +Brand#25|PROMO PLATED NICKEL|3|8 +Brand#25|SMALL BURNISHED BRASS|3|8 +Brand#31|LARGE ANODIZED BRASS|3|8 +Brand#31|SMALL ANODIZED COPPER|3|8 +Brand#31|SMALL ANODIZED NICKEL|9|8 +Brand#31|SMALL ANODIZED STEEL|14|8 +Brand#32|MEDIUM ANODIZED STEEL|49|8 +Brand#32|MEDIUM BURNISHED COPPER|19|8 +Brand#32|SMALL BURNISHED STEEL|23|8 +Brand#32|STANDARD BURNISHED STEEL|45|8 +Brand#34|ECONOMY ANODIZED NICKEL|49|8 +Brand#34|LARGE BURNISHED TIN|49|8 +Brand#34|MEDIUM BURNISHED NICKEL|3|8 +Brand#34|PROMO ANODIZED TIN|3|8 +Brand#34|SMALL BRUSHED TIN|3|8 +Brand#34|STANDARD BURNISHED TIN|23|8 +Brand#35|MEDIUM BRUSHED STEEL|45|8 +Brand#35|PROMO BURNISHED STEEL|14|8 +Brand#35|SMALL BURNISHED STEEL|23|8 +Brand#35|SMALL POLISHED COPPER|14|8 +Brand#35|STANDARD PLATED COPPER|9|8 +Brand#41|ECONOMY BRUSHED BRASS|23|8 +Brand#41|LARGE BURNISHED STEEL|23|8 +Brand#41|PROMO BURNISHED TIN|14|8 +Brand#41|PROMO PLATED STEEL|36|8 +Brand#41|PROMO POLISHED TIN|19|8 +Brand#41|SMALL BURNISHED COPPER|23|8 +Brand#42|LARGE POLISHED TIN|14|8 +Brand#42|MEDIUM ANODIZED TIN|49|8 +Brand#42|MEDIUM BRUSHED TIN|14|8 +Brand#42|MEDIUM BURNISHED NICKEL|23|8 +Brand#42|MEDIUM PLATED COPPER|45|8 +Brand#42|MEDIUM PLATED TIN|45|8 +Brand#42|SMALL PLATED COPPER|36|8 +Brand#43|ECONOMY BRUSHED STEEL|45|8 +Brand#43|LARGE BRUSHED COPPER|19|8 +Brand#43|PROMO BRUSHED BRASS|36|8 +Brand#43|SMALL BURNISHED TIN|45|8 +Brand#43|SMALL PLATED COPPER|45|8 +Brand#44|PROMO POLISHED TIN|23|8 +Brand#44|SMALL POLISHED NICKEL|14|8 +Brand#44|SMALL POLISHED TIN|45|8 +Brand#44|STANDARD BURNISHED COPPER|3|8 +Brand#51|LARGE ANODIZED BRASS|19|8 +Brand#51|LARGE POLISHED COPPER|23|8 +Brand#51|MEDIUM ANODIZED TIN|9|8 +Brand#51|MEDIUM ANODIZED TIN|14|8 +Brand#51|MEDIUM BURNISHED NICKEL|23|8 +Brand#51|SMALL ANODIZED COPPER|45|8 +Brand#51|SMALL ANODIZED COPPER|49|8 +Brand#51|SMALL BRUSHED COPPER|45|8 +Brand#51|SMALL BRUSHED TIN|36|8 +Brand#51|STANDARD POLISHED TIN|3|8 +Brand#52|ECONOMY ANODIZED STEEL|3|8 +Brand#52|ECONOMY PLATED TIN|19|8 +Brand#52|LARGE PLATED TIN|3|8 +Brand#52|MEDIUM ANODIZED TIN|19|8 +Brand#52|MEDIUM BURNISHED COPPER|3|8 +Brand#52|PROMO POLISHED BRASS|23|8 +Brand#52|SMALL PLATED COPPER|36|8 +Brand#52|SMALL POLISHED NICKEL|9|8 +Brand#52|STANDARD POLISHED NICKEL|45|8 +Brand#53|ECONOMY POLISHED STEEL|45|8 +Brand#53|LARGE POLISHED NICKEL|3|8 +Brand#53|SMALL BRUSHED COPPER|14|8 +Brand#53|STANDARD PLATED STEEL|45|8 +Brand#54|ECONOMY POLISHED BRASS|49|8 +Brand#54|ECONOMY POLISHED TIN|23|8 +Brand#54|LARGE ANODIZED NICKEL|49|8 +Brand#54|MEDIUM BRUSHED STEEL|9|8 +Brand#54|SMALL BURNISHED NICKEL|14|8 +Brand#54|SMALL PLATED TIN|14|8 +Brand#54|STANDARD BURNISHED STEEL|14|8 +Brand#54|STANDARD PLATED BRASS|23|8 +Brand#55|MEDIUM BURNISHED TIN|36|8 +Brand#55|PROMO ANODIZED BRASS|14|8 +Brand#55|STANDARD BURNISHED COPPER|45|8 +Brand#15|STANDARD PLATED TIN|36|7 +Brand#23|SMALL POLISHED BRASS|49|7 +Brand#42|STANDARD PLATED COPPER|19|7 +Brand#51|LARGE POLISHED NICKEL|14|7 +Brand#11|ECONOMY ANODIZED BRASS|19|4 +Brand#11|ECONOMY ANODIZED BRASS|45|4 +Brand#11|ECONOMY ANODIZED NICKEL|36|4 +Brand#11|ECONOMY BRUSHED COPPER|3|4 +Brand#11|ECONOMY BRUSHED COPPER|9|4 +Brand#11|ECONOMY BRUSHED STEEL|9|4 +Brand#11|ECONOMY BRUSHED STEEL|36|4 +Brand#11|ECONOMY BURNISHED BRASS|36|4 +Brand#11|ECONOMY BURNISHED COPPER|9|4 +Brand#11|ECONOMY BURNISHED COPPER|49|4 +Brand#11|ECONOMY BURNISHED NICKEL|14|4 +Brand#11|ECONOMY BURNISHED NICKEL|49|4 +Brand#11|ECONOMY PLATED COPPER|19|4 +Brand#11|ECONOMY PLATED NICKEL|45|4 +Brand#11|ECONOMY PLATED TIN|9|4 +Brand#11|ECONOMY POLISHED BRASS|3|4 +Brand#11|ECONOMY POLISHED COPPER|3|4 +Brand#11|ECONOMY POLISHED COPPER|45|4 +Brand#11|ECONOMY POLISHED NICKEL|36|4 +Brand#11|ECONOMY POLISHED STEEL|23|4 +Brand#11|ECONOMY POLISHED TIN|14|4 +Brand#11|LARGE ANODIZED COPPER|23|4 +Brand#11|LARGE ANODIZED NICKEL|9|4 +Brand#11|LARGE ANODIZED STEEL|9|4 +Brand#11|LARGE ANODIZED TIN|45|4 +Brand#11|LARGE BRUSHED STEEL|19|4 +Brand#11|LARGE BRUSHED TIN|3|4 +Brand#11|LARGE BRUSHED TIN|14|4 +Brand#11|LARGE BURNISHED COPPER|9|4 +Brand#11|LARGE BURNISHED COPPER|19|4 +Brand#11|LARGE BURNISHED STEEL|23|4 +Brand#11|LARGE BURNISHED TIN|9|4 +Brand#11|LARGE PLATED COPPER|23|4 +Brand#11|LARGE PLATED TIN|9|4 +Brand#11|LARGE PLATED TIN|14|4 +Brand#11|LARGE PLATED TIN|23|4 +Brand#11|LARGE POLISHED NICKEL|49|4 +Brand#11|MEDIUM ANODIZED BRASS|45|4 +Brand#11|MEDIUM ANODIZED TIN|14|4 +Brand#11|MEDIUM BRUSHED BRASS|14|4 +Brand#11|MEDIUM BRUSHED BRASS|45|4 +Brand#11|MEDIUM BRUSHED NICKEL|14|4 +Brand#11|MEDIUM BRUSHED NICKEL|36|4 +Brand#11|MEDIUM BRUSHED STEEL|19|4 +Brand#11|MEDIUM BURNISHED COPPER|9|4 +Brand#11|MEDIUM BURNISHED TIN|36|4 +Brand#11|MEDIUM PLATED BRASS|3|4 +Brand#11|MEDIUM PLATED TIN|19|4 +Brand#11|PROMO ANODIZED BRASS|3|4 +Brand#11|PROMO ANODIZED BRASS|19|4 +Brand#11|PROMO ANODIZED BRASS|45|4 +Brand#11|PROMO ANODIZED BRASS|49|4 +Brand#11|PROMO ANODIZED STEEL|23|4 +Brand#11|PROMO ANODIZED TIN|45|4 +Brand#11|PROMO BRUSHED BRASS|23|4 +Brand#11|PROMO BRUSHED STEEL|3|4 +Brand#11|PROMO BURNISHED BRASS|23|4 +Brand#11|PROMO BURNISHED BRASS|36|4 +Brand#11|PROMO BURNISHED BRASS|49|4 +Brand#11|PROMO BURNISHED TIN|9|4 +Brand#11|PROMO PLATED BRASS|9|4 +Brand#11|PROMO PLATED BRASS|45|4 +Brand#11|PROMO PLATED NICKEL|19|4 +Brand#11|PROMO POLISHED BRASS|3|4 +Brand#11|PROMO POLISHED BRASS|9|4 +Brand#11|PROMO POLISHED BRASS|19|4 +Brand#11|PROMO POLISHED COPPER|14|4 +Brand#11|PROMO POLISHED COPPER|45|4 +Brand#11|PROMO POLISHED TIN|49|4 +Brand#11|SMALL ANODIZED COPPER|36|4 +Brand#11|SMALL ANODIZED NICKEL|3|4 +Brand#11|SMALL ANODIZED NICKEL|14|4 +Brand#11|SMALL ANODIZED TIN|14|4 +Brand#11|SMALL ANODIZED TIN|19|4 +Brand#11|SMALL ANODIZED TIN|45|4 +Brand#11|SMALL BRUSHED TIN|14|4 +Brand#11|SMALL BRUSHED TIN|23|4 +Brand#11|SMALL BRUSHED TIN|45|4 +Brand#11|SMALL BURNISHED BRASS|49|4 +Brand#11|SMALL BURNISHED COPPER|23|4 +Brand#11|SMALL PLATED COPPER|45|4 +Brand#11|SMALL PLATED NICKEL|3|4 +Brand#11|SMALL PLATED STEEL|36|4 +Brand#11|SMALL PLATED TIN|19|4 +Brand#11|SMALL POLISHED BRASS|14|4 +Brand#11|SMALL POLISHED BRASS|23|4 +Brand#11|SMALL POLISHED COPPER|14|4 +Brand#11|SMALL POLISHED COPPER|36|4 +Brand#11|SMALL POLISHED STEEL|9|4 +Brand#11|STANDARD BRUSHED COPPER|23|4 +Brand#11|STANDARD BRUSHED NICKEL|14|4 +Brand#11|STANDARD BRUSHED TIN|14|4 +Brand#11|STANDARD BURNISHED BRASS|3|4 +Brand#11|STANDARD BURNISHED STEEL|23|4 +Brand#11|STANDARD PLATED BRASS|19|4 +Brand#11|STANDARD PLATED TIN|19|4 +Brand#11|STANDARD POLISHED NICKEL|45|4 +Brand#11|STANDARD POLISHED TIN|14|4 +Brand#11|STANDARD POLISHED TIN|45|4 +Brand#12|ECONOMY ANODIZED BRASS|23|4 +Brand#12|ECONOMY ANODIZED COPPER|14|4 +Brand#12|ECONOMY ANODIZED NICKEL|19|4 +Brand#12|ECONOMY ANODIZED NICKEL|45|4 +Brand#12|ECONOMY ANODIZED STEEL|9|4 +Brand#12|ECONOMY BRUSHED COPPER|36|4 +Brand#12|ECONOMY BRUSHED NICKEL|49|4 +Brand#12|ECONOMY BRUSHED STEEL|49|4 +Brand#12|ECONOMY BURNISHED COPPER|45|4 +Brand#12|ECONOMY PLATED COPPER|23|4 +Brand#12|ECONOMY PLATED STEEL|23|4 +Brand#12|ECONOMY PLATED TIN|36|4 +Brand#12|ECONOMY POLISHED BRASS|14|4 +Brand#12|ECONOMY POLISHED COPPER|45|4 +Brand#12|ECONOMY POLISHED NICKEL|9|4 +Brand#12|LARGE ANODIZED NICKEL|9|4 +Brand#12|LARGE ANODIZED NICKEL|49|4 +Brand#12|LARGE ANODIZED STEEL|49|4 +Brand#12|LARGE ANODIZED TIN|36|4 +Brand#12|LARGE ANODIZED TIN|45|4 +Brand#12|LARGE BURNISHED BRASS|14|4 +Brand#12|LARGE BURNISHED BRASS|19|4 +Brand#12|LARGE BURNISHED COPPER|9|4 +Brand#12|LARGE BURNISHED NICKEL|45|4 +Brand#12|LARGE BURNISHED TIN|36|4 +Brand#12|LARGE PLATED BRASS|3|4 +Brand#12|LARGE PLATED STEEL|36|4 +Brand#12|LARGE PLATED STEEL|45|4 +Brand#12|LARGE PLATED TIN|23|4 +Brand#12|LARGE POLISHED COPPER|14|4 +Brand#12|LARGE POLISHED COPPER|19|4 +Brand#12|LARGE POLISHED COPPER|49|4 +Brand#12|LARGE POLISHED STEEL|3|4 +Brand#12|MEDIUM ANODIZED COPPER|9|4 +Brand#12|MEDIUM ANODIZED COPPER|45|4 +Brand#12|MEDIUM ANODIZED NICKEL|45|4 +Brand#12|MEDIUM BRUSHED BRASS|19|4 +Brand#12|MEDIUM BRUSHED COPPER|9|4 +Brand#12|MEDIUM BRUSHED COPPER|36|4 +Brand#12|MEDIUM BRUSHED COPPER|49|4 +Brand#12|MEDIUM BRUSHED NICKEL|3|4 +Brand#12|MEDIUM BRUSHED NICKEL|14|4 +Brand#12|MEDIUM BRUSHED NICKEL|23|4 +Brand#12|MEDIUM BURNISHED BRASS|3|4 +Brand#12|MEDIUM BURNISHED COPPER|36|4 +Brand#12|MEDIUM BURNISHED NICKEL|19|4 +Brand#12|MEDIUM BURNISHED TIN|14|4 +Brand#12|MEDIUM PLATED BRASS|23|4 +Brand#12|MEDIUM PLATED TIN|19|4 +Brand#12|MEDIUM PLATED TIN|23|4 +Brand#12|MEDIUM PLATED TIN|49|4 +Brand#12|PROMO ANODIZED BRASS|9|4 +Brand#12|PROMO ANODIZED BRASS|45|4 +Brand#12|PROMO ANODIZED NICKEL|14|4 +Brand#12|PROMO ANODIZED STEEL|49|4 +Brand#12|PROMO ANODIZED TIN|3|4 +Brand#12|PROMO ANODIZED TIN|19|4 +Brand#12|PROMO BRUSHED COPPER|14|4 +Brand#12|PROMO BRUSHED COPPER|19|4 +Brand#12|PROMO BRUSHED NICKEL|23|4 +Brand#12|PROMO BRUSHED STEEL|23|4 +Brand#12|PROMO BRUSHED STEEL|36|4 +Brand#12|PROMO BURNISHED BRASS|49|4 +Brand#12|PROMO BURNISHED TIN|9|4 +Brand#12|PROMO BURNISHED TIN|14|4 +Brand#12|PROMO PLATED BRASS|36|4 +Brand#12|PROMO POLISHED COPPER|23|4 +Brand#12|PROMO POLISHED NICKEL|3|4 +Brand#12|PROMO POLISHED NICKEL|9|4 +Brand#12|PROMO POLISHED STEEL|14|4 +Brand#12|PROMO POLISHED TIN|23|4 +Brand#12|PROMO POLISHED TIN|36|4 +Brand#12|SMALL ANODIZED BRASS|36|4 +Brand#12|SMALL ANODIZED COPPER|23|4 +Brand#12|SMALL ANODIZED STEEL|36|4 +Brand#12|SMALL ANODIZED TIN|14|4 +Brand#12|SMALL BRUSHED COPPER|19|4 +Brand#12|SMALL BRUSHED COPPER|36|4 +Brand#12|SMALL BRUSHED TIN|36|4 +Brand#12|SMALL BURNISHED BRASS|14|4 +Brand#12|SMALL BURNISHED COPPER|9|4 +Brand#12|SMALL BURNISHED COPPER|36|4 +Brand#12|SMALL PLATED BRASS|9|4 +Brand#12|SMALL POLISHED BRASS|49|4 +Brand#12|SMALL POLISHED NICKEL|19|4 +Brand#12|SMALL POLISHED TIN|3|4 +Brand#12|STANDARD ANODIZED BRASS|19|4 +Brand#12|STANDARD ANODIZED NICKEL|19|4 +Brand#12|STANDARD ANODIZED STEEL|19|4 +Brand#12|STANDARD BRUSHED COPPER|36|4 +Brand#12|STANDARD BRUSHED NICKEL|23|4 +Brand#12|STANDARD BRUSHED STEEL|49|4 +Brand#12|STANDARD BURNISHED BRASS|23|4 +Brand#12|STANDARD BURNISHED COPPER|14|4 +Brand#12|STANDARD BURNISHED NICKEL|45|4 +Brand#12|STANDARD BURNISHED NICKEL|49|4 +Brand#12|STANDARD BURNISHED TIN|3|4 +Brand#12|STANDARD BURNISHED TIN|14|4 +Brand#12|STANDARD PLATED BRASS|19|4 +Brand#12|STANDARD PLATED NICKEL|45|4 +Brand#12|STANDARD PLATED STEEL|36|4 +Brand#12|STANDARD PLATED STEEL|45|4 +Brand#12|STANDARD PLATED TIN|9|4 +Brand#12|STANDARD POLISHED BRASS|49|4 +Brand#12|STANDARD POLISHED COPPER|3|4 +Brand#12|STANDARD POLISHED NICKEL|23|4 +Brand#12|STANDARD POLISHED TIN|14|4 +Brand#13|ECONOMY ANODIZED NICKEL|14|4 +Brand#13|ECONOMY ANODIZED NICKEL|19|4 +Brand#13|ECONOMY ANODIZED STEEL|45|4 +Brand#13|ECONOMY ANODIZED STEEL|49|4 +Brand#13|ECONOMY BRUSHED BRASS|3|4 +Brand#13|ECONOMY BURNISHED STEEL|14|4 +Brand#13|ECONOMY BURNISHED TIN|19|4 +Brand#13|ECONOMY BURNISHED TIN|45|4 +Brand#13|ECONOMY PLATED COPPER|19|4 +Brand#13|ECONOMY PLATED NICKEL|3|4 +Brand#13|ECONOMY PLATED STEEL|23|4 +Brand#13|ECONOMY PLATED TIN|3|4 +Brand#13|ECONOMY POLISHED BRASS|3|4 +Brand#13|ECONOMY POLISHED COPPER|9|4 +Brand#13|ECONOMY POLISHED COPPER|49|4 +Brand#13|ECONOMY POLISHED STEEL|23|4 +Brand#13|ECONOMY POLISHED STEEL|49|4 +Brand#13|LARGE ANODIZED BRASS|23|4 +Brand#13|LARGE ANODIZED COPPER|19|4 +Brand#13|LARGE ANODIZED NICKEL|9|4 +Brand#13|LARGE ANODIZED STEEL|45|4 +Brand#13|LARGE ANODIZED TIN|19|4 +Brand#13|LARGE BRUSHED BRASS|3|4 +Brand#13|LARGE BRUSHED BRASS|9|4 +Brand#13|LARGE BRUSHED BRASS|19|4 +Brand#13|LARGE BRUSHED COPPER|9|4 +Brand#13|LARGE BRUSHED COPPER|36|4 +Brand#13|LARGE BRUSHED NICKEL|3|4 +Brand#13|LARGE BRUSHED NICKEL|9|4 +Brand#13|LARGE BRUSHED NICKEL|14|4 +Brand#13|LARGE BRUSHED NICKEL|23|4 +Brand#13|LARGE BRUSHED STEEL|19|4 +Brand#13|LARGE BRUSHED TIN|49|4 +Brand#13|LARGE BURNISHED BRASS|49|4 +Brand#13|LARGE BURNISHED TIN|49|4 +Brand#13|LARGE PLATED COPPER|23|4 +Brand#13|LARGE PLATED STEEL|14|4 +Brand#13|LARGE PLATED STEEL|19|4 +Brand#13|LARGE PLATED STEEL|36|4 +Brand#13|LARGE PLATED TIN|14|4 +Brand#13|LARGE PLATED TIN|45|4 +Brand#13|LARGE POLISHED BRASS|3|4 +Brand#13|LARGE POLISHED BRASS|23|4 +Brand#13|LARGE POLISHED BRASS|49|4 +Brand#13|MEDIUM ANODIZED BRASS|3|4 +Brand#13|MEDIUM ANODIZED BRASS|36|4 +Brand#13|MEDIUM ANODIZED COPPER|14|4 +Brand#13|MEDIUM ANODIZED NICKEL|3|4 +Brand#13|MEDIUM ANODIZED STEEL|14|4 +Brand#13|MEDIUM ANODIZED STEEL|19|4 +Brand#13|MEDIUM ANODIZED STEEL|36|4 +Brand#13|MEDIUM BRUSHED BRASS|49|4 +Brand#13|MEDIUM BRUSHED COPPER|23|4 +Brand#13|MEDIUM BRUSHED NICKEL|45|4 +Brand#13|MEDIUM BURNISHED BRASS|9|4 +Brand#13|MEDIUM BURNISHED STEEL|19|4 +Brand#13|MEDIUM BURNISHED STEEL|49|4 +Brand#13|MEDIUM PLATED BRASS|3|4 +Brand#13|MEDIUM PLATED BRASS|23|4 +Brand#13|MEDIUM PLATED BRASS|36|4 +Brand#13|MEDIUM PLATED COPPER|19|4 +Brand#13|MEDIUM PLATED COPPER|23|4 +Brand#13|MEDIUM PLATED STEEL|3|4 +Brand#13|PROMO ANODIZED BRASS|14|4 +Brand#13|PROMO ANODIZED COPPER|9|4 +Brand#13|PROMO ANODIZED COPPER|45|4 +Brand#13|PROMO ANODIZED STEEL|23|4 +Brand#13|PROMO BRUSHED COPPER|49|4 +Brand#13|PROMO BURNISHED COPPER|19|4 +Brand#13|PROMO BURNISHED NICKEL|9|4 +Brand#13|PROMO BURNISHED STEEL|23|4 +Brand#13|PROMO BURNISHED STEEL|45|4 +Brand#13|PROMO BURNISHED TIN|19|4 +Brand#13|PROMO PLATED BRASS|14|4 +Brand#13|PROMO PLATED BRASS|19|4 +Brand#13|PROMO PLATED COPPER|3|4 +Brand#13|PROMO PLATED COPPER|19|4 +Brand#13|PROMO PLATED TIN|19|4 +Brand#13|PROMO POLISHED BRASS|49|4 +Brand#13|PROMO POLISHED STEEL|45|4 +Brand#13|PROMO POLISHED TIN|14|4 +Brand#13|SMALL ANODIZED STEEL|23|4 +Brand#13|SMALL ANODIZED TIN|3|4 +Brand#13|SMALL ANODIZED TIN|45|4 +Brand#13|SMALL BRUSHED COPPER|3|4 +Brand#13|SMALL BRUSHED NICKEL|19|4 +Brand#13|SMALL BRUSHED TIN|9|4 +Brand#13|SMALL BRUSHED TIN|45|4 +Brand#13|SMALL BURNISHED BRASS|19|4 +Brand#13|SMALL BURNISHED BRASS|45|4 +Brand#13|SMALL PLATED BRASS|9|4 +Brand#13|SMALL PLATED TIN|45|4 +Brand#13|SMALL POLISHED NICKEL|19|4 +Brand#13|SMALL POLISHED STEEL|49|4 +Brand#13|STANDARD ANODIZED COPPER|45|4 +Brand#13|STANDARD ANODIZED NICKEL|9|4 +Brand#13|STANDARD ANODIZED NICKEL|19|4 +Brand#13|STANDARD ANODIZED STEEL|14|4 +Brand#13|STANDARD ANODIZED TIN|9|4 +Brand#13|STANDARD ANODIZED TIN|36|4 +Brand#13|STANDARD BRUSHED BRASS|19|4 +Brand#13|STANDARD BRUSHED TIN|9|4 +Brand#13|STANDARD BURNISHED BRASS|9|4 +Brand#13|STANDARD BURNISHED BRASS|14|4 +Brand#13|STANDARD BURNISHED COPPER|45|4 +Brand#13|STANDARD PLATED BRASS|49|4 +Brand#13|STANDARD PLATED COPPER|19|4 +Brand#13|STANDARD PLATED NICKEL|23|4 +Brand#13|STANDARD PLATED TIN|9|4 +Brand#13|STANDARD POLISHED BRASS|49|4 +Brand#13|STANDARD POLISHED COPPER|9|4 +Brand#13|STANDARD POLISHED COPPER|49|4 +Brand#13|STANDARD POLISHED NICKEL|14|4 +Brand#13|STANDARD POLISHED NICKEL|19|4 +Brand#13|STANDARD POLISHED STEEL|23|4 +Brand#14|ECONOMY ANODIZED BRASS|19|4 +Brand#14|ECONOMY ANODIZED COPPER|9|4 +Brand#14|ECONOMY ANODIZED STEEL|19|4 +Brand#14|ECONOMY ANODIZED STEEL|45|4 +Brand#14|ECONOMY BRUSHED BRASS|19|4 +Brand#14|ECONOMY BRUSHED COPPER|45|4 +Brand#14|ECONOMY BRUSHED NICKEL|14|4 +Brand#14|ECONOMY BRUSHED TIN|14|4 +Brand#14|ECONOMY BURNISHED COPPER|9|4 +Brand#14|ECONOMY BURNISHED COPPER|19|4 +Brand#14|ECONOMY BURNISHED STEEL|36|4 +Brand#14|ECONOMY BURNISHED TIN|3|4 +Brand#14|ECONOMY PLATED BRASS|36|4 +Brand#14|ECONOMY PLATED COPPER|49|4 +Brand#14|ECONOMY PLATED STEEL|45|4 +Brand#14|ECONOMY PLATED TIN|9|4 +Brand#14|ECONOMY POLISHED COPPER|3|4 +Brand#14|ECONOMY POLISHED TIN|19|4 +Brand#14|LARGE ANODIZED COPPER|9|4 +Brand#14|LARGE ANODIZED COPPER|23|4 +Brand#14|LARGE ANODIZED NICKEL|3|4 +Brand#14|LARGE ANODIZED NICKEL|9|4 +Brand#14|LARGE ANODIZED NICKEL|19|4 +Brand#14|LARGE ANODIZED TIN|9|4 +Brand#14|LARGE BRUSHED COPPER|14|4 +Brand#14|LARGE BRUSHED NICKEL|45|4 +Brand#14|LARGE PLATED BRASS|3|4 +Brand#14|LARGE PLATED NICKEL|3|4 +Brand#14|LARGE PLATED NICKEL|14|4 +Brand#14|LARGE PLATED NICKEL|49|4 +Brand#14|LARGE PLATED TIN|49|4 +Brand#14|LARGE POLISHED BRASS|9|4 +Brand#14|LARGE POLISHED BRASS|14|4 +Brand#14|LARGE POLISHED BRASS|36|4 +Brand#14|LARGE POLISHED NICKEL|3|4 +Brand#14|LARGE POLISHED NICKEL|14|4 +Brand#14|LARGE POLISHED STEEL|9|4 +Brand#14|LARGE POLISHED STEEL|23|4 +Brand#14|LARGE POLISHED STEEL|36|4 +Brand#14|MEDIUM ANODIZED NICKEL|3|4 +Brand#14|MEDIUM ANODIZED NICKEL|49|4 +Brand#14|MEDIUM ANODIZED STEEL|23|4 +Brand#14|MEDIUM ANODIZED STEEL|36|4 +Brand#14|MEDIUM BRUSHED BRASS|9|4 +Brand#14|MEDIUM BRUSHED COPPER|23|4 +Brand#14|MEDIUM BRUSHED STEEL|14|4 +Brand#14|MEDIUM BURNISHED COPPER|14|4 +Brand#14|MEDIUM BURNISHED STEEL|3|4 +Brand#14|MEDIUM BURNISHED STEEL|49|4 +Brand#14|MEDIUM PLATED BRASS|36|4 +Brand#14|MEDIUM PLATED STEEL|49|4 +Brand#14|MEDIUM PLATED TIN|14|4 +Brand#14|PROMO ANODIZED BRASS|49|4 +Brand#14|PROMO ANODIZED STEEL|36|4 +Brand#14|PROMO BRUSHED STEEL|19|4 +Brand#14|PROMO BURNISHED BRASS|23|4 +Brand#14|PROMO BURNISHED STEEL|36|4 +Brand#14|PROMO PLATED BRASS|9|4 +Brand#14|PROMO PLATED BRASS|45|4 +Brand#14|PROMO PLATED COPPER|45|4 +Brand#14|PROMO PLATED STEEL|3|4 +Brand#14|PROMO POLISHED BRASS|9|4 +Brand#14|PROMO POLISHED COPPER|49|4 +Brand#14|PROMO POLISHED STEEL|19|4 +Brand#14|SMALL ANODIZED STEEL|23|4 +Brand#14|SMALL ANODIZED TIN|23|4 +Brand#14|SMALL BRUSHED BRASS|19|4 +Brand#14|SMALL BRUSHED BRASS|36|4 +Brand#14|SMALL BRUSHED COPPER|9|4 +Brand#14|SMALL BRUSHED TIN|36|4 +Brand#14|SMALL BURNISHED BRASS|45|4 +Brand#14|SMALL BURNISHED COPPER|9|4 +Brand#14|SMALL BURNISHED COPPER|14|4 +Brand#14|SMALL BURNISHED COPPER|45|4 +Brand#14|SMALL BURNISHED NICKEL|36|4 +Brand#14|SMALL BURNISHED STEEL|36|4 +Brand#14|SMALL BURNISHED TIN|23|4 +Brand#14|SMALL PLATED NICKEL|3|4 +Brand#14|SMALL PLATED NICKEL|9|4 +Brand#14|SMALL PLATED STEEL|14|4 +Brand#14|SMALL POLISHED BRASS|36|4 +Brand#14|SMALL POLISHED COPPER|36|4 +Brand#14|SMALL POLISHED NICKEL|9|4 +Brand#14|SMALL POLISHED STEEL|14|4 +Brand#14|SMALL POLISHED TIN|14|4 +Brand#14|STANDARD ANODIZED BRASS|19|4 +Brand#14|STANDARD ANODIZED NICKEL|14|4 +Brand#14|STANDARD ANODIZED STEEL|9|4 +Brand#14|STANDARD BRUSHED COPPER|45|4 +Brand#14|STANDARD BRUSHED NICKEL|45|4 +Brand#14|STANDARD BRUSHED TIN|45|4 +Brand#14|STANDARD BURNISHED BRASS|9|4 +Brand#14|STANDARD BURNISHED BRASS|23|4 +Brand#14|STANDARD BURNISHED BRASS|49|4 +Brand#14|STANDARD BURNISHED NICKEL|9|4 +Brand#14|STANDARD PLATED BRASS|36|4 +Brand#14|STANDARD PLATED COPPER|45|4 +Brand#14|STANDARD POLISHED NICKEL|3|4 +Brand#14|STANDARD POLISHED NICKEL|9|4 +Brand#14|STANDARD POLISHED TIN|19|4 +Brand#15|ECONOMY ANODIZED COPPER|14|4 +Brand#15|ECONOMY ANODIZED STEEL|19|4 +Brand#15|ECONOMY ANODIZED STEEL|36|4 +Brand#15|ECONOMY BRUSHED BRASS|36|4 +Brand#15|ECONOMY BRUSHED COPPER|14|4 +Brand#15|ECONOMY BRUSHED NICKEL|14|4 +Brand#15|ECONOMY BRUSHED STEEL|3|4 +Brand#15|ECONOMY BRUSHED TIN|3|4 +Brand#15|ECONOMY BURNISHED BRASS|14|4 +Brand#15|ECONOMY BURNISHED COPPER|3|4 +Brand#15|ECONOMY BURNISHED COPPER|23|4 +Brand#15|ECONOMY PLATED NICKEL|49|4 +Brand#15|ECONOMY PLATED STEEL|3|4 +Brand#15|ECONOMY PLATED STEEL|19|4 +Brand#15|ECONOMY PLATED STEEL|45|4 +Brand#15|LARGE ANODIZED BRASS|19|4 +Brand#15|LARGE ANODIZED BRASS|36|4 +Brand#15|LARGE ANODIZED BRASS|45|4 +Brand#15|LARGE ANODIZED COPPER|3|4 +Brand#15|LARGE ANODIZED NICKEL|9|4 +Brand#15|LARGE ANODIZED TIN|19|4 +Brand#15|LARGE BRUSHED BRASS|9|4 +Brand#15|LARGE BRUSHED BRASS|19|4 +Brand#15|LARGE BRUSHED COPPER|14|4 +Brand#15|LARGE BRUSHED STEEL|9|4 +Brand#15|LARGE BRUSHED STEEL|14|4 +Brand#15|LARGE BRUSHED STEEL|19|4 +Brand#15|LARGE BRUSHED STEEL|36|4 +Brand#15|LARGE BURNISHED BRASS|14|4 +Brand#15|LARGE BURNISHED BRASS|19|4 +Brand#15|LARGE BURNISHED COPPER|9|4 +Brand#15|LARGE BURNISHED COPPER|45|4 +Brand#15|LARGE BURNISHED TIN|49|4 +Brand#15|LARGE PLATED BRASS|19|4 +Brand#15|LARGE PLATED COPPER|3|4 +Brand#15|LARGE PLATED COPPER|23|4 +Brand#15|LARGE PLATED NICKEL|36|4 +Brand#15|MEDIUM ANODIZED BRASS|23|4 +Brand#15|MEDIUM ANODIZED COPPER|9|4 +Brand#15|MEDIUM ANODIZED NICKEL|3|4 +Brand#15|MEDIUM ANODIZED TIN|19|4 +Brand#15|MEDIUM BRUSHED BRASS|9|4 +Brand#15|MEDIUM BRUSHED TIN|23|4 +Brand#15|MEDIUM BURNISHED COPPER|36|4 +Brand#15|MEDIUM BURNISHED TIN|45|4 +Brand#15|MEDIUM PLATED COPPER|9|4 +Brand#15|MEDIUM PLATED NICKEL|9|4 +Brand#15|MEDIUM PLATED NICKEL|19|4 +Brand#15|MEDIUM PLATED STEEL|36|4 +Brand#15|MEDIUM PLATED STEEL|49|4 +Brand#15|MEDIUM PLATED TIN|9|4 +Brand#15|MEDIUM PLATED TIN|14|4 +Brand#15|MEDIUM PLATED TIN|23|4 +Brand#15|PROMO ANODIZED COPPER|23|4 +Brand#15|PROMO ANODIZED STEEL|14|4 +Brand#15|PROMO ANODIZED TIN|45|4 +Brand#15|PROMO BRUSHED COPPER|14|4 +Brand#15|PROMO BRUSHED COPPER|19|4 +Brand#15|PROMO BRUSHED NICKEL|19|4 +Brand#15|PROMO BRUSHED NICKEL|23|4 +Brand#15|PROMO BRUSHED STEEL|14|4 +Brand#15|PROMO BRUSHED TIN|36|4 +Brand#15|PROMO BURNISHED NICKEL|9|4 +Brand#15|PROMO BURNISHED STEEL|45|4 +Brand#15|PROMO PLATED COPPER|3|4 +Brand#15|PROMO PLATED COPPER|36|4 +Brand#15|PROMO PLATED STEEL|3|4 +Brand#15|PROMO PLATED TIN|49|4 +Brand#15|PROMO POLISHED COPPER|3|4 +Brand#15|PROMO POLISHED NICKEL|36|4 +Brand#15|PROMO POLISHED STEEL|36|4 +Brand#15|PROMO POLISHED TIN|49|4 +Brand#15|SMALL ANODIZED BRASS|14|4 +Brand#15|SMALL ANODIZED BRASS|19|4 +Brand#15|SMALL ANODIZED COPPER|9|4 +Brand#15|SMALL ANODIZED TIN|45|4 +Brand#15|SMALL BRUSHED BRASS|3|4 +Brand#15|SMALL BRUSHED COPPER|19|4 +Brand#15|SMALL BRUSHED STEEL|23|4 +Brand#15|SMALL BRUSHED TIN|45|4 +Brand#15|SMALL BURNISHED BRASS|19|4 +Brand#15|SMALL BURNISHED COPPER|14|4 +Brand#15|SMALL BURNISHED NICKEL|19|4 +Brand#15|SMALL BURNISHED NICKEL|49|4 +Brand#15|SMALL BURNISHED STEEL|9|4 +Brand#15|SMALL BURNISHED TIN|19|4 +Brand#15|SMALL BURNISHED TIN|23|4 +Brand#15|SMALL BURNISHED TIN|36|4 +Brand#15|SMALL PLATED BRASS|3|4 +Brand#15|SMALL PLATED COPPER|23|4 +Brand#15|SMALL PLATED COPPER|49|4 +Brand#15|SMALL PLATED NICKEL|36|4 +Brand#15|SMALL PLATED NICKEL|45|4 +Brand#15|SMALL PLATED STEEL|3|4 +Brand#15|SMALL PLATED TIN|9|4 +Brand#15|SMALL POLISHED COPPER|9|4 +Brand#15|SMALL POLISHED NICKEL|3|4 +Brand#15|SMALL POLISHED STEEL|19|4 +Brand#15|SMALL POLISHED STEEL|36|4 +Brand#15|SMALL POLISHED TIN|19|4 +Brand#15|SMALL POLISHED TIN|49|4 +Brand#15|STANDARD ANODIZED NICKEL|19|4 +Brand#15|STANDARD ANODIZED NICKEL|49|4 +Brand#15|STANDARD ANODIZED TIN|36|4 +Brand#15|STANDARD BRUSHED NICKEL|3|4 +Brand#15|STANDARD BURNISHED BRASS|23|4 +Brand#15|STANDARD BURNISHED STEEL|3|4 +Brand#15|STANDARD BURNISHED STEEL|45|4 +Brand#15|STANDARD PLATED BRASS|36|4 +Brand#15|STANDARD PLATED COPPER|14|4 +Brand#15|STANDARD PLATED COPPER|23|4 +Brand#15|STANDARD PLATED NICKEL|19|4 +Brand#15|STANDARD PLATED TIN|45|4 +Brand#15|STANDARD POLISHED BRASS|14|4 +Brand#15|STANDARD POLISHED COPPER|23|4 +Brand#15|STANDARD POLISHED NICKEL|45|4 +Brand#21|ECONOMY ANODIZED BRASS|3|4 +Brand#21|ECONOMY ANODIZED NICKEL|14|4 +Brand#21|ECONOMY ANODIZED STEEL|19|4 +Brand#21|ECONOMY ANODIZED STEEL|23|4 +Brand#21|ECONOMY ANODIZED STEEL|49|4 +Brand#21|ECONOMY ANODIZED TIN|19|4 +Brand#21|ECONOMY BRUSHED BRASS|9|4 +Brand#21|ECONOMY BRUSHED BRASS|14|4 +Brand#21|ECONOMY BRUSHED BRASS|36|4 +Brand#21|ECONOMY BRUSHED COPPER|49|4 +Brand#21|ECONOMY BRUSHED STEEL|45|4 +Brand#21|ECONOMY BRUSHED TIN|49|4 +Brand#21|ECONOMY BURNISHED BRASS|3|4 +Brand#21|ECONOMY BURNISHED COPPER|45|4 +Brand#21|ECONOMY BURNISHED STEEL|19|4 +Brand#21|ECONOMY BURNISHED STEEL|36|4 +Brand#21|ECONOMY PLATED BRASS|36|4 +Brand#21|ECONOMY PLATED COPPER|3|4 +Brand#21|ECONOMY PLATED COPPER|14|4 +Brand#21|ECONOMY PLATED NICKEL|49|4 +Brand#21|ECONOMY POLISHED NICKEL|3|4 +Brand#21|ECONOMY POLISHED NICKEL|9|4 +Brand#21|LARGE ANODIZED COPPER|3|4 +Brand#21|LARGE ANODIZED COPPER|9|4 +Brand#21|LARGE ANODIZED STEEL|36|4 +Brand#21|LARGE ANODIZED TIN|45|4 +Brand#21|LARGE BRUSHED COPPER|45|4 +Brand#21|LARGE BRUSHED STEEL|23|4 +Brand#21|LARGE BURNISHED BRASS|49|4 +Brand#21|LARGE BURNISHED COPPER|19|4 +Brand#21|LARGE BURNISHED STEEL|49|4 +Brand#21|LARGE BURNISHED TIN|49|4 +Brand#21|LARGE PLATED BRASS|19|4 +Brand#21|LARGE PLATED NICKEL|23|4 +Brand#21|LARGE PLATED NICKEL|49|4 +Brand#21|LARGE PLATED TIN|19|4 +Brand#21|LARGE POLISHED BRASS|49|4 +Brand#21|LARGE POLISHED COPPER|14|4 +Brand#21|LARGE POLISHED NICKEL|3|4 +Brand#21|LARGE POLISHED NICKEL|14|4 +Brand#21|LARGE POLISHED STEEL|14|4 +Brand#21|LARGE POLISHED TIN|49|4 +Brand#21|MEDIUM ANODIZED COPPER|14|4 +Brand#21|MEDIUM ANODIZED NICKEL|49|4 +Brand#21|MEDIUM BRUSHED COPPER|3|4 +Brand#21|MEDIUM BRUSHED COPPER|49|4 +Brand#21|MEDIUM BRUSHED STEEL|23|4 +Brand#21|MEDIUM BRUSHED TIN|3|4 +Brand#21|MEDIUM BRUSHED TIN|14|4 +Brand#21|MEDIUM BURNISHED NICKEL|14|4 +Brand#21|MEDIUM BURNISHED STEEL|23|4 +Brand#21|MEDIUM BURNISHED TIN|3|4 +Brand#21|MEDIUM PLATED BRASS|3|4 +Brand#21|MEDIUM PLATED BRASS|19|4 +Brand#21|MEDIUM PLATED STEEL|36|4 +Brand#21|PROMO ANODIZED BRASS|9|4 +Brand#21|PROMO ANODIZED COPPER|14|4 +Brand#21|PROMO ANODIZED NICKEL|23|4 +Brand#21|PROMO ANODIZED STEEL|3|4 +Brand#21|PROMO ANODIZED STEEL|14|4 +Brand#21|PROMO ANODIZED STEEL|36|4 +Brand#21|PROMO BRUSHED NICKEL|45|4 +Brand#21|PROMO BRUSHED STEEL|14|4 +Brand#21|PROMO BRUSHED STEEL|23|4 +Brand#21|PROMO BRUSHED STEEL|45|4 +Brand#21|PROMO BURNISHED BRASS|19|4 +Brand#21|PROMO BURNISHED COPPER|19|4 +Brand#21|PROMO BURNISHED NICKEL|9|4 +Brand#21|PROMO BURNISHED TIN|19|4 +Brand#21|PROMO PLATED NICKEL|9|4 +Brand#21|PROMO PLATED NICKEL|36|4 +Brand#21|PROMO PLATED STEEL|49|4 +Brand#21|PROMO PLATED TIN|3|4 +Brand#21|PROMO POLISHED NICKEL|23|4 +Brand#21|PROMO POLISHED TIN|14|4 +Brand#21|PROMO POLISHED TIN|19|4 +Brand#21|PROMO POLISHED TIN|23|4 +Brand#21|SMALL BRUSHED BRASS|23|4 +Brand#21|SMALL BRUSHED COPPER|49|4 +Brand#21|SMALL BURNISHED BRASS|23|4 +Brand#21|SMALL BURNISHED BRASS|36|4 +Brand#21|SMALL BURNISHED STEEL|19|4 +Brand#21|SMALL BURNISHED TIN|19|4 +Brand#21|SMALL PLATED BRASS|45|4 +Brand#21|SMALL PLATED COPPER|45|4 +Brand#21|SMALL PLATED STEEL|45|4 +Brand#21|SMALL PLATED TIN|14|4 +Brand#21|SMALL PLATED TIN|45|4 +Brand#21|SMALL POLISHED COPPER|9|4 +Brand#21|SMALL POLISHED NICKEL|23|4 +Brand#21|SMALL POLISHED TIN|3|4 +Brand#21|STANDARD ANODIZED BRASS|9|4 +Brand#21|STANDARD ANODIZED NICKEL|19|4 +Brand#21|STANDARD ANODIZED TIN|45|4 +Brand#21|STANDARD BURNISHED COPPER|36|4 +Brand#21|STANDARD BURNISHED NICKEL|23|4 +Brand#21|STANDARD BURNISHED TIN|9|4 +Brand#21|STANDARD PLATED BRASS|14|4 +Brand#21|STANDARD PLATED COPPER|19|4 +Brand#21|STANDARD PLATED NICKEL|3|4 +Brand#21|STANDARD PLATED STEEL|9|4 +Brand#21|STANDARD PLATED TIN|9|4 +Brand#21|STANDARD POLISHED BRASS|9|4 +Brand#21|STANDARD POLISHED COPPER|49|4 +Brand#21|STANDARD POLISHED STEEL|36|4 +Brand#21|STANDARD POLISHED TIN|36|4 +Brand#22|ECONOMY ANODIZED STEEL|9|4 +Brand#22|ECONOMY ANODIZED STEEL|14|4 +Brand#22|ECONOMY ANODIZED STEEL|23|4 +Brand#22|ECONOMY ANODIZED TIN|9|4 +Brand#22|ECONOMY ANODIZED TIN|36|4 +Brand#22|ECONOMY BRUSHED NICKEL|36|4 +Brand#22|ECONOMY BRUSHED NICKEL|45|4 +Brand#22|ECONOMY BURNISHED BRASS|9|4 +Brand#22|ECONOMY BURNISHED BRASS|23|4 +Brand#22|ECONOMY BURNISHED BRASS|45|4 +Brand#22|ECONOMY BURNISHED NICKEL|19|4 +Brand#22|ECONOMY BURNISHED NICKEL|49|4 +Brand#22|ECONOMY BURNISHED STEEL|9|4 +Brand#22|ECONOMY BURNISHED STEEL|14|4 +Brand#22|ECONOMY BURNISHED STEEL|23|4 +Brand#22|ECONOMY PLATED BRASS|36|4 +Brand#22|ECONOMY PLATED COPPER|23|4 +Brand#22|ECONOMY PLATED TIN|3|4 +Brand#22|ECONOMY POLISHED TIN|49|4 +Brand#22|LARGE ANODIZED BRASS|19|4 +Brand#22|LARGE ANODIZED COPPER|36|4 +Brand#22|LARGE ANODIZED STEEL|3|4 +Brand#22|LARGE BRUSHED BRASS|23|4 +Brand#22|LARGE BRUSHED BRASS|49|4 +Brand#22|LARGE BRUSHED STEEL|49|4 +Brand#22|LARGE BURNISHED COPPER|19|4 +Brand#22|LARGE BURNISHED STEEL|23|4 +Brand#22|LARGE BURNISHED STEEL|45|4 +Brand#22|LARGE BURNISHED TIN|45|4 +Brand#22|LARGE PLATED COPPER|14|4 +Brand#22|LARGE PLATED STEEL|49|4 +Brand#22|LARGE POLISHED BRASS|19|4 +Brand#22|LARGE POLISHED COPPER|19|4 +Brand#22|LARGE POLISHED COPPER|23|4 +Brand#22|LARGE POLISHED NICKEL|19|4 +Brand#22|LARGE POLISHED TIN|49|4 +Brand#22|MEDIUM ANODIZED BRASS|45|4 +Brand#22|MEDIUM ANODIZED COPPER|19|4 +Brand#22|MEDIUM ANODIZED COPPER|49|4 +Brand#22|MEDIUM ANODIZED NICKEL|9|4 +Brand#22|MEDIUM ANODIZED NICKEL|14|4 +Brand#22|MEDIUM ANODIZED NICKEL|36|4 +Brand#22|MEDIUM ANODIZED TIN|3|4 +Brand#22|MEDIUM ANODIZED TIN|9|4 +Brand#22|MEDIUM BRUSHED BRASS|3|4 +Brand#22|MEDIUM BRUSHED BRASS|14|4 +Brand#22|MEDIUM BRUSHED COPPER|3|4 +Brand#22|MEDIUM BRUSHED COPPER|45|4 +Brand#22|MEDIUM BRUSHED NICKEL|14|4 +Brand#22|MEDIUM BRUSHED TIN|45|4 +Brand#22|MEDIUM BURNISHED COPPER|36|4 +Brand#22|MEDIUM BURNISHED TIN|19|4 +Brand#22|MEDIUM BURNISHED TIN|23|4 +Brand#22|MEDIUM BURNISHED TIN|49|4 +Brand#22|MEDIUM PLATED BRASS|49|4 +Brand#22|MEDIUM PLATED COPPER|9|4 +Brand#22|MEDIUM PLATED STEEL|3|4 +Brand#22|PROMO ANODIZED BRASS|9|4 +Brand#22|PROMO ANODIZED STEEL|36|4 +Brand#22|PROMO ANODIZED TIN|45|4 +Brand#22|PROMO BRUSHED BRASS|3|4 +Brand#22|PROMO BRUSHED BRASS|9|4 +Brand#22|PROMO BRUSHED BRASS|36|4 +Brand#22|PROMO BRUSHED STEEL|36|4 +Brand#22|PROMO BURNISHED BRASS|23|4 +Brand#22|PROMO BURNISHED COPPER|9|4 +Brand#22|PROMO PLATED BRASS|14|4 +Brand#22|PROMO PLATED BRASS|45|4 +Brand#22|PROMO PLATED NICKEL|3|4 +Brand#22|PROMO PLATED STEEL|19|4 +Brand#22|PROMO POLISHED BRASS|3|4 +Brand#22|PROMO POLISHED STEEL|14|4 +Brand#22|PROMO POLISHED STEEL|23|4 +Brand#22|SMALL ANODIZED TIN|36|4 +Brand#22|SMALL ANODIZED TIN|49|4 +Brand#22|SMALL BRUSHED NICKEL|3|4 +Brand#22|SMALL BRUSHED NICKEL|36|4 +Brand#22|SMALL BRUSHED NICKEL|45|4 +Brand#22|SMALL BRUSHED TIN|45|4 +Brand#22|SMALL BURNISHED STEEL|23|4 +Brand#22|SMALL BURNISHED TIN|14|4 +Brand#22|SMALL PLATED STEEL|3|4 +Brand#22|SMALL PLATED TIN|9|4 +Brand#22|SMALL PLATED TIN|36|4 +Brand#22|SMALL POLISHED BRASS|23|4 +Brand#22|SMALL POLISHED NICKEL|19|4 +Brand#22|STANDARD ANODIZED BRASS|14|4 +Brand#22|STANDARD ANODIZED BRASS|23|4 +Brand#22|STANDARD BRUSHED COPPER|49|4 +Brand#22|STANDARD BRUSHED NICKEL|3|4 +Brand#22|STANDARD BRUSHED NICKEL|23|4 +Brand#22|STANDARD BRUSHED STEEL|9|4 +Brand#22|STANDARD BRUSHED TIN|19|4 +Brand#22|STANDARD BURNISHED COPPER|45|4 +Brand#22|STANDARD BURNISHED NICKEL|3|4 +Brand#22|STANDARD BURNISHED NICKEL|14|4 +Brand#22|STANDARD BURNISHED NICKEL|45|4 +Brand#22|STANDARD BURNISHED STEEL|3|4 +Brand#22|STANDARD BURNISHED STEEL|36|4 +Brand#22|STANDARD BURNISHED STEEL|45|4 +Brand#22|STANDARD BURNISHED STEEL|49|4 +Brand#22|STANDARD PLATED BRASS|45|4 +Brand#22|STANDARD PLATED NICKEL|3|4 +Brand#22|STANDARD PLATED NICKEL|45|4 +Brand#22|STANDARD PLATED STEEL|14|4 +Brand#22|STANDARD PLATED TIN|19|4 +Brand#22|STANDARD PLATED TIN|49|4 +Brand#22|STANDARD POLISHED COPPER|9|4 +Brand#22|STANDARD POLISHED STEEL|49|4 +Brand#22|STANDARD POLISHED TIN|45|4 +Brand#23|ECONOMY ANODIZED NICKEL|49|4 +Brand#23|ECONOMY ANODIZED STEEL|14|4 +Brand#23|ECONOMY ANODIZED STEEL|49|4 +Brand#23|ECONOMY ANODIZED TIN|49|4 +Brand#23|ECONOMY BRUSHED BRASS|3|4 +Brand#23|ECONOMY BRUSHED COPPER|9|4 +Brand#23|ECONOMY BRUSHED TIN|9|4 +Brand#23|ECONOMY BURNISHED STEEL|49|4 +Brand#23|ECONOMY PLATED COPPER|14|4 +Brand#23|ECONOMY PLATED NICKEL|23|4 +Brand#23|ECONOMY PLATED STEEL|14|4 +Brand#23|ECONOMY POLISHED NICKEL|9|4 +Brand#23|LARGE ANODIZED BRASS|14|4 +Brand#23|LARGE ANODIZED COPPER|9|4 +Brand#23|LARGE ANODIZED COPPER|14|4 +Brand#23|LARGE ANODIZED COPPER|45|4 +Brand#23|LARGE ANODIZED STEEL|19|4 +Brand#23|LARGE ANODIZED STEEL|36|4 +Brand#23|LARGE ANODIZED STEEL|49|4 +Brand#23|LARGE ANODIZED TIN|9|4 +Brand#23|LARGE PLATED BRASS|9|4 +Brand#23|LARGE PLATED BRASS|49|4 +Brand#23|LARGE PLATED COPPER|3|4 +Brand#23|LARGE POLISHED BRASS|45|4 +Brand#23|LARGE POLISHED STEEL|9|4 +Brand#23|MEDIUM ANODIZED BRASS|19|4 +Brand#23|MEDIUM ANODIZED NICKEL|3|4 +Brand#23|MEDIUM ANODIZED NICKEL|14|4 +Brand#23|MEDIUM ANODIZED STEEL|45|4 +Brand#23|MEDIUM ANODIZED TIN|36|4 +Brand#23|MEDIUM ANODIZED TIN|45|4 +Brand#23|MEDIUM BRUSHED COPPER|3|4 +Brand#23|MEDIUM BRUSHED COPPER|23|4 +Brand#23|MEDIUM BRUSHED NICKEL|3|4 +Brand#23|MEDIUM BRUSHED TIN|14|4 +Brand#23|MEDIUM BURNISHED BRASS|9|4 +Brand#23|MEDIUM BURNISHED BRASS|45|4 +Brand#23|MEDIUM BURNISHED COPPER|19|4 +Brand#23|MEDIUM PLATED COPPER|19|4 +Brand#23|MEDIUM PLATED COPPER|36|4 +Brand#23|MEDIUM PLATED COPPER|45|4 +Brand#23|MEDIUM PLATED NICKEL|9|4 +Brand#23|MEDIUM PLATED NICKEL|14|4 +Brand#23|PROMO ANODIZED COPPER|9|4 +Brand#23|PROMO ANODIZED COPPER|19|4 +Brand#23|PROMO ANODIZED STEEL|36|4 +Brand#23|PROMO ANODIZED TIN|14|4 +Brand#23|PROMO BRUSHED BRASS|3|4 +Brand#23|PROMO BRUSHED BRASS|19|4 +Brand#23|PROMO BRUSHED BRASS|36|4 +Brand#23|PROMO BRUSHED COPPER|3|4 +Brand#23|PROMO BRUSHED TIN|49|4 +Brand#23|PROMO BURNISHED BRASS|14|4 +Brand#23|PROMO BURNISHED BRASS|45|4 +Brand#23|PROMO BURNISHED COPPER|14|4 +Brand#23|PROMO PLATED BRASS|23|4 +Brand#23|PROMO POLISHED BRASS|14|4 +Brand#23|PROMO POLISHED BRASS|23|4 +Brand#23|PROMO POLISHED COPPER|36|4 +Brand#23|PROMO POLISHED STEEL|36|4 +Brand#23|SMALL ANODIZED BRASS|23|4 +Brand#23|SMALL ANODIZED STEEL|23|4 +Brand#23|SMALL BRUSHED BRASS|49|4 +Brand#23|SMALL BRUSHED COPPER|45|4 +Brand#23|SMALL BRUSHED STEEL|3|4 +Brand#23|SMALL BRUSHED STEEL|19|4 +Brand#23|SMALL BURNISHED BRASS|36|4 +Brand#23|SMALL BURNISHED COPPER|45|4 +Brand#23|SMALL BURNISHED COPPER|49|4 +Brand#23|SMALL BURNISHED STEEL|45|4 +Brand#23|SMALL PLATED BRASS|36|4 +Brand#23|SMALL PLATED BRASS|49|4 +Brand#23|SMALL PLATED COPPER|14|4 +Brand#23|SMALL PLATED TIN|14|4 +Brand#23|SMALL POLISHED BRASS|9|4 +Brand#23|SMALL POLISHED BRASS|14|4 +Brand#23|SMALL POLISHED NICKEL|3|4 +Brand#23|SMALL POLISHED STEEL|14|4 +Brand#23|SMALL POLISHED TIN|9|4 +Brand#23|STANDARD ANODIZED BRASS|19|4 +Brand#23|STANDARD ANODIZED BRASS|45|4 +Brand#23|STANDARD ANODIZED COPPER|19|4 +Brand#23|STANDARD ANODIZED TIN|3|4 +Brand#23|STANDARD BRUSHED COPPER|36|4 +Brand#23|STANDARD BRUSHED NICKEL|19|4 +Brand#23|STANDARD BRUSHED STEEL|49|4 +Brand#23|STANDARD BURNISHED COPPER|19|4 +Brand#23|STANDARD PLATED BRASS|3|4 +Brand#23|STANDARD PLATED BRASS|9|4 +Brand#23|STANDARD PLATED STEEL|36|4 +Brand#23|STANDARD PLATED TIN|19|4 +Brand#23|STANDARD POLISHED BRASS|9|4 +Brand#23|STANDARD POLISHED BRASS|49|4 +Brand#23|STANDARD POLISHED STEEL|19|4 +Brand#23|STANDARD POLISHED STEEL|49|4 +Brand#23|STANDARD POLISHED TIN|23|4 +Brand#24|ECONOMY ANODIZED BRASS|3|4 +Brand#24|ECONOMY ANODIZED BRASS|9|4 +Brand#24|ECONOMY ANODIZED BRASS|23|4 +Brand#24|ECONOMY ANODIZED COPPER|9|4 +Brand#24|ECONOMY ANODIZED COPPER|49|4 +Brand#24|ECONOMY BRUSHED BRASS|36|4 +Brand#24|ECONOMY BRUSHED COPPER|23|4 +Brand#24|ECONOMY BURNISHED COPPER|3|4 +Brand#24|ECONOMY BURNISHED NICKEL|19|4 +Brand#24|ECONOMY BURNISHED STEEL|45|4 +Brand#24|ECONOMY PLATED BRASS|23|4 +Brand#24|ECONOMY PLATED COPPER|36|4 +Brand#24|ECONOMY PLATED STEEL|45|4 +Brand#24|ECONOMY POLISHED BRASS|23|4 +Brand#24|ECONOMY POLISHED COPPER|45|4 +Brand#24|ECONOMY POLISHED NICKEL|36|4 +Brand#24|ECONOMY POLISHED STEEL|14|4 +Brand#24|ECONOMY POLISHED STEEL|36|4 +Brand#24|LARGE ANODIZED NICKEL|23|4 +Brand#24|LARGE ANODIZED NICKEL|45|4 +Brand#24|LARGE ANODIZED TIN|45|4 +Brand#24|LARGE BRUSHED BRASS|14|4 +Brand#24|LARGE BRUSHED BRASS|23|4 +Brand#24|LARGE BRUSHED STEEL|9|4 +Brand#24|LARGE BRUSHED STEEL|23|4 +Brand#24|LARGE BRUSHED STEEL|45|4 +Brand#24|LARGE BRUSHED TIN|49|4 +Brand#24|LARGE BURNISHED BRASS|3|4 +Brand#24|LARGE BURNISHED NICKEL|19|4 +Brand#24|LARGE PLATED BRASS|9|4 +Brand#24|LARGE PLATED NICKEL|36|4 +Brand#24|LARGE PLATED NICKEL|49|4 +Brand#24|LARGE PLATED TIN|9|4 +Brand#24|LARGE PLATED TIN|19|4 +Brand#24|LARGE PLATED TIN|36|4 +Brand#24|LARGE PLATED TIN|49|4 +Brand#24|LARGE POLISHED BRASS|9|4 +Brand#24|LARGE POLISHED COPPER|9|4 +Brand#24|LARGE POLISHED COPPER|49|4 +Brand#24|LARGE POLISHED NICKEL|19|4 +Brand#24|LARGE POLISHED STEEL|23|4 +Brand#24|LARGE POLISHED TIN|14|4 +Brand#24|MEDIUM ANODIZED COPPER|45|4 +Brand#24|MEDIUM BRUSHED COPPER|9|4 +Brand#24|MEDIUM BRUSHED COPPER|14|4 +Brand#24|MEDIUM BRUSHED NICKEL|9|4 +Brand#24|MEDIUM BRUSHED NICKEL|23|4 +Brand#24|MEDIUM BRUSHED STEEL|14|4 +Brand#24|MEDIUM BRUSHED STEEL|45|4 +Brand#24|MEDIUM BRUSHED STEEL|49|4 +Brand#24|MEDIUM BURNISHED BRASS|36|4 +Brand#24|MEDIUM BURNISHED NICKEL|36|4 +Brand#24|MEDIUM BURNISHED STEEL|36|4 +Brand#24|MEDIUM PLATED COPPER|14|4 +Brand#24|MEDIUM PLATED STEEL|3|4 +Brand#24|MEDIUM PLATED STEEL|19|4 +Brand#24|PROMO ANODIZED NICKEL|9|4 +Brand#24|PROMO ANODIZED NICKEL|19|4 +Brand#24|PROMO ANODIZED NICKEL|45|4 +Brand#24|PROMO ANODIZED STEEL|3|4 +Brand#24|PROMO ANODIZED TIN|45|4 +Brand#24|PROMO BRUSHED BRASS|19|4 +Brand#24|PROMO BRUSHED NICKEL|19|4 +Brand#24|PROMO BRUSHED NICKEL|45|4 +Brand#24|PROMO BRUSHED STEEL|49|4 +Brand#24|PROMO BURNISHED BRASS|3|4 +Brand#24|PROMO BURNISHED BRASS|45|4 +Brand#24|PROMO BURNISHED STEEL|49|4 +Brand#24|PROMO PLATED BRASS|3|4 +Brand#24|PROMO PLATED COPPER|23|4 +Brand#24|PROMO PLATED COPPER|49|4 +Brand#24|PROMO POLISHED BRASS|3|4 +Brand#24|PROMO POLISHED BRASS|14|4 +Brand#24|PROMO POLISHED NICKEL|3|4 +Brand#24|PROMO POLISHED STEEL|14|4 +Brand#24|PROMO POLISHED STEEL|19|4 +Brand#24|PROMO POLISHED STEEL|23|4 +Brand#24|SMALL ANODIZED BRASS|19|4 +Brand#24|SMALL ANODIZED COPPER|3|4 +Brand#24|SMALL ANODIZED NICKEL|14|4 +Brand#24|SMALL ANODIZED STEEL|36|4 +Brand#24|SMALL ANODIZED TIN|3|4 +Brand#24|SMALL ANODIZED TIN|36|4 +Brand#24|SMALL BRUSHED COPPER|49|4 +Brand#24|SMALL BRUSHED NICKEL|49|4 +Brand#24|SMALL BURNISHED BRASS|14|4 +Brand#24|SMALL BURNISHED BRASS|19|4 +Brand#24|SMALL BURNISHED TIN|9|4 +Brand#24|SMALL PLATED BRASS|3|4 +Brand#24|SMALL PLATED COPPER|14|4 +Brand#24|SMALL PLATED COPPER|36|4 +Brand#24|SMALL PLATED NICKEL|14|4 +Brand#24|SMALL PLATED NICKEL|49|4 +Brand#24|SMALL POLISHED BRASS|3|4 +Brand#24|SMALL POLISHED NICKEL|9|4 +Brand#24|SMALL POLISHED NICKEL|19|4 +Brand#24|SMALL POLISHED NICKEL|36|4 +Brand#24|SMALL POLISHED STEEL|9|4 +Brand#24|SMALL POLISHED STEEL|36|4 +Brand#24|STANDARD ANODIZED TIN|9|4 +Brand#24|STANDARD ANODIZED TIN|49|4 +Brand#24|STANDARD BRUSHED BRASS|14|4 +Brand#24|STANDARD BRUSHED COPPER|23|4 +Brand#24|STANDARD BRUSHED NICKEL|19|4 +Brand#24|STANDARD BRUSHED STEEL|14|4 +Brand#24|STANDARD BRUSHED TIN|36|4 +Brand#24|STANDARD BURNISHED COPPER|19|4 +Brand#24|STANDARD BURNISHED COPPER|36|4 +Brand#24|STANDARD BURNISHED NICKEL|45|4 +Brand#24|STANDARD PLATED BRASS|36|4 +Brand#24|STANDARD PLATED COPPER|45|4 +Brand#24|STANDARD PLATED NICKEL|36|4 +Brand#24|STANDARD PLATED TIN|36|4 +Brand#24|STANDARD POLISHED COPPER|45|4 +Brand#24|STANDARD POLISHED NICKEL|14|4 +Brand#25|ECONOMY ANODIZED BRASS|14|4 +Brand#25|ECONOMY ANODIZED BRASS|49|4 +Brand#25|ECONOMY ANODIZED TIN|9|4 +Brand#25|ECONOMY ANODIZED TIN|19|4 +Brand#25|ECONOMY ANODIZED TIN|49|4 +Brand#25|ECONOMY BRUSHED COPPER|36|4 +Brand#25|ECONOMY BURNISHED COPPER|45|4 +Brand#25|ECONOMY BURNISHED TIN|19|4 +Brand#25|ECONOMY PLATED NICKEL|23|4 +Brand#25|ECONOMY PLATED TIN|14|4 +Brand#25|ECONOMY POLISHED BRASS|23|4 +Brand#25|ECONOMY POLISHED COPPER|9|4 +Brand#25|ECONOMY POLISHED NICKEL|3|4 +Brand#25|ECONOMY POLISHED TIN|9|4 +Brand#25|ECONOMY POLISHED TIN|45|4 +Brand#25|LARGE ANODIZED BRASS|3|4 +Brand#25|LARGE ANODIZED BRASS|14|4 +Brand#25|LARGE ANODIZED COPPER|36|4 +Brand#25|LARGE ANODIZED NICKEL|23|4 +Brand#25|LARGE ANODIZED STEEL|23|4 +Brand#25|LARGE BRUSHED NICKEL|19|4 +Brand#25|LARGE BRUSHED NICKEL|49|4 +Brand#25|LARGE BRUSHED TIN|3|4 +Brand#25|LARGE BRUSHED TIN|9|4 +Brand#25|LARGE BURNISHED BRASS|19|4 +Brand#25|LARGE BURNISHED BRASS|23|4 +Brand#25|LARGE BURNISHED BRASS|49|4 +Brand#25|LARGE BURNISHED NICKEL|14|4 +Brand#25|LARGE BURNISHED TIN|49|4 +Brand#25|LARGE PLATED BRASS|14|4 +Brand#25|LARGE PLATED NICKEL|23|4 +Brand#25|LARGE PLATED NICKEL|45|4 +Brand#25|LARGE PLATED TIN|19|4 +Brand#25|LARGE PLATED TIN|23|4 +Brand#25|LARGE POLISHED BRASS|9|4 +Brand#25|LARGE POLISHED COPPER|14|4 +Brand#25|LARGE POLISHED COPPER|36|4 +Brand#25|MEDIUM ANODIZED TIN|36|4 +Brand#25|MEDIUM BRUSHED COPPER|9|4 +Brand#25|MEDIUM BRUSHED COPPER|36|4 +Brand#25|MEDIUM BRUSHED COPPER|49|4 +Brand#25|MEDIUM BURNISHED COPPER|49|4 +Brand#25|MEDIUM BURNISHED NICKEL|9|4 +Brand#25|MEDIUM BURNISHED NICKEL|49|4 +Brand#25|MEDIUM BURNISHED STEEL|3|4 +Brand#25|MEDIUM BURNISHED STEEL|36|4 +Brand#25|MEDIUM BURNISHED STEEL|45|4 +Brand#25|MEDIUM BURNISHED STEEL|49|4 +Brand#25|MEDIUM BURNISHED TIN|9|4 +Brand#25|MEDIUM BURNISHED TIN|36|4 +Brand#25|MEDIUM PLATED BRASS|45|4 +Brand#25|MEDIUM PLATED COPPER|14|4 +Brand#25|MEDIUM PLATED NICKEL|45|4 +Brand#25|MEDIUM PLATED STEEL|9|4 +Brand#25|MEDIUM PLATED STEEL|36|4 +Brand#25|PROMO ANODIZED COPPER|14|4 +Brand#25|PROMO ANODIZED COPPER|19|4 +Brand#25|PROMO ANODIZED STEEL|36|4 +Brand#25|PROMO ANODIZED TIN|3|4 +Brand#25|PROMO ANODIZED TIN|14|4 +Brand#25|PROMO BRUSHED NICKEL|3|4 +Brand#25|PROMO BRUSHED STEEL|19|4 +Brand#25|PROMO BRUSHED TIN|14|4 +Brand#25|PROMO BRUSHED TIN|36|4 +Brand#25|PROMO BURNISHED COPPER|19|4 +Brand#25|PROMO BURNISHED COPPER|45|4 +Brand#25|PROMO BURNISHED COPPER|49|4 +Brand#25|PROMO BURNISHED NICKEL|36|4 +Brand#25|PROMO BURNISHED TIN|3|4 +Brand#25|PROMO PLATED BRASS|45|4 +Brand#25|PROMO PLATED COPPER|19|4 +Brand#25|PROMO PLATED NICKEL|45|4 +Brand#25|PROMO PLATED NICKEL|49|4 +Brand#25|PROMO PLATED STEEL|23|4 +Brand#25|PROMO POLISHED BRASS|23|4 +Brand#25|SMALL ANODIZED BRASS|45|4 +Brand#25|SMALL ANODIZED NICKEL|19|4 +Brand#25|SMALL ANODIZED STEEL|23|4 +Brand#25|SMALL ANODIZED TIN|14|4 +Brand#25|SMALL ANODIZED TIN|19|4 +Brand#25|SMALL BRUSHED COPPER|45|4 +Brand#25|SMALL BRUSHED NICKEL|9|4 +Brand#25|SMALL BURNISHED COPPER|3|4 +Brand#25|SMALL BURNISHED STEEL|3|4 +Brand#25|SMALL BURNISHED STEEL|14|4 +Brand#25|SMALL BURNISHED TIN|3|4 +Brand#25|SMALL PLATED BRASS|19|4 +Brand#25|SMALL PLATED COPPER|23|4 +Brand#25|SMALL PLATED STEEL|45|4 +Brand#25|SMALL PLATED TIN|36|4 +Brand#25|SMALL POLISHED BRASS|23|4 +Brand#25|SMALL POLISHED COPPER|9|4 +Brand#25|SMALL POLISHED STEEL|14|4 +Brand#25|STANDARD ANODIZED STEEL|3|4 +Brand#25|STANDARD ANODIZED STEEL|19|4 +Brand#25|STANDARD ANODIZED TIN|9|4 +Brand#25|STANDARD BRUSHED BRASS|14|4 +Brand#25|STANDARD BRUSHED NICKEL|19|4 +Brand#25|STANDARD BRUSHED TIN|9|4 +Brand#25|STANDARD BURNISHED NICKEL|9|4 +Brand#25|STANDARD PLATED BRASS|3|4 +Brand#25|STANDARD PLATED COPPER|14|4 +Brand#25|STANDARD PLATED NICKEL|36|4 +Brand#25|STANDARD POLISHED BRASS|45|4 +Brand#25|STANDARD POLISHED COPPER|23|4 +Brand#25|STANDARD POLISHED NICKEL|3|4 +Brand#25|STANDARD POLISHED NICKEL|49|4 +Brand#25|STANDARD POLISHED TIN|36|4 +Brand#25|STANDARD POLISHED TIN|45|4 +Brand#31|ECONOMY ANODIZED BRASS|3|4 +Brand#31|ECONOMY ANODIZED COPPER|45|4 +Brand#31|ECONOMY ANODIZED STEEL|3|4 +Brand#31|ECONOMY ANODIZED TIN|45|4 +Brand#31|ECONOMY BRUSHED BRASS|14|4 +Brand#31|ECONOMY BRUSHED COPPER|19|4 +Brand#31|ECONOMY BRUSHED NICKEL|9|4 +Brand#31|ECONOMY BRUSHED NICKEL|14|4 +Brand#31|ECONOMY BRUSHED NICKEL|49|4 +Brand#31|ECONOMY BURNISHED COPPER|36|4 +Brand#31|ECONOMY BURNISHED STEEL|3|4 +Brand#31|ECONOMY BURNISHED TIN|49|4 +Brand#31|ECONOMY PLATED COPPER|49|4 +Brand#31|ECONOMY PLATED NICKEL|9|4 +Brand#31|ECONOMY PLATED STEEL|23|4 +Brand#31|ECONOMY PLATED TIN|36|4 +Brand#31|ECONOMY PLATED TIN|49|4 +Brand#31|ECONOMY POLISHED COPPER|3|4 +Brand#31|ECONOMY POLISHED COPPER|36|4 +Brand#31|ECONOMY POLISHED COPPER|49|4 +Brand#31|ECONOMY POLISHED NICKEL|3|4 +Brand#31|LARGE ANODIZED BRASS|19|4 +Brand#31|LARGE ANODIZED STEEL|45|4 +Brand#31|LARGE BRUSHED BRASS|36|4 +Brand#31|LARGE BRUSHED BRASS|49|4 +Brand#31|LARGE BRUSHED TIN|3|4 +Brand#31|LARGE BURNISHED BRASS|9|4 +Brand#31|LARGE PLATED COPPER|19|4 +Brand#31|LARGE PLATED NICKEL|14|4 +Brand#31|LARGE PLATED TIN|9|4 +Brand#31|LARGE PLATED TIN|14|4 +Brand#31|LARGE POLISHED BRASS|14|4 +Brand#31|LARGE POLISHED STEEL|14|4 +Brand#31|LARGE POLISHED STEEL|45|4 +Brand#31|LARGE POLISHED TIN|19|4 +Brand#31|MEDIUM ANODIZED BRASS|23|4 +Brand#31|MEDIUM ANODIZED BRASS|36|4 +Brand#31|MEDIUM ANODIZED COPPER|14|4 +Brand#31|MEDIUM ANODIZED COPPER|19|4 +Brand#31|MEDIUM ANODIZED COPPER|36|4 +Brand#31|MEDIUM ANODIZED STEEL|14|4 +Brand#31|MEDIUM ANODIZED STEEL|49|4 +Brand#31|MEDIUM ANODIZED TIN|19|4 +Brand#31|MEDIUM ANODIZED TIN|49|4 +Brand#31|MEDIUM BRUSHED BRASS|36|4 +Brand#31|MEDIUM BRUSHED STEEL|14|4 +Brand#31|MEDIUM BURNISHED BRASS|14|4 +Brand#31|MEDIUM BURNISHED COPPER|3|4 +Brand#31|MEDIUM BURNISHED NICKEL|9|4 +Brand#31|MEDIUM BURNISHED STEEL|9|4 +Brand#31|MEDIUM BURNISHED TIN|14|4 +Brand#31|MEDIUM BURNISHED TIN|23|4 +Brand#31|MEDIUM PLATED BRASS|3|4 +Brand#31|MEDIUM PLATED TIN|9|4 +Brand#31|MEDIUM PLATED TIN|36|4 +Brand#31|MEDIUM PLATED TIN|45|4 +Brand#31|PROMO ANODIZED BRASS|3|4 +Brand#31|PROMO ANODIZED NICKEL|9|4 +Brand#31|PROMO BRUSHED BRASS|3|4 +Brand#31|PROMO BRUSHED BRASS|23|4 +Brand#31|PROMO BRUSHED COPPER|23|4 +Brand#31|PROMO BRUSHED NICKEL|45|4 +Brand#31|PROMO BURNISHED COPPER|36|4 +Brand#31|PROMO BURNISHED STEEL|3|4 +Brand#31|PROMO BURNISHED TIN|3|4 +Brand#31|PROMO PLATED BRASS|19|4 +Brand#31|PROMO PLATED NICKEL|36|4 +Brand#31|PROMO POLISHED BRASS|49|4 +Brand#31|PROMO POLISHED COPPER|14|4 +Brand#31|PROMO POLISHED NICKEL|3|4 +Brand#31|PROMO POLISHED NICKEL|9|4 +Brand#31|PROMO POLISHED TIN|3|4 +Brand#31|PROMO POLISHED TIN|23|4 +Brand#31|SMALL ANODIZED COPPER|45|4 +Brand#31|SMALL ANODIZED STEEL|23|4 +Brand#31|SMALL ANODIZED TIN|3|4 +Brand#31|SMALL BRUSHED COPPER|36|4 +Brand#31|SMALL BRUSHED COPPER|49|4 +Brand#31|SMALL BRUSHED NICKEL|19|4 +Brand#31|SMALL BRUSHED NICKEL|23|4 +Brand#31|SMALL BURNISHED BRASS|45|4 +Brand#31|SMALL BURNISHED NICKEL|9|4 +Brand#31|SMALL BURNISHED NICKEL|36|4 +Brand#31|SMALL PLATED COPPER|36|4 +Brand#31|SMALL PLATED NICKEL|9|4 +Brand#31|SMALL PLATED NICKEL|36|4 +Brand#31|SMALL POLISHED BRASS|3|4 +Brand#31|SMALL POLISHED COPPER|45|4 +Brand#31|SMALL POLISHED NICKEL|45|4 +Brand#31|SMALL POLISHED TIN|23|4 +Brand#31|SMALL POLISHED TIN|49|4 +Brand#31|STANDARD BRUSHED STEEL|23|4 +Brand#31|STANDARD BRUSHED STEEL|49|4 +Brand#31|STANDARD BURNISHED BRASS|14|4 +Brand#31|STANDARD BURNISHED NICKEL|45|4 +Brand#31|STANDARD PLATED NICKEL|3|4 +Brand#31|STANDARD POLISHED BRASS|3|4 +Brand#31|STANDARD POLISHED BRASS|45|4 +Brand#31|STANDARD POLISHED STEEL|36|4 +Brand#32|ECONOMY ANODIZED BRASS|19|4 +Brand#32|ECONOMY ANODIZED COPPER|36|4 +Brand#32|ECONOMY ANODIZED STEEL|23|4 +Brand#32|ECONOMY ANODIZED STEEL|36|4 +Brand#32|ECONOMY ANODIZED STEEL|45|4 +Brand#32|ECONOMY ANODIZED TIN|19|4 +Brand#32|ECONOMY BRUSHED COPPER|45|4 +Brand#32|ECONOMY BRUSHED TIN|45|4 +Brand#32|ECONOMY BURNISHED BRASS|23|4 +Brand#32|ECONOMY BURNISHED COPPER|36|4 +Brand#32|ECONOMY BURNISHED COPPER|45|4 +Brand#32|ECONOMY BURNISHED STEEL|19|4 +Brand#32|ECONOMY PLATED BRASS|9|4 +Brand#32|ECONOMY PLATED COPPER|9|4 +Brand#32|ECONOMY PLATED NICKEL|23|4 +Brand#32|ECONOMY PLATED TIN|45|4 +Brand#32|ECONOMY POLISHED STEEL|3|4 +Brand#32|LARGE ANODIZED BRASS|23|4 +Brand#32|LARGE ANODIZED BRASS|36|4 +Brand#32|LARGE ANODIZED NICKEL|45|4 +Brand#32|LARGE ANODIZED STEEL|3|4 +Brand#32|LARGE ANODIZED STEEL|14|4 +Brand#32|LARGE BRUSHED STEEL|45|4 +Brand#32|LARGE BRUSHED TIN|45|4 +Brand#32|LARGE BURNISHED NICKEL|36|4 +Brand#32|LARGE BURNISHED TIN|19|4 +Brand#32|LARGE BURNISHED TIN|45|4 +Brand#32|LARGE PLATED BRASS|3|4 +Brand#32|LARGE PLATED NICKEL|49|4 +Brand#32|LARGE PLATED STEEL|19|4 +Brand#32|LARGE PLATED STEEL|36|4 +Brand#32|LARGE POLISHED BRASS|45|4 +Brand#32|LARGE POLISHED COPPER|9|4 +Brand#32|LARGE POLISHED COPPER|49|4 +Brand#32|LARGE POLISHED NICKEL|3|4 +Brand#32|MEDIUM ANODIZED BRASS|3|4 +Brand#32|MEDIUM ANODIZED BRASS|9|4 +Brand#32|MEDIUM ANODIZED TIN|23|4 +Brand#32|MEDIUM BRUSHED BRASS|23|4 +Brand#32|MEDIUM BRUSHED BRASS|49|4 +Brand#32|MEDIUM BRUSHED COPPER|9|4 +Brand#32|MEDIUM BRUSHED COPPER|19|4 +Brand#32|MEDIUM BRUSHED TIN|49|4 +Brand#32|MEDIUM BURNISHED BRASS|9|4 +Brand#32|MEDIUM BURNISHED BRASS|36|4 +Brand#32|MEDIUM BURNISHED BRASS|49|4 +Brand#32|MEDIUM BURNISHED COPPER|9|4 +Brand#32|MEDIUM BURNISHED COPPER|45|4 +Brand#32|MEDIUM BURNISHED NICKEL|49|4 +Brand#32|MEDIUM BURNISHED TIN|9|4 +Brand#32|MEDIUM BURNISHED TIN|45|4 +Brand#32|MEDIUM PLATED BRASS|3|4 +Brand#32|MEDIUM PLATED BRASS|49|4 +Brand#32|MEDIUM PLATED COPPER|3|4 +Brand#32|MEDIUM PLATED STEEL|9|4 +Brand#32|MEDIUM PLATED TIN|9|4 +Brand#32|PROMO ANODIZED BRASS|3|4 +Brand#32|PROMO ANODIZED COPPER|19|4 +Brand#32|PROMO ANODIZED NICKEL|23|4 +Brand#32|PROMO BRUSHED COPPER|23|4 +Brand#32|PROMO BRUSHED NICKEL|14|4 +Brand#32|PROMO BRUSHED NICKEL|36|4 +Brand#32|PROMO BRUSHED STEEL|14|4 +Brand#32|PROMO BRUSHED STEEL|23|4 +Brand#32|PROMO BRUSHED STEEL|49|4 +Brand#32|PROMO BURNISHED BRASS|45|4 +Brand#32|PROMO BURNISHED NICKEL|45|4 +Brand#32|PROMO BURNISHED TIN|14|4 +Brand#32|PROMO BURNISHED TIN|45|4 +Brand#32|PROMO PLATED TIN|19|4 +Brand#32|PROMO POLISHED NICKEL|36|4 +Brand#32|PROMO POLISHED TIN|3|4 +Brand#32|SMALL ANODIZED BRASS|3|4 +Brand#32|SMALL ANODIZED NICKEL|3|4 +Brand#32|SMALL ANODIZED NICKEL|14|4 +Brand#32|SMALL ANODIZED TIN|9|4 +Brand#32|SMALL BRUSHED BRASS|9|4 +Brand#32|SMALL BRUSHED BRASS|19|4 +Brand#32|SMALL BRUSHED COPPER|3|4 +Brand#32|SMALL BRUSHED COPPER|23|4 +Brand#32|SMALL BRUSHED NICKEL|9|4 +Brand#32|SMALL BRUSHED NICKEL|45|4 +Brand#32|SMALL BRUSHED STEEL|23|4 +Brand#32|SMALL BRUSHED TIN|9|4 +Brand#32|SMALL BURNISHED NICKEL|36|4 +Brand#32|SMALL BURNISHED STEEL|3|4 +Brand#32|SMALL BURNISHED TIN|23|4 +Brand#32|SMALL PLATED BRASS|49|4 +Brand#32|SMALL PLATED COPPER|36|4 +Brand#32|SMALL PLATED COPPER|45|4 +Brand#32|SMALL PLATED NICKEL|45|4 +Brand#32|SMALL PLATED STEEL|45|4 +Brand#32|SMALL PLATED TIN|23|4 +Brand#32|SMALL PLATED TIN|36|4 +Brand#32|SMALL PLATED TIN|45|4 +Brand#32|SMALL POLISHED NICKEL|36|4 +Brand#32|SMALL POLISHED STEEL|14|4 +Brand#32|SMALL POLISHED STEEL|23|4 +Brand#32|SMALL POLISHED STEEL|36|4 +Brand#32|SMALL POLISHED TIN|36|4 +Brand#32|SMALL POLISHED TIN|45|4 +Brand#32|STANDARD ANODIZED NICKEL|19|4 +Brand#32|STANDARD ANODIZED TIN|9|4 +Brand#32|STANDARD ANODIZED TIN|14|4 +Brand#32|STANDARD ANODIZED TIN|19|4 +Brand#32|STANDARD BRUSHED NICKEL|23|4 +Brand#32|STANDARD BURNISHED BRASS|36|4 +Brand#32|STANDARD BURNISHED BRASS|45|4 +Brand#32|STANDARD BURNISHED COPPER|3|4 +Brand#32|STANDARD BURNISHED COPPER|36|4 +Brand#32|STANDARD BURNISHED NICKEL|49|4 +Brand#32|STANDARD BURNISHED STEEL|49|4 +Brand#32|STANDARD BURNISHED TIN|23|4 +Brand#32|STANDARD PLATED BRASS|9|4 +Brand#32|STANDARD PLATED BRASS|45|4 +Brand#32|STANDARD PLATED STEEL|36|4 +Brand#32|STANDARD POLISHED BRASS|14|4 +Brand#32|STANDARD POLISHED COPPER|36|4 +Brand#32|STANDARD POLISHED STEEL|14|4 +Brand#33|ECONOMY ANODIZED BRASS|23|4 +Brand#33|ECONOMY ANODIZED COPPER|9|4 +Brand#33|ECONOMY ANODIZED NICKEL|3|4 +Brand#33|ECONOMY ANODIZED NICKEL|9|4 +Brand#33|ECONOMY ANODIZED NICKEL|23|4 +Brand#33|ECONOMY ANODIZED NICKEL|36|4 +Brand#33|ECONOMY BRUSHED BRASS|14|4 +Brand#33|ECONOMY BRUSHED COPPER|23|4 +Brand#33|ECONOMY BURNISHED BRASS|49|4 +Brand#33|ECONOMY BURNISHED COPPER|3|4 +Brand#33|ECONOMY BURNISHED COPPER|14|4 +Brand#33|ECONOMY BURNISHED STEEL|3|4 +Brand#33|ECONOMY BURNISHED TIN|36|4 +Brand#33|ECONOMY BURNISHED TIN|45|4 +Brand#33|ECONOMY PLATED COPPER|19|4 +Brand#33|ECONOMY PLATED COPPER|45|4 +Brand#33|ECONOMY PLATED NICKEL|14|4 +Brand#33|ECONOMY PLATED NICKEL|36|4 +Brand#33|ECONOMY PLATED STEEL|3|4 +Brand#33|ECONOMY PLATED STEEL|23|4 +Brand#33|ECONOMY PLATED STEEL|36|4 +Brand#33|ECONOMY POLISHED BRASS|14|4 +Brand#33|ECONOMY POLISHED NICKEL|19|4 +Brand#33|ECONOMY POLISHED TIN|9|4 +Brand#33|LARGE ANODIZED BRASS|36|4 +Brand#33|LARGE ANODIZED COPPER|19|4 +Brand#33|LARGE ANODIZED COPPER|45|4 +Brand#33|LARGE ANODIZED NICKEL|36|4 +Brand#33|LARGE ANODIZED NICKEL|45|4 +Brand#33|LARGE ANODIZED STEEL|3|4 +Brand#33|LARGE ANODIZED STEEL|45|4 +Brand#33|LARGE ANODIZED TIN|45|4 +Brand#33|LARGE BRUSHED BRASS|3|4 +Brand#33|LARGE BRUSHED BRASS|49|4 +Brand#33|LARGE BRUSHED STEEL|19|4 +Brand#33|LARGE BRUSHED TIN|36|4 +Brand#33|LARGE BURNISHED COPPER|45|4 +Brand#33|LARGE BURNISHED NICKEL|23|4 +Brand#33|LARGE BURNISHED STEEL|19|4 +Brand#33|LARGE PLATED BRASS|3|4 +Brand#33|LARGE PLATED COPPER|19|4 +Brand#33|LARGE PLATED STEEL|3|4 +Brand#33|LARGE PLATED STEEL|19|4 +Brand#33|LARGE PLATED TIN|45|4 +Brand#33|LARGE POLISHED BRASS|45|4 +Brand#33|LARGE POLISHED STEEL|14|4 +Brand#33|LARGE POLISHED STEEL|23|4 +Brand#33|LARGE POLISHED TIN|23|4 +Brand#33|MEDIUM ANODIZED BRASS|3|4 +Brand#33|MEDIUM ANODIZED COPPER|9|4 +Brand#33|MEDIUM ANODIZED COPPER|36|4 +Brand#33|MEDIUM ANODIZED COPPER|49|4 +Brand#33|MEDIUM ANODIZED NICKEL|3|4 +Brand#33|MEDIUM ANODIZED NICKEL|19|4 +Brand#33|MEDIUM BRUSHED BRASS|3|4 +Brand#33|MEDIUM BRUSHED STEEL|19|4 +Brand#33|MEDIUM BRUSHED TIN|14|4 +Brand#33|MEDIUM BURNISHED COPPER|14|4 +Brand#33|MEDIUM BURNISHED COPPER|49|4 +Brand#33|MEDIUM BURNISHED TIN|36|4 +Brand#33|MEDIUM PLATED BRASS|3|4 +Brand#33|MEDIUM PLATED STEEL|3|4 +Brand#33|MEDIUM PLATED STEEL|49|4 +Brand#33|PROMO ANODIZED BRASS|3|4 +Brand#33|PROMO BRUSHED BRASS|49|4 +Brand#33|PROMO BURNISHED COPPER|23|4 +Brand#33|PROMO BURNISHED NICKEL|14|4 +Brand#33|PROMO BURNISHED NICKEL|36|4 +Brand#33|PROMO BURNISHED TIN|19|4 +Brand#33|PROMO BURNISHED TIN|23|4 +Brand#33|PROMO PLATED COPPER|14|4 +Brand#33|PROMO PLATED STEEL|45|4 +Brand#33|PROMO PLATED STEEL|49|4 +Brand#33|PROMO PLATED TIN|49|4 +Brand#33|PROMO POLISHED COPPER|3|4 +Brand#33|PROMO POLISHED STEEL|3|4 +Brand#33|PROMO POLISHED STEEL|9|4 +Brand#33|PROMO POLISHED STEEL|23|4 +Brand#33|SMALL ANODIZED BRASS|19|4 +Brand#33|SMALL ANODIZED COPPER|23|4 +Brand#33|SMALL ANODIZED COPPER|49|4 +Brand#33|SMALL ANODIZED STEEL|9|4 +Brand#33|SMALL BRUSHED BRASS|3|4 +Brand#33|SMALL BRUSHED COPPER|3|4 +Brand#33|SMALL BRUSHED NICKEL|45|4 +Brand#33|SMALL BRUSHED STEEL|3|4 +Brand#33|SMALL BRUSHED TIN|9|4 +Brand#33|SMALL BURNISHED BRASS|19|4 +Brand#33|SMALL BURNISHED NICKEL|3|4 +Brand#33|SMALL PLATED BRASS|3|4 +Brand#33|SMALL PLATED STEEL|14|4 +Brand#33|SMALL PLATED STEEL|45|4 +Brand#33|SMALL PLATED TIN|23|4 +Brand#33|SMALL PLATED TIN|36|4 +Brand#33|SMALL POLISHED NICKEL|23|4 +Brand#33|SMALL POLISHED TIN|19|4 +Brand#33|SMALL POLISHED TIN|23|4 +Brand#33|SMALL POLISHED TIN|45|4 +Brand#33|STANDARD ANODIZED COPPER|49|4 +Brand#33|STANDARD ANODIZED STEEL|14|4 +Brand#33|STANDARD ANODIZED STEEL|45|4 +Brand#33|STANDARD ANODIZED STEEL|49|4 +Brand#33|STANDARD ANODIZED TIN|45|4 +Brand#33|STANDARD BRUSHED BRASS|9|4 +Brand#33|STANDARD BRUSHED NICKEL|45|4 +Brand#33|STANDARD BRUSHED STEEL|9|4 +Brand#33|STANDARD BRUSHED TIN|36|4 +Brand#33|STANDARD BURNISHED BRASS|9|4 +Brand#33|STANDARD BURNISHED BRASS|23|4 +Brand#33|STANDARD BURNISHED NICKEL|49|4 +Brand#33|STANDARD PLATED BRASS|49|4 +Brand#33|STANDARD PLATED COPPER|3|4 +Brand#33|STANDARD PLATED COPPER|14|4 +Brand#33|STANDARD PLATED NICKEL|36|4 +Brand#33|STANDARD PLATED STEEL|3|4 +Brand#33|STANDARD PLATED STEEL|36|4 +Brand#33|STANDARD PLATED TIN|14|4 +Brand#33|STANDARD POLISHED BRASS|9|4 +Brand#33|STANDARD POLISHED BRASS|19|4 +Brand#33|STANDARD POLISHED STEEL|3|4 +Brand#33|STANDARD POLISHED STEEL|9|4 +Brand#33|STANDARD POLISHED STEEL|14|4 +Brand#34|ECONOMY ANODIZED BRASS|9|4 +Brand#34|ECONOMY ANODIZED COPPER|3|4 +Brand#34|ECONOMY ANODIZED COPPER|14|4 +Brand#34|ECONOMY ANODIZED COPPER|19|4 +Brand#34|ECONOMY ANODIZED STEEL|9|4 +Brand#34|ECONOMY ANODIZED TIN|49|4 +Brand#34|ECONOMY BRUSHED BRASS|14|4 +Brand#34|ECONOMY BRUSHED NICKEL|49|4 +Brand#34|ECONOMY BURNISHED COPPER|9|4 +Brand#34|ECONOMY BURNISHED STEEL|19|4 +Brand#34|ECONOMY BURNISHED TIN|3|4 +Brand#34|ECONOMY BURNISHED TIN|23|4 +Brand#34|ECONOMY PLATED BRASS|9|4 +Brand#34|ECONOMY PLATED BRASS|14|4 +Brand#34|ECONOMY PLATED COPPER|3|4 +Brand#34|ECONOMY PLATED NICKEL|45|4 +Brand#34|ECONOMY PLATED TIN|14|4 +Brand#34|ECONOMY PLATED TIN|45|4 +Brand#34|ECONOMY POLISHED BRASS|45|4 +Brand#34|LARGE ANODIZED BRASS|14|4 +Brand#34|LARGE ANODIZED BRASS|23|4 +Brand#34|LARGE ANODIZED BRASS|36|4 +Brand#34|LARGE ANODIZED NICKEL|3|4 +Brand#34|LARGE ANODIZED TIN|49|4 +Brand#34|LARGE BRUSHED BRASS|49|4 +Brand#34|LARGE BRUSHED COPPER|23|4 +Brand#34|LARGE BRUSHED NICKEL|23|4 +Brand#34|LARGE BRUSHED STEEL|14|4 +Brand#34|LARGE BRUSHED STEEL|19|4 +Brand#34|LARGE BRUSHED TIN|9|4 +Brand#34|LARGE BURNISHED BRASS|23|4 +Brand#34|LARGE BURNISHED COPPER|3|4 +Brand#34|LARGE BURNISHED COPPER|36|4 +Brand#34|LARGE BURNISHED NICKEL|19|4 +Brand#34|LARGE PLATED BRASS|23|4 +Brand#34|LARGE PLATED BRASS|36|4 +Brand#34|LARGE PLATED BRASS|45|4 +Brand#34|LARGE PLATED COPPER|23|4 +Brand#34|LARGE PLATED COPPER|49|4 +Brand#34|LARGE PLATED STEEL|49|4 +Brand#34|LARGE POLISHED NICKEL|49|4 +Brand#34|MEDIUM ANODIZED COPPER|36|4 +Brand#34|MEDIUM ANODIZED TIN|3|4 +Brand#34|MEDIUM BRUSHED BRASS|49|4 +Brand#34|MEDIUM BRUSHED COPPER|9|4 +Brand#34|MEDIUM BRUSHED NICKEL|9|4 +Brand#34|MEDIUM BRUSHED NICKEL|23|4 +Brand#34|MEDIUM BRUSHED TIN|3|4 +Brand#34|MEDIUM BRUSHED TIN|14|4 +Brand#34|MEDIUM BURNISHED STEEL|45|4 +Brand#34|MEDIUM BURNISHED STEEL|49|4 +Brand#34|MEDIUM PLATED COPPER|36|4 +Brand#34|MEDIUM PLATED TIN|3|4 +Brand#34|MEDIUM PLATED TIN|14|4 +Brand#34|PROMO ANODIZED COPPER|45|4 +Brand#34|PROMO ANODIZED NICKEL|14|4 +Brand#34|PROMO ANODIZED STEEL|49|4 +Brand#34|PROMO ANODIZED TIN|14|4 +Brand#34|PROMO BRUSHED BRASS|9|4 +Brand#34|PROMO BRUSHED BRASS|23|4 +Brand#34|PROMO BRUSHED COPPER|36|4 +Brand#34|PROMO BRUSHED STEEL|36|4 +Brand#34|PROMO BURNISHED BRASS|49|4 +Brand#34|PROMO BURNISHED STEEL|3|4 +Brand#34|PROMO PLATED BRASS|9|4 +Brand#34|PROMO PLATED STEEL|49|4 +Brand#34|PROMO POLISHED BRASS|23|4 +Brand#34|PROMO POLISHED NICKEL|3|4 +Brand#34|PROMO POLISHED NICKEL|36|4 +Brand#34|SMALL ANODIZED BRASS|36|4 +Brand#34|SMALL ANODIZED COPPER|45|4 +Brand#34|SMALL ANODIZED NICKEL|14|4 +Brand#34|SMALL ANODIZED NICKEL|36|4 +Brand#34|SMALL ANODIZED STEEL|3|4 +Brand#34|SMALL ANODIZED STEEL|19|4 +Brand#34|SMALL ANODIZED STEEL|23|4 +Brand#34|SMALL ANODIZED STEEL|36|4 +Brand#34|SMALL BRUSHED BRASS|14|4 +Brand#34|SMALL BRUSHED BRASS|36|4 +Brand#34|SMALL BRUSHED NICKEL|14|4 +Brand#34|SMALL BRUSHED NICKEL|36|4 +Brand#34|SMALL BRUSHED NICKEL|45|4 +Brand#34|SMALL BRUSHED TIN|9|4 +Brand#34|SMALL BRUSHED TIN|23|4 +Brand#34|SMALL BRUSHED TIN|36|4 +Brand#34|SMALL BURNISHED COPPER|9|4 +Brand#34|SMALL BURNISHED TIN|36|4 +Brand#34|SMALL PLATED BRASS|14|4 +Brand#34|SMALL PLATED COPPER|36|4 +Brand#34|SMALL PLATED TIN|45|4 +Brand#34|SMALL POLISHED NICKEL|14|4 +Brand#34|SMALL POLISHED NICKEL|45|4 +Brand#34|SMALL POLISHED TIN|9|4 +Brand#34|SMALL POLISHED TIN|14|4 +Brand#34|SMALL POLISHED TIN|19|4 +Brand#34|STANDARD ANODIZED BRASS|23|4 +Brand#34|STANDARD ANODIZED BRASS|36|4 +Brand#34|STANDARD ANODIZED COPPER|45|4 +Brand#34|STANDARD ANODIZED NICKEL|36|4 +Brand#34|STANDARD ANODIZED STEEL|9|4 +Brand#34|STANDARD ANODIZED STEEL|49|4 +Brand#34|STANDARD ANODIZED TIN|9|4 +Brand#34|STANDARD BRUSHED BRASS|19|4 +Brand#34|STANDARD BRUSHED BRASS|23|4 +Brand#34|STANDARD BRUSHED NICKEL|23|4 +Brand#34|STANDARD BRUSHED STEEL|3|4 +Brand#34|STANDARD BRUSHED TIN|19|4 +Brand#34|STANDARD BURNISHED COPPER|45|4 +Brand#34|STANDARD BURNISHED NICKEL|19|4 +Brand#34|STANDARD BURNISHED NICKEL|45|4 +Brand#34|STANDARD BURNISHED STEEL|36|4 +Brand#34|STANDARD BURNISHED TIN|45|4 +Brand#34|STANDARD PLATED BRASS|9|4 +Brand#34|STANDARD PLATED COPPER|9|4 +Brand#34|STANDARD PLATED NICKEL|36|4 +Brand#35|ECONOMY ANODIZED COPPER|3|4 +Brand#35|ECONOMY ANODIZED STEEL|45|4 +Brand#35|ECONOMY BRUSHED BRASS|3|4 +Brand#35|ECONOMY BRUSHED NICKEL|49|4 +Brand#35|ECONOMY BRUSHED STEEL|23|4 +Brand#35|ECONOMY BRUSHED STEEL|45|4 +Brand#35|ECONOMY BRUSHED TIN|14|4 +Brand#35|ECONOMY BRUSHED TIN|23|4 +Brand#35|ECONOMY BURNISHED NICKEL|19|4 +Brand#35|ECONOMY BURNISHED STEEL|36|4 +Brand#35|ECONOMY BURNISHED TIN|9|4 +Brand#35|ECONOMY BURNISHED TIN|19|4 +Brand#35|ECONOMY BURNISHED TIN|49|4 +Brand#35|ECONOMY POLISHED COPPER|9|4 +Brand#35|ECONOMY POLISHED TIN|19|4 +Brand#35|LARGE ANODIZED BRASS|3|4 +Brand#35|LARGE ANODIZED BRASS|23|4 +Brand#35|LARGE ANODIZED COPPER|49|4 +Brand#35|LARGE ANODIZED STEEL|36|4 +Brand#35|LARGE ANODIZED TIN|9|4 +Brand#35|LARGE BRUSHED COPPER|9|4 +Brand#35|LARGE BRUSHED COPPER|23|4 +Brand#35|LARGE BRUSHED STEEL|3|4 +Brand#35|LARGE BRUSHED STEEL|9|4 +Brand#35|LARGE BURNISHED BRASS|36|4 +Brand#35|LARGE BURNISHED BRASS|45|4 +Brand#35|LARGE BURNISHED COPPER|23|4 +Brand#35|LARGE BURNISHED NICKEL|23|4 +Brand#35|LARGE PLATED BRASS|9|4 +Brand#35|LARGE PLATED COPPER|36|4 +Brand#35|LARGE POLISHED BRASS|49|4 +Brand#35|LARGE POLISHED STEEL|9|4 +Brand#35|LARGE POLISHED TIN|14|4 +Brand#35|MEDIUM ANODIZED BRASS|9|4 +Brand#35|MEDIUM ANODIZED BRASS|36|4 +Brand#35|MEDIUM ANODIZED COPPER|9|4 +Brand#35|MEDIUM BRUSHED BRASS|14|4 +Brand#35|MEDIUM BRUSHED COPPER|9|4 +Brand#35|MEDIUM BRUSHED COPPER|36|4 +Brand#35|MEDIUM BURNISHED BRASS|49|4 +Brand#35|MEDIUM BURNISHED NICKEL|45|4 +Brand#35|MEDIUM BURNISHED TIN|36|4 +Brand#35|MEDIUM PLATED BRASS|23|4 +Brand#35|MEDIUM PLATED COPPER|9|4 +Brand#35|MEDIUM PLATED NICKEL|45|4 +Brand#35|MEDIUM PLATED NICKEL|49|4 +Brand#35|MEDIUM PLATED STEEL|49|4 +Brand#35|PROMO ANODIZED COPPER|49|4 +Brand#35|PROMO ANODIZED NICKEL|19|4 +Brand#35|PROMO ANODIZED NICKEL|23|4 +Brand#35|PROMO ANODIZED TIN|3|4 +Brand#35|PROMO ANODIZED TIN|14|4 +Brand#35|PROMO BRUSHED BRASS|49|4 +Brand#35|PROMO BRUSHED NICKEL|14|4 +Brand#35|PROMO BRUSHED NICKEL|19|4 +Brand#35|PROMO BURNISHED BRASS|3|4 +Brand#35|PROMO BURNISHED STEEL|3|4 +Brand#35|PROMO PLATED BRASS|19|4 +Brand#35|PROMO PLATED COPPER|14|4 +Brand#35|PROMO PLATED STEEL|23|4 +Brand#35|PROMO PLATED STEEL|36|4 +Brand#35|PROMO PLATED TIN|19|4 +Brand#35|PROMO POLISHED BRASS|9|4 +Brand#35|PROMO POLISHED BRASS|36|4 +Brand#35|PROMO POLISHED NICKEL|36|4 +Brand#35|PROMO POLISHED STEEL|23|4 +Brand#35|PROMO POLISHED TIN|36|4 +Brand#35|PROMO POLISHED TIN|45|4 +Brand#35|SMALL ANODIZED COPPER|9|4 +Brand#35|SMALL ANODIZED STEEL|19|4 +Brand#35|SMALL ANODIZED TIN|19|4 +Brand#35|SMALL BRUSHED BRASS|36|4 +Brand#35|SMALL BRUSHED STEEL|49|4 +Brand#35|SMALL BRUSHED TIN|3|4 +Brand#35|SMALL BRUSHED TIN|19|4 +Brand#35|SMALL BRUSHED TIN|23|4 +Brand#35|SMALL BURNISHED BRASS|23|4 +Brand#35|SMALL BURNISHED STEEL|36|4 +Brand#35|SMALL BURNISHED TIN|3|4 +Brand#35|SMALL BURNISHED TIN|36|4 +Brand#35|SMALL BURNISHED TIN|49|4 +Brand#35|SMALL PLATED BRASS|23|4 +Brand#35|SMALL PLATED STEEL|14|4 +Brand#35|SMALL POLISHED BRASS|36|4 +Brand#35|SMALL POLISHED STEEL|3|4 +Brand#35|SMALL POLISHED STEEL|49|4 +Brand#35|SMALL POLISHED TIN|23|4 +Brand#35|SMALL POLISHED TIN|45|4 +Brand#35|STANDARD ANODIZED NICKEL|14|4 +Brand#35|STANDARD ANODIZED STEEL|23|4 +Brand#35|STANDARD ANODIZED STEEL|45|4 +Brand#35|STANDARD ANODIZED TIN|9|4 +Brand#35|STANDARD ANODIZED TIN|19|4 +Brand#35|STANDARD BRUSHED BRASS|3|4 +Brand#35|STANDARD BRUSHED BRASS|23|4 +Brand#35|STANDARD BRUSHED BRASS|36|4 +Brand#35|STANDARD BRUSHED COPPER|36|4 +Brand#35|STANDARD BRUSHED NICKEL|36|4 +Brand#35|STANDARD BRUSHED NICKEL|49|4 +Brand#35|STANDARD BRUSHED TIN|9|4 +Brand#35|STANDARD BURNISHED BRASS|9|4 +Brand#35|STANDARD BURNISHED BRASS|19|4 +Brand#35|STANDARD BURNISHED BRASS|23|4 +Brand#35|STANDARD BURNISHED COPPER|36|4 +Brand#35|STANDARD BURNISHED STEEL|14|4 +Brand#35|STANDARD PLATED COPPER|19|4 +Brand#35|STANDARD PLATED NICKEL|23|4 +Brand#35|STANDARD PLATED STEEL|14|4 +Brand#35|STANDARD PLATED STEEL|23|4 +Brand#35|STANDARD PLATED TIN|49|4 +Brand#35|STANDARD POLISHED NICKEL|23|4 +Brand#35|STANDARD POLISHED TIN|23|4 +Brand#35|STANDARD POLISHED TIN|45|4 +Brand#41|ECONOMY ANODIZED STEEL|49|4 +Brand#41|ECONOMY BRUSHED BRASS|3|4 +Brand#41|ECONOMY BRUSHED COPPER|36|4 +Brand#41|ECONOMY BRUSHED NICKEL|23|4 +Brand#41|ECONOMY BRUSHED STEEL|36|4 +Brand#41|ECONOMY BRUSHED STEEL|45|4 +Brand#41|ECONOMY BRUSHED TIN|14|4 +Brand#41|ECONOMY PLATED COPPER|3|4 +Brand#41|ECONOMY PLATED STEEL|3|4 +Brand#41|ECONOMY PLATED TIN|23|4 +Brand#41|ECONOMY POLISHED COPPER|19|4 +Brand#41|ECONOMY POLISHED NICKEL|9|4 +Brand#41|ECONOMY POLISHED NICKEL|14|4 +Brand#41|ECONOMY POLISHED NICKEL|23|4 +Brand#41|ECONOMY POLISHED NICKEL|49|4 +Brand#41|ECONOMY POLISHED STEEL|9|4 +Brand#41|ECONOMY POLISHED STEEL|19|4 +Brand#41|ECONOMY POLISHED STEEL|45|4 +Brand#41|ECONOMY POLISHED TIN|19|4 +Brand#41|LARGE ANODIZED BRASS|14|4 +Brand#41|LARGE ANODIZED BRASS|23|4 +Brand#41|LARGE ANODIZED COPPER|49|4 +Brand#41|LARGE ANODIZED STEEL|3|4 +Brand#41|LARGE ANODIZED STEEL|23|4 +Brand#41|LARGE BRUSHED COPPER|23|4 +Brand#41|LARGE BRUSHED COPPER|49|4 +Brand#41|LARGE BRUSHED STEEL|19|4 +Brand#41|LARGE BURNISHED BRASS|45|4 +Brand#41|LARGE BURNISHED COPPER|3|4 +Brand#41|LARGE BURNISHED NICKEL|23|4 +Brand#41|LARGE BURNISHED TIN|9|4 +Brand#41|LARGE PLATED NICKEL|3|4 +Brand#41|LARGE PLATED NICKEL|23|4 +Brand#41|LARGE PLATED STEEL|9|4 +Brand#41|LARGE PLATED STEEL|36|4 +Brand#41|LARGE PLATED TIN|9|4 +Brand#41|LARGE POLISHED BRASS|36|4 +Brand#41|LARGE POLISHED COPPER|19|4 +Brand#41|LARGE POLISHED COPPER|49|4 +Brand#41|LARGE POLISHED NICKEL|36|4 +Brand#41|LARGE POLISHED STEEL|14|4 +Brand#41|MEDIUM ANODIZED BRASS|9|4 +Brand#41|MEDIUM ANODIZED COPPER|14|4 +Brand#41|MEDIUM ANODIZED NICKEL|3|4 +Brand#41|MEDIUM ANODIZED NICKEL|9|4 +Brand#41|MEDIUM ANODIZED STEEL|14|4 +Brand#41|MEDIUM BRUSHED COPPER|3|4 +Brand#41|MEDIUM BRUSHED TIN|9|4 +Brand#41|MEDIUM BURNISHED COPPER|23|4 +Brand#41|MEDIUM BURNISHED STEEL|9|4 +Brand#41|MEDIUM BURNISHED STEEL|45|4 +Brand#41|MEDIUM BURNISHED TIN|3|4 +Brand#41|MEDIUM PLATED BRASS|19|4 +Brand#41|MEDIUM PLATED BRASS|45|4 +Brand#41|MEDIUM PLATED COPPER|19|4 +Brand#41|MEDIUM PLATED STEEL|19|4 +Brand#41|MEDIUM PLATED STEEL|23|4 +Brand#41|PROMO ANODIZED BRASS|19|4 +Brand#41|PROMO ANODIZED COPPER|9|4 +Brand#41|PROMO ANODIZED NICKEL|9|4 +Brand#41|PROMO BRUSHED BRASS|14|4 +Brand#41|PROMO BRUSHED COPPER|36|4 +Brand#41|PROMO BRUSHED NICKEL|14|4 +Brand#41|PROMO BURNISHED BRASS|49|4 +Brand#41|PROMO BURNISHED NICKEL|36|4 +Brand#41|PROMO BURNISHED TIN|3|4 +Brand#41|PROMO PLATED NICKEL|14|4 +Brand#41|PROMO PLATED NICKEL|45|4 +Brand#41|PROMO PLATED STEEL|3|4 +Brand#41|PROMO PLATED TIN|3|4 +Brand#41|PROMO POLISHED COPPER|23|4 +Brand#41|SMALL ANODIZED BRASS|3|4 +Brand#41|SMALL ANODIZED BRASS|14|4 +Brand#41|SMALL ANODIZED STEEL|45|4 +Brand#41|SMALL ANODIZED TIN|9|4 +Brand#41|SMALL BRUSHED TIN|19|4 +Brand#41|SMALL BURNISHED COPPER|9|4 +Brand#41|SMALL BURNISHED NICKEL|3|4 +Brand#41|SMALL BURNISHED TIN|45|4 +Brand#41|SMALL PLATED COPPER|14|4 +Brand#41|SMALL PLATED COPPER|36|4 +Brand#41|SMALL PLATED COPPER|49|4 +Brand#41|SMALL PLATED TIN|19|4 +Brand#41|SMALL POLISHED COPPER|14|4 +Brand#41|SMALL POLISHED COPPER|19|4 +Brand#41|SMALL POLISHED COPPER|36|4 +Brand#41|SMALL POLISHED TIN|45|4 +Brand#41|STANDARD ANODIZED COPPER|19|4 +Brand#41|STANDARD ANODIZED NICKEL|9|4 +Brand#41|STANDARD ANODIZED STEEL|49|4 +Brand#41|STANDARD ANODIZED TIN|9|4 +Brand#41|STANDARD ANODIZED TIN|36|4 +Brand#41|STANDARD ANODIZED TIN|49|4 +Brand#41|STANDARD BRUSHED BRASS|19|4 +Brand#41|STANDARD BRUSHED NICKEL|3|4 +Brand#41|STANDARD BRUSHED NICKEL|9|4 +Brand#41|STANDARD BRUSHED STEEL|45|4 +Brand#41|STANDARD BRUSHED TIN|45|4 +Brand#41|STANDARD BURNISHED BRASS|23|4 +Brand#41|STANDARD BURNISHED BRASS|36|4 +Brand#41|STANDARD BURNISHED COPPER|49|4 +Brand#41|STANDARD BURNISHED STEEL|45|4 +Brand#41|STANDARD PLATED BRASS|45|4 +Brand#41|STANDARD PLATED NICKEL|14|4 +Brand#41|STANDARD PLATED STEEL|45|4 +Brand#41|STANDARD PLATED TIN|49|4 +Brand#41|STANDARD POLISHED STEEL|9|4 +Brand#41|STANDARD POLISHED STEEL|19|4 +Brand#41|STANDARD POLISHED TIN|45|4 +Brand#42|ECONOMY ANODIZED NICKEL|19|4 +Brand#42|ECONOMY BRUSHED BRASS|14|4 +Brand#42|ECONOMY BRUSHED COPPER|3|4 +Brand#42|ECONOMY BRUSHED COPPER|14|4 +Brand#42|ECONOMY BRUSHED NICKEL|14|4 +Brand#42|ECONOMY BRUSHED STEEL|14|4 +Brand#42|ECONOMY BRUSHED TIN|19|4 +Brand#42|ECONOMY BRUSHED TIN|49|4 +Brand#42|ECONOMY BURNISHED BRASS|19|4 +Brand#42|ECONOMY BURNISHED COPPER|23|4 +Brand#42|ECONOMY BURNISHED NICKEL|14|4 +Brand#42|ECONOMY BURNISHED TIN|14|4 +Brand#42|ECONOMY PLATED COPPER|23|4 +Brand#42|ECONOMY POLISHED BRASS|3|4 +Brand#42|ECONOMY POLISHED COPPER|9|4 +Brand#42|ECONOMY POLISHED STEEL|9|4 +Brand#42|ECONOMY POLISHED STEEL|36|4 +Brand#42|ECONOMY POLISHED TIN|14|4 +Brand#42|LARGE ANODIZED BRASS|49|4 +Brand#42|LARGE ANODIZED COPPER|14|4 +Brand#42|LARGE ANODIZED COPPER|49|4 +Brand#42|LARGE ANODIZED NICKEL|45|4 +Brand#42|LARGE ANODIZED NICKEL|49|4 +Brand#42|LARGE ANODIZED TIN|45|4 +Brand#42|LARGE BRUSHED BRASS|49|4 +Brand#42|LARGE BURNISHED BRASS|45|4 +Brand#42|LARGE BURNISHED BRASS|49|4 +Brand#42|LARGE BURNISHED COPPER|9|4 +Brand#42|LARGE BURNISHED TIN|9|4 +Brand#42|LARGE PLATED BRASS|45|4 +Brand#42|LARGE PLATED COPPER|9|4 +Brand#42|LARGE PLATED NICKEL|36|4 +Brand#42|LARGE PLATED TIN|23|4 +Brand#42|LARGE POLISHED BRASS|9|4 +Brand#42|LARGE POLISHED NICKEL|3|4 +Brand#42|LARGE POLISHED NICKEL|23|4 +Brand#42|LARGE POLISHED STEEL|9|4 +Brand#42|MEDIUM ANODIZED BRASS|23|4 +Brand#42|MEDIUM ANODIZED COPPER|19|4 +Brand#42|MEDIUM ANODIZED NICKEL|14|4 +Brand#42|MEDIUM ANODIZED NICKEL|19|4 +Brand#42|MEDIUM ANODIZED NICKEL|23|4 +Brand#42|MEDIUM ANODIZED STEEL|9|4 +Brand#42|MEDIUM ANODIZED STEEL|14|4 +Brand#42|MEDIUM ANODIZED STEEL|23|4 +Brand#42|MEDIUM ANODIZED TIN|14|4 +Brand#42|MEDIUM ANODIZED TIN|19|4 +Brand#42|MEDIUM BRUSHED COPPER|45|4 +Brand#42|MEDIUM BRUSHED COPPER|49|4 +Brand#42|MEDIUM BRUSHED STEEL|36|4 +Brand#42|MEDIUM BURNISHED COPPER|49|4 +Brand#42|MEDIUM BURNISHED TIN|3|4 +Brand#42|MEDIUM BURNISHED TIN|49|4 +Brand#42|MEDIUM PLATED NICKEL|45|4 +Brand#42|MEDIUM PLATED STEEL|3|4 +Brand#42|MEDIUM PLATED STEEL|23|4 +Brand#42|MEDIUM PLATED STEEL|45|4 +Brand#42|PROMO ANODIZED NICKEL|3|4 +Brand#42|PROMO ANODIZED NICKEL|19|4 +Brand#42|PROMO ANODIZED STEEL|49|4 +Brand#42|PROMO BRUSHED COPPER|45|4 +Brand#42|PROMO BRUSHED STEEL|19|4 +Brand#42|PROMO BRUSHED TIN|45|4 +Brand#42|PROMO BURNISHED COPPER|45|4 +Brand#42|PROMO BURNISHED NICKEL|3|4 +Brand#42|PROMO BURNISHED STEEL|9|4 +Brand#42|PROMO BURNISHED TIN|49|4 +Brand#42|PROMO PLATED BRASS|45|4 +Brand#42|PROMO PLATED NICKEL|23|4 +Brand#42|PROMO PLATED STEEL|19|4 +Brand#42|PROMO PLATED STEEL|45|4 +Brand#42|PROMO POLISHED COPPER|36|4 +Brand#42|PROMO POLISHED NICKEL|3|4 +Brand#42|SMALL ANODIZED BRASS|23|4 +Brand#42|SMALL ANODIZED COPPER|14|4 +Brand#42|SMALL ANODIZED COPPER|19|4 +Brand#42|SMALL ANODIZED NICKEL|23|4 +Brand#42|SMALL BRUSHED TIN|49|4 +Brand#42|SMALL BURNISHED BRASS|3|4 +Brand#42|SMALL BURNISHED BRASS|36|4 +Brand#42|SMALL BURNISHED COPPER|9|4 +Brand#42|SMALL BURNISHED NICKEL|9|4 +Brand#42|SMALL BURNISHED TIN|9|4 +Brand#42|SMALL PLATED NICKEL|9|4 +Brand#42|SMALL PLATED TIN|36|4 +Brand#42|SMALL POLISHED BRASS|3|4 +Brand#42|SMALL POLISHED COPPER|36|4 +Brand#42|SMALL POLISHED NICKEL|23|4 +Brand#42|SMALL POLISHED STEEL|49|4 +Brand#42|SMALL POLISHED TIN|3|4 +Brand#42|STANDARD ANODIZED BRASS|49|4 +Brand#42|STANDARD ANODIZED COPPER|49|4 +Brand#42|STANDARD ANODIZED NICKEL|36|4 +Brand#42|STANDARD ANODIZED NICKEL|45|4 +Brand#42|STANDARD BRUSHED NICKEL|23|4 +Brand#42|STANDARD BURNISHED NICKEL|49|4 +Brand#42|STANDARD BURNISHED STEEL|3|4 +Brand#42|STANDARD BURNISHED TIN|19|4 +Brand#42|STANDARD PLATED BRASS|19|4 +Brand#42|STANDARD PLATED COPPER|9|4 +Brand#42|STANDARD PLATED NICKEL|45|4 +Brand#42|STANDARD PLATED STEEL|3|4 +Brand#42|STANDARD POLISHED BRASS|36|4 +Brand#42|STANDARD POLISHED BRASS|45|4 +Brand#42|STANDARD POLISHED COPPER|14|4 +Brand#42|STANDARD POLISHED NICKEL|45|4 +Brand#42|STANDARD POLISHED TIN|9|4 +Brand#42|STANDARD POLISHED TIN|19|4 +Brand#42|STANDARD POLISHED TIN|23|4 +Brand#42|STANDARD POLISHED TIN|36|4 +Brand#43|ECONOMY ANODIZED COPPER|19|4 +Brand#43|ECONOMY ANODIZED COPPER|45|4 +Brand#43|ECONOMY ANODIZED NICKEL|3|4 +Brand#43|ECONOMY ANODIZED NICKEL|49|4 +Brand#43|ECONOMY ANODIZED STEEL|23|4 +Brand#43|ECONOMY ANODIZED TIN|49|4 +Brand#43|ECONOMY BRUSHED BRASS|49|4 +Brand#43|ECONOMY BRUSHED COPPER|45|4 +Brand#43|ECONOMY BRUSHED NICKEL|9|4 +Brand#43|ECONOMY BURNISHED NICKEL|9|4 +Brand#43|ECONOMY BURNISHED TIN|19|4 +Brand#43|ECONOMY PLATED COPPER|36|4 +Brand#43|ECONOMY PLATED STEEL|9|4 +Brand#43|ECONOMY PLATED TIN|14|4 +Brand#43|ECONOMY PLATED TIN|19|4 +Brand#43|ECONOMY PLATED TIN|49|4 +Brand#43|ECONOMY POLISHED COPPER|19|4 +Brand#43|ECONOMY POLISHED NICKEL|36|4 +Brand#43|ECONOMY POLISHED TIN|14|4 +Brand#43|ECONOMY POLISHED TIN|45|4 +Brand#43|LARGE ANODIZED BRASS|14|4 +Brand#43|LARGE ANODIZED BRASS|36|4 +Brand#43|LARGE ANODIZED COPPER|45|4 +Brand#43|LARGE BRUSHED COPPER|3|4 +Brand#43|LARGE BRUSHED NICKEL|14|4 +Brand#43|LARGE BRUSHED NICKEL|19|4 +Brand#43|LARGE BRUSHED NICKEL|45|4 +Brand#43|LARGE BRUSHED NICKEL|49|4 +Brand#43|LARGE BURNISHED COPPER|3|4 +Brand#43|LARGE BURNISHED TIN|23|4 +Brand#43|LARGE BURNISHED TIN|45|4 +Brand#43|LARGE PLATED BRASS|45|4 +Brand#43|LARGE PLATED STEEL|14|4 +Brand#43|LARGE PLATED TIN|36|4 +Brand#43|LARGE PLATED TIN|45|4 +Brand#43|LARGE POLISHED BRASS|9|4 +Brand#43|LARGE POLISHED COPPER|9|4 +Brand#43|LARGE POLISHED COPPER|19|4 +Brand#43|LARGE POLISHED STEEL|14|4 +Brand#43|LARGE POLISHED TIN|45|4 +Brand#43|MEDIUM ANODIZED BRASS|14|4 +Brand#43|MEDIUM ANODIZED COPPER|36|4 +Brand#43|MEDIUM ANODIZED COPPER|49|4 +Brand#43|MEDIUM ANODIZED STEEL|19|4 +Brand#43|MEDIUM ANODIZED STEEL|36|4 +Brand#43|MEDIUM BRUSHED BRASS|9|4 +Brand#43|MEDIUM BRUSHED BRASS|49|4 +Brand#43|MEDIUM BRUSHED COPPER|3|4 +Brand#43|MEDIUM BRUSHED NICKEL|9|4 +Brand#43|MEDIUM BRUSHED STEEL|23|4 +Brand#43|MEDIUM BURNISHED COPPER|14|4 +Brand#43|MEDIUM BURNISHED COPPER|45|4 +Brand#43|MEDIUM BURNISHED TIN|23|4 +Brand#43|MEDIUM PLATED BRASS|3|4 +Brand#43|MEDIUM PLATED COPPER|14|4 +Brand#43|MEDIUM PLATED NICKEL|36|4 +Brand#43|MEDIUM PLATED NICKEL|45|4 +Brand#43|MEDIUM PLATED TIN|49|4 +Brand#43|PROMO ANODIZED NICKEL|45|4 +Brand#43|PROMO ANODIZED TIN|14|4 +Brand#43|PROMO BRUSHED NICKEL|14|4 +Brand#43|PROMO BRUSHED STEEL|14|4 +Brand#43|PROMO BRUSHED TIN|45|4 +Brand#43|PROMO BURNISHED BRASS|49|4 +Brand#43|PROMO BURNISHED NICKEL|9|4 +Brand#43|PROMO BURNISHED STEEL|3|4 +Brand#43|PROMO BURNISHED STEEL|36|4 +Brand#43|PROMO BURNISHED TIN|36|4 +Brand#43|PROMO PLATED BRASS|19|4 +Brand#43|PROMO PLATED COPPER|45|4 +Brand#43|PROMO PLATED COPPER|49|4 +Brand#43|PROMO PLATED TIN|3|4 +Brand#43|PROMO POLISHED BRASS|19|4 +Brand#43|PROMO POLISHED BRASS|23|4 +Brand#43|PROMO POLISHED NICKEL|49|4 +Brand#43|PROMO POLISHED STEEL|14|4 +Brand#43|PROMO POLISHED STEEL|19|4 +Brand#43|PROMO POLISHED STEEL|23|4 +Brand#43|PROMO POLISHED STEEL|36|4 +Brand#43|SMALL ANODIZED BRASS|19|4 +Brand#43|SMALL ANODIZED NICKEL|9|4 +Brand#43|SMALL BRUSHED NICKEL|3|4 +Brand#43|SMALL BRUSHED NICKEL|9|4 +Brand#43|SMALL BURNISHED BRASS|49|4 +Brand#43|SMALL BURNISHED STEEL|23|4 +Brand#43|SMALL PLATED BRASS|14|4 +Brand#43|SMALL PLATED BRASS|36|4 +Brand#43|SMALL PLATED COPPER|23|4 +Brand#43|SMALL PLATED COPPER|49|4 +Brand#43|SMALL PLATED NICKEL|36|4 +Brand#43|SMALL PLATED NICKEL|49|4 +Brand#43|SMALL PLATED STEEL|14|4 +Brand#43|SMALL PLATED TIN|49|4 +Brand#43|SMALL POLISHED STEEL|19|4 +Brand#43|STANDARD ANODIZED BRASS|3|4 +Brand#43|STANDARD ANODIZED COPPER|49|4 +Brand#43|STANDARD ANODIZED NICKEL|14|4 +Brand#43|STANDARD BRUSHED TIN|14|4 +Brand#43|STANDARD BURNISHED BRASS|23|4 +Brand#43|STANDARD BURNISHED STEEL|19|4 +Brand#43|STANDARD BURNISHED STEEL|23|4 +Brand#43|STANDARD PLATED BRASS|9|4 +Brand#43|STANDARD PLATED BRASS|19|4 +Brand#43|STANDARD PLATED BRASS|49|4 +Brand#43|STANDARD PLATED COPPER|36|4 +Brand#43|STANDARD PLATED NICKEL|14|4 +Brand#43|STANDARD PLATED NICKEL|19|4 +Brand#43|STANDARD PLATED TIN|14|4 +Brand#43|STANDARD POLISHED BRASS|23|4 +Brand#43|STANDARD POLISHED TIN|9|4 +Brand#44|ECONOMY ANODIZED BRASS|3|4 +Brand#44|ECONOMY ANODIZED BRASS|45|4 +Brand#44|ECONOMY ANODIZED NICKEL|36|4 +Brand#44|ECONOMY ANODIZED STEEL|19|4 +Brand#44|ECONOMY BRUSHED COPPER|23|4 +Brand#44|ECONOMY BRUSHED TIN|49|4 +Brand#44|ECONOMY BURNISHED COPPER|19|4 +Brand#44|ECONOMY BURNISHED STEEL|45|4 +Brand#44|ECONOMY PLATED STEEL|19|4 +Brand#44|ECONOMY PLATED STEEL|23|4 +Brand#44|ECONOMY PLATED TIN|23|4 +Brand#44|ECONOMY POLISHED BRASS|23|4 +Brand#44|ECONOMY POLISHED COPPER|9|4 +Brand#44|ECONOMY POLISHED COPPER|45|4 +Brand#44|ECONOMY POLISHED NICKEL|14|4 +Brand#44|ECONOMY POLISHED NICKEL|23|4 +Brand#44|ECONOMY POLISHED STEEL|49|4 +Brand#44|ECONOMY POLISHED TIN|23|4 +Brand#44|ECONOMY POLISHED TIN|36|4 +Brand#44|LARGE ANODIZED BRASS|19|4 +Brand#44|LARGE ANODIZED TIN|3|4 +Brand#44|LARGE ANODIZED TIN|14|4 +Brand#44|LARGE BRUSHED TIN|3|4 +Brand#44|LARGE BRUSHED TIN|23|4 +Brand#44|LARGE BURNISHED BRASS|23|4 +Brand#44|LARGE BURNISHED BRASS|49|4 +Brand#44|LARGE BURNISHED COPPER|3|4 +Brand#44|LARGE BURNISHED COPPER|19|4 +Brand#44|LARGE BURNISHED COPPER|36|4 +Brand#44|LARGE BURNISHED TIN|14|4 +Brand#44|LARGE PLATED BRASS|9|4 +Brand#44|LARGE PLATED BRASS|49|4 +Brand#44|LARGE PLATED NICKEL|14|4 +Brand#44|LARGE PLATED STEEL|14|4 +Brand#44|LARGE PLATED TIN|19|4 +Brand#44|LARGE PLATED TIN|23|4 +Brand#44|LARGE POLISHED STEEL|23|4 +Brand#44|LARGE POLISHED STEEL|49|4 +Brand#44|MEDIUM ANODIZED COPPER|45|4 +Brand#44|MEDIUM ANODIZED NICKEL|45|4 +Brand#44|MEDIUM BRUSHED BRASS|49|4 +Brand#44|MEDIUM BRUSHED COPPER|3|4 +Brand#44|MEDIUM BRUSHED COPPER|45|4 +Brand#44|MEDIUM BRUSHED STEEL|19|4 +Brand#44|MEDIUM BRUSHED TIN|49|4 +Brand#44|MEDIUM BURNISHED COPPER|45|4 +Brand#44|MEDIUM BURNISHED NICKEL|23|4 +Brand#44|MEDIUM BURNISHED TIN|23|4 +Brand#44|MEDIUM PLATED COPPER|14|4 +Brand#44|PROMO ANODIZED COPPER|23|4 +Brand#44|PROMO ANODIZED STEEL|36|4 +Brand#44|PROMO BRUSHED COPPER|23|4 +Brand#44|PROMO BRUSHED COPPER|36|4 +Brand#44|PROMO BRUSHED TIN|19|4 +Brand#44|PROMO PLATED BRASS|3|4 +Brand#44|PROMO PLATED COPPER|36|4 +Brand#44|PROMO PLATED STEEL|3|4 +Brand#44|PROMO PLATED STEEL|36|4 +Brand#44|PROMO PLATED STEEL|49|4 +Brand#44|PROMO POLISHED BRASS|3|4 +Brand#44|PROMO POLISHED BRASS|19|4 +Brand#44|PROMO POLISHED COPPER|45|4 +Brand#44|PROMO POLISHED STEEL|36|4 +Brand#44|PROMO POLISHED TIN|9|4 +Brand#44|SMALL ANODIZED COPPER|23|4 +Brand#44|SMALL ANODIZED STEEL|23|4 +Brand#44|SMALL ANODIZED TIN|45|4 +Brand#44|SMALL BRUSHED COPPER|14|4 +Brand#44|SMALL BRUSHED STEEL|45|4 +Brand#44|SMALL BURNISHED COPPER|14|4 +Brand#44|SMALL BURNISHED COPPER|49|4 +Brand#44|SMALL BURNISHED NICKEL|14|4 +Brand#44|SMALL BURNISHED STEEL|23|4 +Brand#44|SMALL BURNISHED TIN|49|4 +Brand#44|SMALL PLATED BRASS|36|4 +Brand#44|SMALL PLATED COPPER|19|4 +Brand#44|SMALL PLATED NICKEL|3|4 +Brand#44|SMALL POLISHED COPPER|3|4 +Brand#44|SMALL POLISHED COPPER|49|4 +Brand#44|SMALL POLISHED STEEL|3|4 +Brand#44|STANDARD ANODIZED BRASS|3|4 +Brand#44|STANDARD ANODIZED COPPER|3|4 +Brand#44|STANDARD ANODIZED NICKEL|3|4 +Brand#44|STANDARD ANODIZED NICKEL|36|4 +Brand#44|STANDARD ANODIZED STEEL|14|4 +Brand#44|STANDARD ANODIZED TIN|3|4 +Brand#44|STANDARD ANODIZED TIN|9|4 +Brand#44|STANDARD ANODIZED TIN|36|4 +Brand#44|STANDARD BRUSHED COPPER|36|4 +Brand#44|STANDARD BRUSHED COPPER|45|4 +Brand#44|STANDARD BRUSHED TIN|9|4 +Brand#44|STANDARD BRUSHED TIN|49|4 +Brand#44|STANDARD BURNISHED COPPER|9|4 +Brand#44|STANDARD BURNISHED STEEL|23|4 +Brand#44|STANDARD PLATED BRASS|14|4 +Brand#44|STANDARD PLATED BRASS|23|4 +Brand#44|STANDARD PLATED BRASS|49|4 +Brand#44|STANDARD PLATED COPPER|14|4 +Brand#44|STANDARD POLISHED NICKEL|19|4 +Brand#44|STANDARD POLISHED TIN|9|4 +Brand#51|ECONOMY ANODIZED BRASS|9|4 +Brand#51|ECONOMY ANODIZED BRASS|23|4 +Brand#51|ECONOMY ANODIZED NICKEL|3|4 +Brand#51|ECONOMY ANODIZED NICKEL|23|4 +Brand#51|ECONOMY ANODIZED STEEL|19|4 +Brand#51|ECONOMY ANODIZED STEEL|23|4 +Brand#51|ECONOMY ANODIZED STEEL|49|4 +Brand#51|ECONOMY BRUSHED BRASS|3|4 +Brand#51|ECONOMY BRUSHED BRASS|49|4 +Brand#51|ECONOMY BRUSHED NICKEL|14|4 +Brand#51|ECONOMY BRUSHED STEEL|45|4 +Brand#51|ECONOMY BRUSHED TIN|36|4 +Brand#51|ECONOMY BURNISHED BRASS|14|4 +Brand#51|ECONOMY BURNISHED COPPER|45|4 +Brand#51|ECONOMY PLATED NICKEL|49|4 +Brand#51|ECONOMY PLATED TIN|36|4 +Brand#51|ECONOMY POLISHED COPPER|9|4 +Brand#51|ECONOMY POLISHED STEEL|14|4 +Brand#51|ECONOMY POLISHED STEEL|49|4 +Brand#51|LARGE ANODIZED COPPER|9|4 +Brand#51|LARGE ANODIZED COPPER|49|4 +Brand#51|LARGE ANODIZED NICKEL|14|4 +Brand#51|LARGE ANODIZED STEEL|36|4 +Brand#51|LARGE BRUSHED NICKEL|3|4 +Brand#51|LARGE BRUSHED NICKEL|9|4 +Brand#51|LARGE BURNISHED BRASS|19|4 +Brand#51|LARGE BURNISHED BRASS|36|4 +Brand#51|LARGE BURNISHED COPPER|14|4 +Brand#51|LARGE BURNISHED NICKEL|14|4 +Brand#51|LARGE PLATED BRASS|36|4 +Brand#51|LARGE POLISHED COPPER|14|4 +Brand#51|LARGE POLISHED NICKEL|23|4 +Brand#51|LARGE POLISHED NICKEL|36|4 +Brand#51|LARGE POLISHED STEEL|19|4 +Brand#51|MEDIUM ANODIZED COPPER|9|4 +Brand#51|MEDIUM ANODIZED STEEL|3|4 +Brand#51|MEDIUM BRUSHED BRASS|36|4 +Brand#51|MEDIUM BRUSHED BRASS|45|4 +Brand#51|MEDIUM BRUSHED STEEL|3|4 +Brand#51|MEDIUM BRUSHED TIN|36|4 +Brand#51|MEDIUM BURNISHED NICKEL|3|4 +Brand#51|MEDIUM BURNISHED NICKEL|36|4 +Brand#51|MEDIUM BURNISHED STEEL|14|4 +Brand#51|MEDIUM BURNISHED TIN|9|4 +Brand#51|MEDIUM PLATED STEEL|19|4 +Brand#51|MEDIUM PLATED TIN|3|4 +Brand#51|PROMO ANODIZED NICKEL|14|4 +Brand#51|PROMO ANODIZED STEEL|23|4 +Brand#51|PROMO ANODIZED TIN|19|4 +Brand#51|PROMO BRUSHED BRASS|23|4 +Brand#51|PROMO BRUSHED COPPER|45|4 +Brand#51|PROMO BRUSHED STEEL|45|4 +Brand#51|PROMO BRUSHED TIN|9|4 +Brand#51|PROMO BURNISHED BRASS|19|4 +Brand#51|PROMO BURNISHED BRASS|23|4 +Brand#51|PROMO BURNISHED NICKEL|14|4 +Brand#51|PROMO PLATED BRASS|3|4 +Brand#51|PROMO PLATED BRASS|23|4 +Brand#51|PROMO PLATED TIN|19|4 +Brand#51|PROMO PLATED TIN|23|4 +Brand#51|PROMO POLISHED BRASS|23|4 +Brand#51|PROMO POLISHED COPPER|9|4 +Brand#51|PROMO POLISHED NICKEL|9|4 +Brand#51|PROMO POLISHED STEEL|49|4 +Brand#51|SMALL ANODIZED STEEL|14|4 +Brand#51|SMALL BRUSHED BRASS|23|4 +Brand#51|SMALL BRUSHED TIN|19|4 +Brand#51|SMALL BURNISHED NICKEL|23|4 +Brand#51|SMALL PLATED COPPER|49|4 +Brand#51|SMALL PLATED NICKEL|3|4 +Brand#51|SMALL PLATED NICKEL|14|4 +Brand#51|SMALL PLATED STEEL|45|4 +Brand#51|SMALL POLISHED NICKEL|14|4 +Brand#51|SMALL POLISHED NICKEL|23|4 +Brand#51|SMALL POLISHED STEEL|3|4 +Brand#51|SMALL POLISHED STEEL|19|4 +Brand#51|SMALL POLISHED STEEL|49|4 +Brand#51|STANDARD ANODIZED NICKEL|3|4 +Brand#51|STANDARD ANODIZED NICKEL|49|4 +Brand#51|STANDARD BRUSHED BRASS|3|4 +Brand#51|STANDARD BRUSHED COPPER|3|4 +Brand#51|STANDARD BRUSHED NICKEL|19|4 +Brand#51|STANDARD BRUSHED STEEL|36|4 +Brand#51|STANDARD BURNISHED COPPER|19|4 +Brand#51|STANDARD BURNISHED NICKEL|49|4 +Brand#51|STANDARD BURNISHED STEEL|23|4 +Brand#51|STANDARD BURNISHED STEEL|36|4 +Brand#51|STANDARD BURNISHED TIN|45|4 +Brand#51|STANDARD PLATED BRASS|36|4 +Brand#51|STANDARD PLATED BRASS|49|4 +Brand#51|STANDARD PLATED COPPER|14|4 +Brand#51|STANDARD PLATED COPPER|23|4 +Brand#51|STANDARD POLISHED BRASS|14|4 +Brand#51|STANDARD POLISHED BRASS|45|4 +Brand#51|STANDARD POLISHED STEEL|36|4 +Brand#51|STANDARD POLISHED STEEL|49|4 +Brand#51|STANDARD POLISHED TIN|45|4 +Brand#52|ECONOMY ANODIZED BRASS|14|4 +Brand#52|ECONOMY ANODIZED BRASS|23|4 +Brand#52|ECONOMY ANODIZED COPPER|36|4 +Brand#52|ECONOMY ANODIZED NICKEL|49|4 +Brand#52|ECONOMY ANODIZED STEEL|19|4 +Brand#52|ECONOMY BRUSHED COPPER|49|4 +Brand#52|ECONOMY BURNISHED BRASS|36|4 +Brand#52|ECONOMY BURNISHED COPPER|19|4 +Brand#52|ECONOMY BURNISHED COPPER|45|4 +Brand#52|ECONOMY BURNISHED NICKEL|19|4 +Brand#52|ECONOMY BURNISHED STEEL|36|4 +Brand#52|ECONOMY PLATED TIN|14|4 +Brand#52|ECONOMY PLATED TIN|23|4 +Brand#52|ECONOMY POLISHED BRASS|23|4 +Brand#52|ECONOMY POLISHED BRASS|45|4 +Brand#52|ECONOMY POLISHED NICKEL|36|4 +Brand#52|ECONOMY POLISHED STEEL|49|4 +Brand#52|LARGE ANODIZED COPPER|14|4 +Brand#52|LARGE ANODIZED NICKEL|3|4 +Brand#52|LARGE ANODIZED NICKEL|45|4 +Brand#52|LARGE ANODIZED TIN|45|4 +Brand#52|LARGE BRUSHED COPPER|19|4 +Brand#52|LARGE BRUSHED NICKEL|3|4 +Brand#52|LARGE BRUSHED NICKEL|19|4 +Brand#52|LARGE BRUSHED NICKEL|23|4 +Brand#52|LARGE BRUSHED STEEL|49|4 +Brand#52|LARGE BRUSHED TIN|14|4 +Brand#52|LARGE BURNISHED NICKEL|9|4 +Brand#52|LARGE BURNISHED TIN|23|4 +Brand#52|LARGE BURNISHED TIN|45|4 +Brand#52|LARGE PLATED BRASS|14|4 +Brand#52|LARGE PLATED COPPER|14|4 +Brand#52|LARGE PLATED COPPER|19|4 +Brand#52|LARGE PLATED NICKEL|45|4 +Brand#52|LARGE PLATED STEEL|9|4 +Brand#52|LARGE PLATED TIN|9|4 +Brand#52|LARGE POLISHED NICKEL|19|4 +Brand#52|LARGE POLISHED NICKEL|23|4 +Brand#52|LARGE POLISHED NICKEL|36|4 +Brand#52|LARGE POLISHED TIN|9|4 +Brand#52|MEDIUM ANODIZED COPPER|36|4 +Brand#52|MEDIUM ANODIZED STEEL|14|4 +Brand#52|MEDIUM ANODIZED TIN|3|4 +Brand#52|MEDIUM ANODIZED TIN|49|4 +Brand#52|MEDIUM BRUSHED COPPER|9|4 +Brand#52|MEDIUM BRUSHED NICKEL|9|4 +Brand#52|MEDIUM BRUSHED STEEL|23|4 +Brand#52|MEDIUM BRUSHED STEEL|49|4 +Brand#52|MEDIUM BURNISHED STEEL|23|4 +Brand#52|MEDIUM BURNISHED TIN|45|4 +Brand#52|MEDIUM BURNISHED TIN|49|4 +Brand#52|MEDIUM PLATED BRASS|36|4 +Brand#52|MEDIUM PLATED STEEL|9|4 +Brand#52|MEDIUM PLATED STEEL|49|4 +Brand#52|MEDIUM PLATED TIN|9|4 +Brand#52|MEDIUM PLATED TIN|49|4 +Brand#52|PROMO ANODIZED BRASS|9|4 +Brand#52|PROMO ANODIZED BRASS|23|4 +Brand#52|PROMO ANODIZED BRASS|36|4 +Brand#52|PROMO ANODIZED NICKEL|45|4 +Brand#52|PROMO ANODIZED STEEL|36|4 +Brand#52|PROMO BRUSHED COPPER|3|4 +Brand#52|PROMO BRUSHED NICKEL|3|4 +Brand#52|PROMO BRUSHED NICKEL|49|4 +Brand#52|PROMO BRUSHED STEEL|14|4 +Brand#52|PROMO BRUSHED TIN|3|4 +Brand#52|PROMO BRUSHED TIN|19|4 +Brand#52|PROMO BRUSHED TIN|36|4 +Brand#52|PROMO BURNISHED COPPER|49|4 +Brand#52|PROMO BURNISHED NICKEL|9|4 +Brand#52|PROMO BURNISHED STEEL|9|4 +Brand#52|PROMO BURNISHED STEEL|23|4 +Brand#52|PROMO BURNISHED TIN|19|4 +Brand#52|PROMO BURNISHED TIN|36|4 +Brand#52|PROMO PLATED BRASS|19|4 +Brand#52|PROMO PLATED BRASS|45|4 +Brand#52|PROMO PLATED BRASS|49|4 +Brand#52|PROMO PLATED COPPER|9|4 +Brand#52|PROMO PLATED NICKEL|3|4 +Brand#52|PROMO PLATED NICKEL|23|4 +Brand#52|PROMO POLISHED NICKEL|14|4 +Brand#52|PROMO POLISHED NICKEL|49|4 +Brand#52|PROMO POLISHED TIN|36|4 +Brand#52|SMALL ANODIZED BRASS|3|4 +Brand#52|SMALL ANODIZED BRASS|14|4 +Brand#52|SMALL ANODIZED COPPER|3|4 +Brand#52|SMALL ANODIZED NICKEL|36|4 +Brand#52|SMALL ANODIZED STEEL|9|4 +Brand#52|SMALL ANODIZED STEEL|19|4 +Brand#52|SMALL BRUSHED NICKEL|19|4 +Brand#52|SMALL BRUSHED STEEL|23|4 +Brand#52|SMALL BRUSHED TIN|14|4 +Brand#52|SMALL BRUSHED TIN|19|4 +Brand#52|SMALL BURNISHED NICKEL|14|4 +Brand#52|SMALL BURNISHED NICKEL|49|4 +Brand#52|SMALL BURNISHED TIN|9|4 +Brand#52|SMALL POLISHED BRASS|36|4 +Brand#52|SMALL POLISHED BRASS|49|4 +Brand#52|SMALL POLISHED TIN|45|4 +Brand#52|STANDARD ANODIZED BRASS|45|4 +Brand#52|STANDARD BRUSHED BRASS|23|4 +Brand#52|STANDARD BRUSHED COPPER|14|4 +Brand#52|STANDARD BRUSHED TIN|36|4 +Brand#52|STANDARD BURNISHED BRASS|49|4 +Brand#52|STANDARD BURNISHED STEEL|19|4 +Brand#52|STANDARD BURNISHED TIN|9|4 +Brand#52|STANDARD BURNISHED TIN|19|4 +Brand#52|STANDARD PLATED NICKEL|36|4 +Brand#52|STANDARD PLATED STEEL|36|4 +Brand#52|STANDARD POLISHED BRASS|36|4 +Brand#52|STANDARD POLISHED COPPER|45|4 +Brand#52|STANDARD POLISHED STEEL|19|4 +Brand#52|STANDARD POLISHED TIN|19|4 +Brand#53|ECONOMY ANODIZED BRASS|45|4 +Brand#53|ECONOMY ANODIZED COPPER|9|4 +Brand#53|ECONOMY ANODIZED NICKEL|3|4 +Brand#53|ECONOMY ANODIZED NICKEL|19|4 +Brand#53|ECONOMY ANODIZED STEEL|45|4 +Brand#53|ECONOMY ANODIZED TIN|14|4 +Brand#53|ECONOMY ANODIZED TIN|36|4 +Brand#53|ECONOMY BRUSHED TIN|45|4 +Brand#53|ECONOMY BURNISHED BRASS|14|4 +Brand#53|ECONOMY BURNISHED COPPER|45|4 +Brand#53|ECONOMY BURNISHED NICKEL|3|4 +Brand#53|ECONOMY BURNISHED NICKEL|49|4 +Brand#53|ECONOMY BURNISHED TIN|45|4 +Brand#53|ECONOMY PLATED BRASS|3|4 +Brand#53|ECONOMY PLATED NICKEL|14|4 +Brand#53|ECONOMY PLATED STEEL|23|4 +Brand#53|ECONOMY PLATED STEEL|36|4 +Brand#53|ECONOMY POLISHED TIN|36|4 +Brand#53|LARGE ANODIZED NICKEL|49|4 +Brand#53|LARGE ANODIZED STEEL|19|4 +Brand#53|LARGE BRUSHED COPPER|3|4 +Brand#53|LARGE BRUSHED COPPER|14|4 +Brand#53|LARGE BRUSHED NICKEL|23|4 +Brand#53|LARGE BRUSHED NICKEL|36|4 +Brand#53|LARGE BRUSHED TIN|36|4 +Brand#53|LARGE BURNISHED BRASS|45|4 +Brand#53|LARGE BURNISHED COPPER|19|4 +Brand#53|LARGE BURNISHED COPPER|36|4 +Brand#53|LARGE BURNISHED NICKEL|23|4 +Brand#53|LARGE BURNISHED STEEL|19|4 +Brand#53|LARGE BURNISHED STEEL|23|4 +Brand#53|LARGE PLATED BRASS|9|4 +Brand#53|LARGE PLATED BRASS|45|4 +Brand#53|LARGE PLATED BRASS|49|4 +Brand#53|LARGE PLATED COPPER|23|4 +Brand#53|LARGE PLATED NICKEL|23|4 +Brand#53|LARGE PLATED NICKEL|49|4 +Brand#53|LARGE PLATED STEEL|49|4 +Brand#53|LARGE PLATED TIN|14|4 +Brand#53|LARGE POLISHED COPPER|49|4 +Brand#53|LARGE POLISHED STEEL|36|4 +Brand#53|LARGE POLISHED TIN|9|4 +Brand#53|MEDIUM ANODIZED BRASS|23|4 +Brand#53|MEDIUM ANODIZED STEEL|14|4 +Brand#53|MEDIUM ANODIZED STEEL|36|4 +Brand#53|MEDIUM ANODIZED TIN|3|4 +Brand#53|MEDIUM ANODIZED TIN|9|4 +Brand#53|MEDIUM BRUSHED BRASS|3|4 +Brand#53|MEDIUM BRUSHED COPPER|3|4 +Brand#53|MEDIUM BRUSHED NICKEL|14|4 +Brand#53|MEDIUM BRUSHED NICKEL|36|4 +Brand#53|MEDIUM BRUSHED NICKEL|49|4 +Brand#53|MEDIUM BRUSHED STEEL|45|4 +Brand#53|MEDIUM BURNISHED BRASS|3|4 +Brand#53|MEDIUM BURNISHED BRASS|36|4 +Brand#53|MEDIUM BURNISHED TIN|9|4 +Brand#53|MEDIUM BURNISHED TIN|14|4 +Brand#53|MEDIUM BURNISHED TIN|36|4 +Brand#53|MEDIUM PLATED BRASS|23|4 +Brand#53|MEDIUM PLATED COPPER|14|4 +Brand#53|MEDIUM PLATED NICKEL|45|4 +Brand#53|MEDIUM PLATED TIN|19|4 +Brand#53|MEDIUM PLATED TIN|45|4 +Brand#53|PROMO ANODIZED BRASS|36|4 +Brand#53|PROMO ANODIZED NICKEL|3|4 +Brand#53|PROMO ANODIZED NICKEL|19|4 +Brand#53|PROMO BRUSHED BRASS|45|4 +Brand#53|PROMO BRUSHED COPPER|3|4 +Brand#53|PROMO BRUSHED COPPER|23|4 +Brand#53|PROMO BRUSHED COPPER|45|4 +Brand#53|PROMO BURNISHED BRASS|23|4 +Brand#53|PROMO BURNISHED BRASS|36|4 +Brand#53|PROMO BURNISHED NICKEL|23|4 +Brand#53|PROMO BURNISHED STEEL|23|4 +Brand#53|PROMO BURNISHED STEEL|49|4 +Brand#53|PROMO PLATED TIN|19|4 +Brand#53|PROMO PLATED TIN|23|4 +Brand#53|PROMO PLATED TIN|36|4 +Brand#53|PROMO POLISHED STEEL|23|4 +Brand#53|PROMO POLISHED TIN|3|4 +Brand#53|SMALL ANODIZED COPPER|23|4 +Brand#53|SMALL ANODIZED COPPER|36|4 +Brand#53|SMALL ANODIZED COPPER|49|4 +Brand#53|SMALL ANODIZED NICKEL|36|4 +Brand#53|SMALL BRUSHED BRASS|36|4 +Brand#53|SMALL BRUSHED COPPER|3|4 +Brand#53|SMALL BRUSHED TIN|3|4 +Brand#53|SMALL BRUSHED TIN|36|4 +Brand#53|SMALL BURNISHED BRASS|9|4 +Brand#53|SMALL BURNISHED BRASS|49|4 +Brand#53|SMALL BURNISHED COPPER|19|4 +Brand#53|SMALL BURNISHED COPPER|45|4 +Brand#53|SMALL PLATED BRASS|9|4 +Brand#53|SMALL PLATED COPPER|3|4 +Brand#53|SMALL PLATED NICKEL|14|4 +Brand#53|SMALL POLISHED NICKEL|19|4 +Brand#53|SMALL POLISHED STEEL|36|4 +Brand#53|SMALL POLISHED TIN|23|4 +Brand#53|STANDARD ANODIZED BRASS|14|4 +Brand#53|STANDARD ANODIZED NICKEL|9|4 +Brand#53|STANDARD ANODIZED NICKEL|23|4 +Brand#53|STANDARD ANODIZED NICKEL|45|4 +Brand#53|STANDARD ANODIZED STEEL|45|4 +Brand#53|STANDARD BRUSHED COPPER|3|4 +Brand#53|STANDARD BRUSHED NICKEL|23|4 +Brand#53|STANDARD BRUSHED TIN|14|4 +Brand#53|STANDARD BURNISHED NICKEL|49|4 +Brand#53|STANDARD BURNISHED STEEL|9|4 +Brand#53|STANDARD PLATED BRASS|36|4 +Brand#53|STANDARD PLATED COPPER|45|4 +Brand#53|STANDARD PLATED NICKEL|36|4 +Brand#53|STANDARD PLATED STEEL|3|4 +Brand#53|STANDARD PLATED STEEL|49|4 +Brand#53|STANDARD PLATED TIN|23|4 +Brand#53|STANDARD POLISHED STEEL|3|4 +Brand#54|ECONOMY ANODIZED BRASS|9|4 +Brand#54|ECONOMY ANODIZED BRASS|45|4 +Brand#54|ECONOMY ANODIZED COPPER|9|4 +Brand#54|ECONOMY ANODIZED STEEL|19|4 +Brand#54|ECONOMY BRUSHED BRASS|45|4 +Brand#54|ECONOMY BRUSHED NICKEL|19|4 +Brand#54|ECONOMY BRUSHED STEEL|3|4 +Brand#54|ECONOMY BRUSHED TIN|19|4 +Brand#54|ECONOMY BURNISHED BRASS|45|4 +Brand#54|ECONOMY BURNISHED COPPER|14|4 +Brand#54|ECONOMY BURNISHED NICKEL|9|4 +Brand#54|ECONOMY BURNISHED NICKEL|36|4 +Brand#54|ECONOMY BURNISHED STEEL|36|4 +Brand#54|ECONOMY BURNISHED TIN|9|4 +Brand#54|ECONOMY BURNISHED TIN|14|4 +Brand#54|ECONOMY BURNISHED TIN|23|4 +Brand#54|ECONOMY PLATED TIN|23|4 +Brand#54|ECONOMY POLISHED BRASS|9|4 +Brand#54|ECONOMY POLISHED BRASS|19|4 +Brand#54|ECONOMY POLISHED COPPER|23|4 +Brand#54|ECONOMY POLISHED STEEL|23|4 +Brand#54|ECONOMY POLISHED TIN|3|4 +Brand#54|LARGE ANODIZED BRASS|14|4 +Brand#54|LARGE ANODIZED BRASS|49|4 +Brand#54|LARGE ANODIZED TIN|9|4 +Brand#54|LARGE BRUSHED BRASS|14|4 +Brand#54|LARGE BRUSHED STEEL|9|4 +Brand#54|LARGE BRUSHED STEEL|23|4 +Brand#54|LARGE BRUSHED TIN|14|4 +Brand#54|LARGE BURNISHED BRASS|49|4 +Brand#54|LARGE BURNISHED COPPER|19|4 +Brand#54|LARGE BURNISHED NICKEL|14|4 +Brand#54|LARGE BURNISHED TIN|14|4 +Brand#54|LARGE PLATED BRASS|19|4 +Brand#54|LARGE PLATED BRASS|23|4 +Brand#54|LARGE POLISHED BRASS|19|4 +Brand#54|LARGE POLISHED BRASS|23|4 +Brand#54|LARGE POLISHED NICKEL|3|4 +Brand#54|LARGE POLISHED NICKEL|14|4 +Brand#54|LARGE POLISHED STEEL|19|4 +Brand#54|LARGE POLISHED TIN|3|4 +Brand#54|LARGE POLISHED TIN|9|4 +Brand#54|LARGE POLISHED TIN|36|4 +Brand#54|MEDIUM ANODIZED NICKEL|9|4 +Brand#54|MEDIUM ANODIZED NICKEL|14|4 +Brand#54|MEDIUM ANODIZED NICKEL|36|4 +Brand#54|MEDIUM BRUSHED NICKEL|9|4 +Brand#54|MEDIUM BRUSHED NICKEL|19|4 +Brand#54|MEDIUM BURNISHED STEEL|3|4 +Brand#54|MEDIUM BURNISHED STEEL|19|4 +Brand#54|MEDIUM BURNISHED STEEL|23|4 +Brand#54|MEDIUM PLATED BRASS|3|4 +Brand#54|MEDIUM PLATED NICKEL|45|4 +Brand#54|PROMO ANODIZED NICKEL|45|4 +Brand#54|PROMO BRUSHED BRASS|3|4 +Brand#54|PROMO BRUSHED STEEL|23|4 +Brand#54|PROMO BRUSHED TIN|14|4 +Brand#54|PROMO BURNISHED COPPER|49|4 +Brand#54|PROMO BURNISHED TIN|9|4 +Brand#54|PROMO PLATED BRASS|14|4 +Brand#54|PROMO PLATED NICKEL|3|4 +Brand#54|PROMO PLATED STEEL|19|4 +Brand#54|PROMO PLATED TIN|23|4 +Brand#54|PROMO PLATED TIN|49|4 +Brand#54|PROMO POLISHED BRASS|3|4 +Brand#54|PROMO POLISHED NICKEL|9|4 +Brand#54|PROMO POLISHED TIN|49|4 +Brand#54|SMALL ANODIZED COPPER|49|4 +Brand#54|SMALL ANODIZED NICKEL|9|4 +Brand#54|SMALL ANODIZED NICKEL|36|4 +Brand#54|SMALL ANODIZED TIN|19|4 +Brand#54|SMALL BRUSHED BRASS|14|4 +Brand#54|SMALL BRUSHED BRASS|19|4 +Brand#54|SMALL BRUSHED BRASS|36|4 +Brand#54|SMALL BRUSHED COPPER|3|4 +Brand#54|SMALL BRUSHED COPPER|9|4 +Brand#54|SMALL BRUSHED COPPER|19|4 +Brand#54|SMALL BRUSHED TIN|9|4 +Brand#54|SMALL BRUSHED TIN|36|4 +Brand#54|SMALL BURNISHED COPPER|9|4 +Brand#54|SMALL BURNISHED COPPER|36|4 +Brand#54|SMALL BURNISHED STEEL|14|4 +Brand#54|SMALL BURNISHED STEEL|19|4 +Brand#54|SMALL BURNISHED TIN|9|4 +Brand#54|SMALL BURNISHED TIN|36|4 +Brand#54|SMALL PLATED BRASS|23|4 +Brand#54|SMALL PLATED COPPER|9|4 +Brand#54|SMALL PLATED COPPER|36|4 +Brand#54|SMALL PLATED COPPER|49|4 +Brand#54|SMALL PLATED NICKEL|9|4 +Brand#54|SMALL PLATED TIN|23|4 +Brand#54|SMALL PLATED TIN|36|4 +Brand#54|SMALL POLISHED BRASS|9|4 +Brand#54|SMALL POLISHED COPPER|9|4 +Brand#54|SMALL POLISHED TIN|9|4 +Brand#54|STANDARD ANODIZED BRASS|3|4 +Brand#54|STANDARD ANODIZED BRASS|9|4 +Brand#54|STANDARD ANODIZED COPPER|3|4 +Brand#54|STANDARD ANODIZED TIN|3|4 +Brand#54|STANDARD BRUSHED COPPER|3|4 +Brand#54|STANDARD BRUSHED NICKEL|45|4 +Brand#54|STANDARD BRUSHED TIN|36|4 +Brand#54|STANDARD BURNISHED BRASS|23|4 +Brand#54|STANDARD BURNISHED BRASS|49|4 +Brand#54|STANDARD BURNISHED COPPER|19|4 +Brand#54|STANDARD BURNISHED NICKEL|23|4 +Brand#54|STANDARD BURNISHED STEEL|45|4 +Brand#54|STANDARD PLATED BRASS|3|4 +Brand#54|STANDARD PLATED BRASS|45|4 +Brand#54|STANDARD PLATED BRASS|49|4 +Brand#54|STANDARD PLATED STEEL|3|4 +Brand#54|STANDARD POLISHED BRASS|36|4 +Brand#54|STANDARD POLISHED STEEL|3|4 +Brand#54|STANDARD POLISHED STEEL|14|4 +Brand#54|STANDARD POLISHED STEEL|45|4 +Brand#55|ECONOMY ANODIZED BRASS|3|4 +Brand#55|ECONOMY BRUSHED BRASS|19|4 +Brand#55|ECONOMY BRUSHED COPPER|9|4 +Brand#55|ECONOMY BRUSHED COPPER|23|4 +Brand#55|ECONOMY BRUSHED COPPER|45|4 +Brand#55|ECONOMY BRUSHED STEEL|23|4 +Brand#55|ECONOMY BURNISHED NICKEL|36|4 +Brand#55|ECONOMY BURNISHED NICKEL|45|4 +Brand#55|ECONOMY BURNISHED TIN|45|4 +Brand#55|ECONOMY PLATED NICKEL|19|4 +Brand#55|ECONOMY POLISHED NICKEL|9|4 +Brand#55|LARGE BRUSHED BRASS|23|4 +Brand#55|LARGE BRUSHED BRASS|45|4 +Brand#55|LARGE BRUSHED COPPER|49|4 +Brand#55|LARGE BRUSHED NICKEL|9|4 +Brand#55|LARGE BRUSHED NICKEL|14|4 +Brand#55|LARGE BURNISHED BRASS|3|4 +Brand#55|LARGE BURNISHED COPPER|14|4 +Brand#55|LARGE BURNISHED COPPER|36|4 +Brand#55|LARGE PLATED BRASS|45|4 +Brand#55|LARGE PLATED COPPER|19|4 +Brand#55|LARGE PLATED NICKEL|9|4 +Brand#55|LARGE PLATED STEEL|9|4 +Brand#55|LARGE PLATED TIN|9|4 +Brand#55|LARGE PLATED TIN|14|4 +Brand#55|LARGE PLATED TIN|23|4 +Brand#55|LARGE POLISHED NICKEL|3|4 +Brand#55|LARGE POLISHED STEEL|36|4 +Brand#55|LARGE POLISHED STEEL|45|4 +Brand#55|MEDIUM ANODIZED COPPER|9|4 +Brand#55|MEDIUM BRUSHED BRASS|3|4 +Brand#55|MEDIUM BRUSHED NICKEL|23|4 +Brand#55|MEDIUM BRUSHED TIN|45|4 +Brand#55|MEDIUM BURNISHED BRASS|23|4 +Brand#55|MEDIUM BURNISHED COPPER|36|4 +Brand#55|MEDIUM BURNISHED NICKEL|3|4 +Brand#55|MEDIUM BURNISHED STEEL|14|4 +Brand#55|MEDIUM BURNISHED STEEL|36|4 +Brand#55|MEDIUM PLATED NICKEL|23|4 +Brand#55|PROMO ANODIZED COPPER|14|4 +Brand#55|PROMO ANODIZED COPPER|49|4 +Brand#55|PROMO ANODIZED STEEL|36|4 +Brand#55|PROMO ANODIZED TIN|23|4 +Brand#55|PROMO BRUSHED NICKEL|36|4 +Brand#55|PROMO BRUSHED STEEL|3|4 +Brand#55|PROMO BRUSHED STEEL|36|4 +Brand#55|PROMO BRUSHED TIN|9|4 +Brand#55|PROMO BURNISHED COPPER|3|4 +Brand#55|PROMO BURNISHED STEEL|14|4 +Brand#55|PROMO BURNISHED TIN|23|4 +Brand#55|PROMO BURNISHED TIN|49|4 +Brand#55|PROMO PLATED COPPER|3|4 +Brand#55|PROMO PLATED NICKEL|3|4 +Brand#55|PROMO PLATED NICKEL|14|4 +Brand#55|PROMO PLATED NICKEL|23|4 +Brand#55|PROMO PLATED TIN|3|4 +Brand#55|PROMO POLISHED COPPER|3|4 +Brand#55|SMALL ANODIZED BRASS|19|4 +Brand#55|SMALL ANODIZED NICKEL|45|4 +Brand#55|SMALL BRUSHED COPPER|14|4 +Brand#55|SMALL BRUSHED COPPER|45|4 +Brand#55|SMALL BURNISHED BRASS|14|4 +Brand#55|SMALL BURNISHED TIN|3|4 +Brand#55|SMALL BURNISHED TIN|49|4 +Brand#55|SMALL PLATED BRASS|45|4 +Brand#55|SMALL PLATED COPPER|23|4 +Brand#55|SMALL PLATED COPPER|36|4 +Brand#55|SMALL PLATED COPPER|45|4 +Brand#55|SMALL PLATED COPPER|49|4 +Brand#55|SMALL PLATED NICKEL|9|4 +Brand#55|SMALL PLATED STEEL|9|4 +Brand#55|SMALL PLATED TIN|14|4 +Brand#55|SMALL PLATED TIN|36|4 +Brand#55|SMALL POLISHED NICKEL|45|4 +Brand#55|SMALL POLISHED STEEL|19|4 +Brand#55|SMALL POLISHED TIN|19|4 +Brand#55|STANDARD ANODIZED BRASS|36|4 +Brand#55|STANDARD ANODIZED BRASS|49|4 +Brand#55|STANDARD ANODIZED STEEL|19|4 +Brand#55|STANDARD ANODIZED TIN|36|4 +Brand#55|STANDARD ANODIZED TIN|49|4 +Brand#55|STANDARD BRUSHED BRASS|36|4 +Brand#55|STANDARD BRUSHED COPPER|3|4 +Brand#55|STANDARD BRUSHED COPPER|9|4 +Brand#55|STANDARD BRUSHED COPPER|23|4 +Brand#55|STANDARD BRUSHED STEEL|19|4 +Brand#55|STANDARD BRUSHED TIN|23|4 +Brand#55|STANDARD BRUSHED TIN|45|4 +Brand#55|STANDARD BURNISHED BRASS|19|4 +Brand#55|STANDARD BURNISHED NICKEL|3|4 +Brand#55|STANDARD BURNISHED NICKEL|36|4 +Brand#55|STANDARD BURNISHED STEEL|19|4 +Brand#55|STANDARD PLATED BRASS|23|4 +Brand#55|STANDARD PLATED NICKEL|9|4 +Brand#55|STANDARD PLATED TIN|36|4 +Brand#55|STANDARD POLISHED BRASS|3|4 +Brand#55|STANDARD POLISHED BRASS|49|4 +Brand#55|STANDARD POLISHED COPPER|19|4 +Brand#55|STANDARD POLISHED COPPER|36|4 +Brand#55|STANDARD POLISHED NICKEL|14|4 +Brand#55|STANDARD POLISHED STEEL|9|4 +Brand#55|STANDARD POLISHED STEEL|36|4 +Brand#12|LARGE BURNISHED NICKEL|14|3 +Brand#12|PROMO POLISHED TIN|3|3 +Brand#21|MEDIUM ANODIZED TIN|9|3 +Brand#22|PROMO BRUSHED BRASS|19|3 +Brand#22|PROMO BURNISHED COPPER|14|3 +Brand#43|STANDARD BRUSHED BRASS|23|3 +Brand#44|MEDIUM ANODIZED NICKEL|9|3 +Brand#53|MEDIUM BURNISHED BRASS|49|3 diff --git a/test/src/test/resources/tpch/sf0.1/q17.csv b/test/src/test/resources/tpch/sf0.1/q17.csv new file mode 100644 index 0000000000..f30f7424e6 --- /dev/null +++ b/test/src/test/resources/tpch/sf0.1/q17.csv @@ -0,0 +1,2 @@ +avg_yearly +23512.752857142856 diff --git a/test/src/test/resources/tpch/sf0.1/q18.csv b/test/src/test/resources/tpch/sf0.1/q18.csv new file mode 100644 index 0000000000..cea16c785b --- /dev/null +++ b/test/src/test/resources/tpch/sf0.1/q18.csv @@ -0,0 +1,6 @@ +c_name|c_custkey|o_orderkey|o_orderdate|o_totalprice|sum +Customer#000001639|1639|502886|1994-04-12|456423.88|312 +Customer#000006655|6655|29158|1995-10-21|452805.02|305 +Customer#000014110|14110|565574|1995-09-24|425099.85|301 +Customer#000001775|1775|6882|1997-04-09|408368.10|303 +Customer#000011459|11459|551136|1993-05-19|386812.74|308 diff --git a/test/src/test/resources/tpch/sf0.1/q19.csv b/test/src/test/resources/tpch/sf0.1/q19.csv new file mode 100644 index 0000000000..3c1e2a34d1 --- /dev/null +++ b/test/src/test/resources/tpch/sf0.1/q19.csv @@ -0,0 +1,2 @@ +revenue +168597.2860 diff --git a/test/src/test/resources/tpch/sf0.1/q2.csv b/test/src/test/resources/tpch/sf0.1/q2.csv new file mode 100644 index 0000000000..9c5612bba0 --- /dev/null +++ b/test/src/test/resources/tpch/sf0.1/q2.csv @@ -0,0 +1,45 @@ +s_acctbal|s_name|n_name|p_partkey|p_mfgr|s_address|s_phone|s_comment +9828.21|Supplier#000000647|UNITED KINGDOM|13120|Manufacturer#5|vV6Teq1EvLlR|33-258-202-4782|mong the carefully quiet accounts slee +9508.37|Supplier#000000070|FRANCE|3563|Manufacturer#1|jd4djZv0cc5KdnA0q9oOqvceaPUbNloOW|16-821-608-1166|n instructions are about the ironic, ironic excuses. instructions cajol +9508.37|Supplier#000000070|FRANCE|17268|Manufacturer#4|jd4djZv0cc5KdnA0q9oOqvceaPUbNloOW|16-821-608-1166|n instructions are about the ironic, ironic excuses. instructions cajol +9453.01|Supplier#000000802|ROMANIA|10021|Manufacturer#5|1Uj23QWxQjj7EyeqHWqGWTbN|29-342-882-6463|s according to the even deposits integrate express packages. express +9453.01|Supplier#000000802|ROMANIA|13275|Manufacturer#4|1Uj23QWxQjj7EyeqHWqGWTbN|29-342-882-6463|s according to the even deposits integrate express packages. express +9192.10|Supplier#000000115|UNITED KINGDOM|13325|Manufacturer#1|EhrYy0MT5M1vfZ0V4skpifdp6pgFz5|33-597-248-1220|onic instructions. ironic, regular deposits haggle f +9032.15|Supplier#000000959|GERMANY|4958|Manufacturer#4|TK qrnjpDvd1Jc|17-108-642-3106|nag across the slyly even pin +8702.02|Supplier#000000333|RUSSIA|11810|Manufacturer#3|fQ5Lr4KvbNHI3WDMhkcI S6xYtgIi1k|32-508-202-6136|ounts around the requests cajole furiously blithely even instructions. slyly +8615.50|Supplier#000000812|FRANCE|10551|Manufacturer#2|TAJWyNst8OGVPINgqtzwyyp002iYNDVub|16-585-724-6633|ress ideas eat quickly. blithely express deposits was slyly. final, +8615.50|Supplier#000000812|FRANCE|13811|Manufacturer#4|TAJWyNst8OGVPINgqtzwyyp002iYNDVub|16-585-724-6633|ress ideas eat quickly. blithely express deposits was slyly. final, +8488.53|Supplier#000000367|RUSSIA|6854|Manufacturer#4|nr8wRQ a5LXXess|32-458-198-9557|ect. quickly pending deposits sleep carefully even, express dependencies. +8430.52|Supplier#000000646|FRANCE|11384|Manufacturer#3|j6szE80YCpLHJ4bZ7F37gUiGhk0WJ0,8h9y|16-601-220-5489|quickly slyly even deposits. quickly ironic theodolites sleep fluffily after the c +8271.39|Supplier#000000146|RUSSIA|4637|Manufacturer#5|ApndKp ,Wu0 LNsoV0KldxyoIlY|32-792-619-3155|slyly regular foxes. unusual accounts about the regular packages +8096.98|Supplier#000000574|RUSSIA|323|Manufacturer#4|ZcSrzuRKYEGpcxmIsH,BrYBMwH0|32-866-246-8752|boost according to the slyly final instructions. furiously ironic packages cajole furiously +7392.78|Supplier#000000170|UNITED KINGDOM|7655|Manufacturer#2|ayz3a18xDGrr3jtS|33-803-340-5398|egular, even packages. pending, +7205.20|Supplier#000000477|GERMANY|10956|Manufacturer#5|6yQdgeVeAxJVtJTIYFNNWvQL|17-180-144-7991|ual accounts use quickly above the carefully quiet dolphins. packages nag closely. iro +6820.35|Supplier#000000007|UNITED KINGDOM|13217|Manufacturer#5| 0W7IPdkpWycUbQ9Adp6B|33-990-965-2201|ke across the slyly ironic packages. carefully special pinto beans wake blithely. even deposits los +6721.70|Supplier#000000954|FRANCE|4191|Manufacturer#3|cXcVBs6lsZbzfE14|16-537-341-8517|mong the quickly express pinto b +6329.90|Supplier#000000996|GERMANY|10735|Manufacturer#2|5uWNawcqv4IL8okyBL e|17-447-811-3282|deas. bold dinos are. carefully reg +6173.87|Supplier#000000408|RUSSIA|18139|Manufacturer#1|BOC Zy0wh3rCGHDgV0NIGt2dEK|32-858-724-2950| are carefully above the carefully final pinto beans. blithely express foxes ab +5364.99|Supplier#000000785|RUSSIA|13784|Manufacturer#4|5r5GjqBatnYAHaH5kB4IPcBEiglMJEnN4tUUG6k2|32-297-653-2203|se carefully after the bravely stealthy instru +5069.27|Supplier#000000328|GERMANY|16327|Manufacturer#1|9eEYWOr4kUZ|17-231-513-5721|es according to the slyly ironic package +4941.88|Supplier#000000321|ROMANIA|7320|Manufacturer#5|CfDKlGVtMePjtCw|29-573-279-1406| instructions boost carefu +4672.25|Supplier#000000239|RUSSIA|12238|Manufacturer#1|4cZ,ZHKj hRKgYlgZ6UapQ7mrEOozeQMx7KhUCS|32-396-654-6826|s wake fluffily slyly special foxes. ironic, bold +4586.49|Supplier#000000680|RUSSIA|5679|Manufacturer#3|7JwnLOmLhJ1aPMT61PSx9kcY77r,HmRUD314m|32-522-382-1620|e even pinto beans. blithely fluffy ideas cajole slyly around the bl +4518.31|Supplier#000000149|FRANCE|18344|Manufacturer#5|C5t4zIcINBkgBWdMg6WtgMtE|16-660-553-2456|silent platelets. ideas hinder carefully among the slyly regular deposits. slyly pending inst +4315.15|Supplier#000000509|FRANCE|18972|Manufacturer#2|9lTN9T5VBg|16-298-154-3365|ep boldly ironic theodolites. special dependencies lose blithely. final, regular packages wake +3526.53|Supplier#000000553|FRANCE|8036|Manufacturer#4|R0FI5DL3Poi|16-599-552-3755|l foxes wake slyly even f +3526.53|Supplier#000000553|FRANCE|17018|Manufacturer#3|R0FI5DL3Poi|16-599-552-3755|l foxes wake slyly even f +3294.68|Supplier#000000350|GERMANY|4841|Manufacturer#4|hilu5UXMCwFvJJ|17-113-181-4017|ronic ideas. blithely blithe accounts sleep blithely. regular requests boost carefully about the r +2972.26|Supplier#000000016|RUSSIA|1015|Manufacturer#4|3HbVoWVsjn4fTfQGgYTsMaDvMINBIDXqeBwK|32-822-502-4215|platelets thrash against the slyly special req +2963.09|Supplier#000000840|ROMANIA|3080|Manufacturer#2|J2s6iuBgJo03|29-781-337-5584|s sleep blithely unusual packages! even, bold accounts sleep slyly about the even +2221.25|Supplier#000000771|ROMANIA|13981|Manufacturer#2|Gv1ri,V ARHE136eJF|29-986-304-9006|lphins affix blithely along the carefully final ide +1381.97|Supplier#000000104|FRANCE|18103|Manufacturer#3|oOFWtl sAwYcbM9dWRPgKTS3Ebmn9Tcp3iz0F|16-434-972-6922|s. blithely pending requests against the regular instructions cajole sometimes according to the qu +906.07|Supplier#000000138|ROMANIA|8363|Manufacturer#4|yyPBFrErKTaEu5L3CdNJP ak4ys9AbN,Aj8wPgv|29-533-434-6776|deas haggle. final, regular packages wake. quiet packages cajole pinto beans +765.69|Supplier#000000799|RUSSIA|11276|Manufacturer#2|IvldT2pX7R el|32-579-339-1495| deposits: pending, unusual forges nag fluffily regular ideas +727.89|Supplier#000000470|ROMANIA|6213|Manufacturer#3|4OGPs qKpfQ6GNLIKhmbIE6e7fSMP8fmwi|29-165-289-1523|ly silent accounts. foxes maintain blithely along the idly +683.07|Supplier#000000651|RUSSIA|4888|Manufacturer#4|D4MGIq5Uz0,K|32-181-426-4490|ve to are slyly ironic asymptot +167.56|Supplier#000000290|FRANCE|2037|Manufacturer#1|VpG,Ul5yv1RgAK,,|16-675-286-5102| carefully furiously stealthy accounts. bold acc +91.39|Supplier#000000949|UNITED KINGDOM|9430|Manufacturer#2|R06m0VD95FZLoBJHcCMyaZQHitqmhZrQZkZk5|33-332-697-2768|sual requests. carefully regular requests bo +-314.06|Supplier#000000510|ROMANIA|17242|Manufacturer#4|6E3aFs0w2SiImzMDSewWtzOwdpLz2|29-207-852-3454|lyly regular accounts. deposits +-820.89|Supplier#000000409|GERMANY|2156|Manufacturer#5|gt362msTQ3AwtUVHgqP7Ryksk90dnpPNyn|17-719-517-9836|nal deposits doubt blithely regular packages. fr +-845.44|Supplier#000000704|ROMANIA|9926|Manufacturer#5|KawFpBPAADrVnKC,pLL9q3TSyHG9x|29-300-896-5991|ous pearls boost carefully +-942.73|Supplier#000000563|GERMANY|5797|Manufacturer#1|aOT6ZP96J2 ,Xhn|17-108-537-2691|are blithely silent requests. quickly even packages use blit diff --git a/test/src/test/resources/tpch/sf0.1/q20.csv b/test/src/test/resources/tpch/sf0.1/q20.csv new file mode 100644 index 0000000000..f1f9843c2e --- /dev/null +++ b/test/src/test/resources/tpch/sf0.1/q20.csv @@ -0,0 +1,10 @@ +s_name|s_address +Supplier#000000157|1EmkCApL5iF +Supplier#000000197|3oYqODDUGH3XsHXmPuzYHW5NLU3,ONZl +Supplier#000000287|UQR8bUA4V2HxVbw9K +Supplier#000000378|mLPJtpu4wOc cSFzBR +Supplier#000000530|0BvoewCPg2scOEfuL93FRKqSxHmdhw1 +Supplier#000000555|8Lp0QWPLFXrJrX1sTWkAEdzUsh5ke +Supplier#000000557|IH,v63JRgXMkVhJOJ Gxur0W +Supplier#000000729|CAOGYCBtTVT7aB1p6qHbxF6VVhXaHLgTpI +Supplier#000000935|JHRSOterYgt4MTNo7cupTzA,6MoNw 4 diff --git a/test/src/test/resources/tpch/sf0.1/q3.csv b/test/src/test/resources/tpch/sf0.1/q3.csv new file mode 100644 index 0000000000..123b6936ab --- /dev/null +++ b/test/src/test/resources/tpch/sf0.1/q3.csv @@ -0,0 +1,11 @@ +l_orderkey|revenue|o_orderdate|o_shippriority +223140|355369.0698|1995-03-14|0 +584291|354494.7318|1995-02-21|0 +405063|353125.4577|1995-03-03|0 +573861|351238.2770|1995-03-09|0 +554757|349181.7426|1995-03-14|0 +506021|321075.5810|1995-03-10|0 +121604|318576.4154|1995-03-07|0 +108514|314967.0754|1995-02-20|0 +462502|312604.5420|1995-03-08|0 +178727|309728.9306|1995-02-25|0 diff --git a/test/src/test/resources/tpch/sf0.1/q5.csv b/test/src/test/resources/tpch/sf0.1/q5.csv new file mode 100644 index 0000000000..b34430123f --- /dev/null +++ b/test/src/test/resources/tpch/sf0.1/q5.csv @@ -0,0 +1,6 @@ +n_name|revenue +CHINA|7822103.0000 +INDIA|6376121.5085 +JAPAN|6000077.2184 +INDONESIA|5580475.4027 +VIETNAM|4497840.5466 diff --git a/test/src/test/resources/tpch/sf0.1/q6.csv b/test/src/test/resources/tpch/sf0.1/q6.csv new file mode 100644 index 0000000000..55f4b53b2b --- /dev/null +++ b/test/src/test/resources/tpch/sf0.1/q6.csv @@ -0,0 +1,2 @@ +revenue +11803420.2534 diff --git a/test/src/test/resources/tpch/sf0.1/q9.csv b/test/src/test/resources/tpch/sf0.1/q9.csv new file mode 100644 index 0000000000..e3ae1a01c4 --- /dev/null +++ b/test/src/test/resources/tpch/sf0.1/q9.csv @@ -0,0 +1,176 @@ +nation|o_year|sum_profit +ALGERIA|1998|2321785.3682 +ALGERIA|1997|3685016.8589 +ALGERIA|1996|4276597.4253 +ALGERIA|1995|4418370.4154 +ALGERIA|1994|3864849.9521 +ALGERIA|1993|3541051.3865 +ALGERIA|1992|4310013.3482 +ARGENTINA|1998|2685983.8005 +ARGENTINA|1997|4242147.8124 +ARGENTINA|1996|3907867.0103 +ARGENTINA|1995|4605921.5011 +ARGENTINA|1994|3542096.1564 +ARGENTINA|1993|3949965.9388 +ARGENTINA|1992|4521180.4695 +BRAZIL|1998|2778730.3931 +BRAZIL|1997|4642037.4687 +BRAZIL|1996|4530304.6034 +BRAZIL|1995|4502344.8657 +BRAZIL|1994|4875806.5015 +BRAZIL|1993|4687478.6531 +BRAZIL|1992|5035200.0464 +CANADA|1998|2194509.0465 +CANADA|1997|3482197.9521 +CANADA|1996|3712231.2814 +CANADA|1995|4014814.8476 +CANADA|1994|4145304.4855 +CANADA|1993|3787069.6045 +CANADA|1992|4168009.4201 +CHINA|1998|3398578.0001 +CHINA|1997|6358959.3338 +CHINA|1996|6435158.3229 +CHINA|1995|6174776.2113 +CHINA|1994|6385751.0812 +CHINA|1993|5765034.1194 +CHINA|1992|6324034.2379 +EGYPT|1998|2333148.3334 +EGYPT|1997|3661244.2731 +EGYPT|1996|3765371.2368 +EGYPT|1995|4094744.2925 +EGYPT|1994|3566508.0818 +EGYPT|1993|3725283.7747 +EGYPT|1992|3373762.3335 +ETHIOPIA|1998|1953927.2682 +ETHIOPIA|1997|3285786.3266 +ETHIOPIA|1996|3525028.7952 +ETHIOPIA|1995|3781674.8911 +ETHIOPIA|1994|3037409.4360 +ETHIOPIA|1993|3008978.2677 +ETHIOPIA|1992|2721203.2355 +FRANCE|1998|2604373.8805 +FRANCE|1997|3982872.0488 +FRANCE|1996|3622479.2413 +FRANCE|1995|4479939.7020 +FRANCE|1994|3531013.1981 +FRANCE|1993|4086437.3102 +FRANCE|1992|3637792.1333 +GERMANY|1998|3291023.2965 +GERMANY|1997|5139337.3443 +GERMANY|1996|4799810.4577 +GERMANY|1995|5405785.7978 +GERMANY|1994|4555556.4592 +GERMANY|1993|4428195.1019 +GERMANY|1992|4656148.4204 +INDIA|1998|2591288.1874 +INDIA|1997|5159562.7033 +INDIA|1996|5307258.3049 +INDIA|1995|5148208.7902 +INDIA|1994|5164001.9582 +INDIA|1993|4321398.4388 +INDIA|1992|5297703.6935 +INDONESIA|1998|3094900.1597 +INDONESIA|1997|5719773.0358 +INDONESIA|1996|6037238.5993 +INDONESIA|1995|5266783.4899 +INDONESIA|1994|5470762.8729 +INDONESIA|1993|6189826.6613 +INDONESIA|1992|4414623.1549 +IRAN|1998|3214864.1209 +IRAN|1997|3688049.0691 +IRAN|1996|3621649.2247 +IRAN|1995|4420783.4205 +IRAN|1994|4373984.6523 +IRAN|1993|3731301.7814 +IRAN|1992|4417133.3662 +IRAQ|1998|2338859.4099 +IRAQ|1997|3622681.5643 +IRAQ|1996|4762291.8722 +IRAQ|1995|4558092.7359 +IRAQ|1994|4951604.1699 +IRAQ|1993|3830077.9911 +IRAQ|1992|3938636.4874 +JAPAN|1998|1849535.0802 +JAPAN|1997|4068688.8537 +JAPAN|1996|4044774.7597 +JAPAN|1995|4793005.8027 +JAPAN|1994|4114717.0568 +JAPAN|1993|3614468.7485 +JAPAN|1992|4266694.4700 +JORDAN|1998|1811488.0719 +JORDAN|1997|2951297.8678 +JORDAN|1996|3302528.3067 +JORDAN|1995|3221813.9990 +JORDAN|1994|2417892.0921 +JORDAN|1993|3107641.7661 +JORDAN|1992|3316379.0585 +KENYA|1998|2579075.4190 +KENYA|1997|2929194.2317 +KENYA|1996|3569129.5619 +KENYA|1995|3542889.1087 +KENYA|1994|3983095.3994 +KENYA|1993|3713988.9708 +KENYA|1992|3304641.8340 +MOROCCO|1998|1815334.8180 +MOROCCO|1997|3693214.8447 +MOROCCO|1996|4116175.9230 +MOROCCO|1995|3515127.1402 +MOROCCO|1994|4003072.1120 +MOROCCO|1993|3599199.6679 +MOROCCO|1992|3958335.4224 +MOZAMBIQUE|1998|1620428.7346 +MOZAMBIQUE|1997|2802166.6473 +MOZAMBIQUE|1996|2409955.1755 +MOZAMBIQUE|1995|2771602.6274 +MOZAMBIQUE|1994|2548226.2158 +MOZAMBIQUE|1993|2843748.9053 +MOZAMBIQUE|1992|2556501.0943 +PERU|1998|2036430.3602 +PERU|1997|4064142.4091 +PERU|1996|4068678.5671 +PERU|1995|4657694.8412 +PERU|1994|4731959.4655 +PERU|1993|4144006.6610 +PERU|1992|3754635.0078 +ROMANIA|1998|1992773.6811 +ROMANIA|1997|2854639.8680 +ROMANIA|1996|3139337.3029 +ROMANIA|1995|3222153.3776 +ROMANIA|1994|3222844.3190 +ROMANIA|1993|3488994.0288 +ROMANIA|1992|3029274.4420 +RUSSIA|1998|2339865.6635 +RUSSIA|1997|4153619.5424 +RUSSIA|1996|3772067.4041 +RUSSIA|1995|4704988.8607 +RUSSIA|1994|4479082.8694 +RUSSIA|1993|4767719.9791 +RUSSIA|1992|4533465.5590 +SAUDI ARABIA|1998|3386948.9564 +SAUDI ARABIA|1997|5425980.3373 +SAUDI ARABIA|1996|5227607.1677 +SAUDI ARABIA|1995|4506731.6411 +SAUDI ARABIA|1994|4698658.7425 +SAUDI ARABIA|1993|5493626.5285 +SAUDI ARABIA|1992|4573560.0150 +UNITED KINGDOM|1998|2252021.5137 +UNITED KINGDOM|1997|4343926.8026 +UNITED KINGDOM|1996|4189476.3065 +UNITED KINGDOM|1995|4469569.8829 +UNITED KINGDOM|1994|4410094.6264 +UNITED KINGDOM|1993|4054677.1050 +UNITED KINGDOM|1992|3978688.8831 +UNITED STATES|1998|2238771.5581 +UNITED STATES|1997|4135581.5734 +UNITED STATES|1996|3624013.2660 +UNITED STATES|1995|3892244.5172 +UNITED STATES|1994|3289224.1138 +UNITED STATES|1993|3626170.2028 +UNITED STATES|1992|3993973.4997 +VIETNAM|1998|1924313.4862 +VIETNAM|1997|3436195.3709 +VIETNAM|1996|4017288.8927 +VIETNAM|1995|3644054.1372 +VIETNAM|1994|4141277.6665 +VIETNAM|1993|2556114.1693 +VIETNAM|1992|4090524.4905 diff --git a/test/src/test/resources/tpch/thu_cloud_download.py b/test/src/test/resources/tpch/thu_cloud_download.py new file mode 100644 index 0000000000..474e9b9dd4 --- /dev/null +++ b/test/src/test/resources/tpch/thu_cloud_download.py @@ -0,0 +1,187 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +import os +import re +import logging +import fnmatch +import requests +import argparse +import urllib.parse +from tqdm import tqdm + + +sess = requests.Session() +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') + + +def parse_args(): + args = argparse.ArgumentParser() + args.add_argument('-l', '--link', type=str, required=True, help='Share link of Tsinghua Cloud') + args.add_argument('-s', '--save_dir', type=str, default=None, help='Path to save the files. Default: Desktop') + args.add_argument('-f', '--file', type=str, default=None, help='Regex to match the file path') + return args.parse_args() + + +def get_share_key(url: str) -> str: + prefix = 'https://cloud.tsinghua.edu.cn/d/' + if not url.startswith(prefix): + raise ValueError('Share link of Tsinghua Cloud should start with {}'.format(prefix)) + share_key = url[len(prefix):].replace('/', '') + logging.info('Share key: {}'.format(share_key)) + return share_key + + +def get_root_dir(share_key: str) -> str: + # Aquire the root directory name of the share link, + # run after verify_password function + global sess + pattern = '' + r = sess.get(f"https://cloud.tsinghua.edu.cn/d/{share_key}/") + root_dir = re.findall(pattern, r.text) + assert root_dir is not None, "Couldn't find title of the share link." + logging.info("Root directory name: {}".format(root_dir[0])) + return root_dir[0] + + +def verify_password(share_key: str) -> None: + # Require password if the share link is password-protected, + # and verify the password provided by the user. + global sess + r = sess.get(f"https://cloud.tsinghua.edu.cn/d/{share_key}/") + pattern = '' + csrfmiddlewaretoken = re.findall(pattern, r.text) + if csrfmiddlewaretoken: + pwd = input("Please enter the password: ") + + csrfmiddlewaretoken = csrfmiddlewaretoken[0] + data = { + "csrfmiddlewaretoken": csrfmiddlewaretoken, + "token": share_key, + "password": pwd + } + r = sess.post(f"https://cloud.tsinghua.edu.cn/d/{share_key}/", data=data, + headers={"Referer": f"https://cloud.tsinghua.edu.cn/d/{share_key}/"}) + if "Please enter a correct password" in r.text: + raise ValueError("Wrong password.") + + +def is_match(file_path: str, pattern: str) -> bool: + # judge if the file path matches the regex provided by the user + file_path = file_path[1:] # remove the first '/' + return pattern is None or fnmatch.fnmatch(file_path, pattern) + + +def dfs_search_files(share_key: str, + path: str = "/", + pattern: str = None) -> list: + global sess + filelist = [] + encoded_path = urllib.parse.quote(path) + r = sess.get(f'https://cloud.tsinghua.edu.cn/api/v2.1/share-links/{share_key}/dirents/?path={encoded_path}') + objects = r.json()['dirent_list'] + for obj in objects: + if obj["is_dir"]: + filelist.extend( + dfs_search_files(share_key, obj['folder_path'], pattern)) + elif is_match(obj["file_path"], pattern): + filelist.append(obj) + return filelist + + +def download_single_file(url: str, fname: str, pbar: tqdm): + global sess + resp = sess.get(url, stream=True) + with open(fname, 'wb') as file: + for data in resp.iter_content(chunk_size=1024): + size = file.write(data) + pbar.update(size) + + +def print_filelist(filelist): + print("=" * 100) + print("Last Modified Time".ljust(25), " ", "File Size".rjust(10), " ", "File Path") + print("-" * 100) + for i, file in enumerate(filelist, 1): + print(file["last_modified"], " ", str(file["size"]).rjust(10), " ", file["file_path"]) + if i == 100: + print("... %d more files" % (len(filelist) - 100)) + break + print("-" * 100) + + +def download(share_key: str, filelist: list, save_dir: str) -> None: + if os.path.exists(save_dir): + logging.warning("Save directory already exists. Files will be overwritten.") + total_size = sum([file["size"] for file in filelist]) + pbar = tqdm(total=total_size, ncols=120, unit='iB', unit_scale=True, unit_divisor=1024) + for i, file in enumerate(filelist): + file_url = 'https://cloud.tsinghua.edu.cn/d/{}/files/?p={}&dl=1'.format(share_key, file["file_path"]) + save_path = os.path.join(save_dir, file["file_path"][1:]) + os.makedirs(os.path.dirname(save_path), exist_ok=True) + # logging.info("[{}/{}] Downloading File: {}".format(i + 1, len(filelist), save_path)) + try: + pbar.set_description("[{}/{}]".format(i + 1, len(filelist))) + download_single_file(file_url, save_path, pbar) + + except Exception as e: + logging.error("Error happened when downloading file: {}".format(save_path)) + logging.error(e) + pbar.close() + logging.info("Download finished.") + + + +def main(): + args = parse_args() + url, pattern, save_dir = args.link, args.file, args.save_dir + share_key = get_share_key(url) + verify_password(share_key) + + # search files + logging.info("Searching for files to be downloaded, Wait a moment...") + filelist = dfs_search_files(share_key, pattern=pattern) + filelist.sort(key=lambda x: x["file_path"]) + if not filelist: + logging.info("No file found.") + return + + print_filelist(filelist) + total_size = sum([file["size"] for file in filelist]) / 1024 / 1024 # MB + logging.info(f"# Files: {len(filelist)}. Total size: {total_size: .1f} MB.") + + # Save to desktop by default. + if save_dir is None: + save_dir = os.path.join(os.path.expanduser("~"), 'Desktop') + assert os.path.exists(save_dir), "Desktop folder not found." + root_dir = get_root_dir(share_key) + save_dir = os.path.join(save_dir, root_dir) + + download(share_key, filelist, save_dir) + + + +if __name__ == '__main__': + """ + 用法: + python main.py \ + -l https://cloud.tsinghua.edu.cn/d/1234567890/ \ + -s "~/path_to_save" \ + -f "*.pptx?" (regex, 正则表达式) \ + """ + main() \ No newline at end of file diff --git a/test/src/test/resources/tpch/udf/udtf_extract_year.py b/test/src/test/resources/tpch/udf/udtf_extract_year.py new file mode 100644 index 0000000000..fc4fcf4f5c --- /dev/null +++ b/test/src/test/resources/tpch/udf/udtf_extract_year.py @@ -0,0 +1,46 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +from datetime import datetime, timezone +import pytz +class UDFExtractYear: + def __init__(self): + pass + + def extractYear(self, num): + # Unix timestamp is in milliseconds + timestamp_in_seconds = num / 1000 + # TODO 直接将timestamp增加8小时 + tz = pytz.timezone('Asia/Shanghai') + dt = datetime.fromtimestamp(timestamp_in_seconds, tz=tz) + return float(dt.year) + def transform(self, data, args, kvargs): + res = self.buildHeader(data) + dateRow = [] + for num in data[2][1:]: + dateRow.append(self.extractYear(num)) + res.append(dateRow) + return res + + def buildHeader(self, data): + colNames = [] + colTypes = [] + for name in data[0][1:]: + colNames.append("extractYear(" + name + ")") + colTypes.append("DOUBLE") + return [colNames, colTypes] diff --git a/test/src/test/resources/transform/TransformScheduledAfter10s.yaml b/test/src/test/resources/transform/TransformScheduledAfter10s.yaml new file mode 100644 index 0000000000..bcbb9817b7 --- /dev/null +++ b/test/src/test/resources/transform/TransformScheduledAfter10s.yaml @@ -0,0 +1,9 @@ +--- +taskList: +- taskType: "iginx" + timeout: 10000000 + sqlList: + - "show columns;" +schedule: "after 10 second" +exportType: "file" +exportFile: "test/src/test/resources/transform/export_file_after_10_s.txt" \ No newline at end of file diff --git a/test/src/test/resources/transform/TransformScheduledAt10sFromNow.yaml b/test/src/test/resources/transform/TransformScheduledAt10sFromNow.yaml new file mode 100644 index 0000000000..9fd046e3cc --- /dev/null +++ b/test/src/test/resources/transform/TransformScheduledAt10sFromNow.yaml @@ -0,0 +1,9 @@ +--- +taskList: + - taskType: "iginx" + timeout: 10000000 + sqlList: + - "show columns;" +exportType: "file" +exportFile: "test/src/test/resources/transform/export_file_at_10_s.txt" + diff --git a/test/src/test/resources/transform/TransformScheduledCron.yaml b/test/src/test/resources/transform/TransformScheduledCron.yaml new file mode 100644 index 0000000000..bea1e3b76f --- /dev/null +++ b/test/src/test/resources/transform/TransformScheduledCron.yaml @@ -0,0 +1,9 @@ +--- +taskList: + - taskType: "iginx" + timeout: 10000000 + sqlList: + - "show columns;" +schedule: "(0 0/1 * 1/1 * ? *)" +exportType: "file" +exportFile: "test/src/test/resources/transform/export_file_every_1_minute_cron.txt" \ No newline at end of file diff --git a/test/src/test/resources/transform/TransformScheduledEvery10s.yaml b/test/src/test/resources/transform/TransformScheduledEvery10s.yaml new file mode 100644 index 0000000000..c49048520f --- /dev/null +++ b/test/src/test/resources/transform/TransformScheduledEvery10s.yaml @@ -0,0 +1,9 @@ +--- +taskList: + - taskType: "iginx" + timeout: 10000000 + sqlList: + - "show columns;" +schedule: "every 10 second" +exportType: "file" +exportFile: "test/src/test/resources/transform/export_file_every_10_s.txt" \ No newline at end of file diff --git a/test/src/test/resources/transform/transformer_add_one.py b/test/src/test/resources/transform/transformer_add_one.py index 6315786272..4fe45d624a 100644 --- a/test/src/test/resources/transform/transformer_add_one.py +++ b/test/src/test/resources/transform/transformer_add_one.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + import pandas as pd diff --git a/test/src/test/resources/transform/transformer_row_sum.py b/test/src/test/resources/transform/transformer_row_sum.py index cfc503c1cd..b85dd676af 100644 --- a/test/src/test/resources/transform/transformer_row_sum.py +++ b/test/src/test/resources/transform/transformer_row_sum.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + import pandas as pd import numpy as np diff --git a/test/src/test/resources/transform/transformer_sleep.py b/test/src/test/resources/transform/transformer_sleep.py index e8145a01c8..909a7841e6 100644 --- a/test/src/test/resources/transform/transformer_sleep.py +++ b/test/src/test/resources/transform/transformer_sleep.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + import time diff --git a/test/src/test/resources/transform/transformer_sum.py b/test/src/test/resources/transform/transformer_sum.py index 63dea29ddb..e4ef01dd18 100644 --- a/test/src/test/resources/transform/transformer_sum.py +++ b/test/src/test/resources/transform/transformer_sum.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + import pandas as pd diff --git a/test/src/test/resources/udf/mock_udf.py b/test/src/test/resources/udf/mock_udf.py index e6e3ba2c3c..e8917fe921 100644 --- a/test/src/test/resources/udf/mock_udf.py +++ b/test/src/test/resources/udf/mock_udf.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + class MockUDF(): def __init__(self): pass diff --git a/test/src/test/resources/udf/my_module/__init__.py b/test/src/test/resources/udf/my_module/__init__.py index e69de29bb2..02225d5e15 100644 --- a/test/src/test/resources/udf/my_module/__init__.py +++ b/test/src/test/resources/udf/my_module/__init__.py @@ -0,0 +1,18 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + diff --git a/test/src/test/resources/udf/my_module/dateutil_test.py b/test/src/test/resources/udf/my_module/dateutil_test.py index a30ae42433..bef6bd3db0 100644 --- a/test/src/test/resources/udf/my_module/dateutil_test.py +++ b/test/src/test/resources/udf/my_module/dateutil_test.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + from dateutil import parser diff --git a/test/src/test/resources/udf/my_module/idle_classes.py b/test/src/test/resources/udf/my_module/idle_classes.py index 0bba39deb2..cb7c0f2d4a 100644 --- a/test/src/test/resources/udf/my_module/idle_classes.py +++ b/test/src/test/resources/udf/my_module/idle_classes.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + class ClassA: def transform(self, data, args, kvargs): return [["col_outer_a"], ["LONG"], [1]] diff --git a/test/src/test/resources/udf/my_module/my_class_a.py b/test/src/test/resources/udf/my_module/my_class_a.py index 439a6a0225..7f1fbe419d 100644 --- a/test/src/test/resources/udf/my_module/my_class_a.py +++ b/test/src/test/resources/udf/my_module/my_class_a.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + class ClassA: def print_self(self): return "class A" diff --git a/test/src/test/resources/udf/my_module/sub_module/__init__.py b/test/src/test/resources/udf/my_module/sub_module/__init__.py index e69de29bb2..02225d5e15 100644 --- a/test/src/test/resources/udf/my_module/sub_module/__init__.py +++ b/test/src/test/resources/udf/my_module/sub_module/__init__.py @@ -0,0 +1,18 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + diff --git a/test/src/test/resources/udf/my_module/sub_module/sub_class_a.py b/test/src/test/resources/udf/my_module/sub_module/sub_class_a.py index 9ba42710b1..ab71bac6b4 100644 --- a/test/src/test/resources/udf/my_module/sub_module/sub_class_a.py +++ b/test/src/test/resources/udf/my_module/sub_module/sub_class_a.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + class SubClassA: def print_self(self): return "sub class A" diff --git a/thrift/pom.xml b/thrift/pom.xml index 061de88fff..9a440b3161 100644 --- a/thrift/pom.xml +++ b/thrift/pom.xml @@ -1,4 +1,23 @@ + 4.0.0 @@ -7,14 +26,12 @@ cn.edu.tsinghua iginx ${revision} - ../pom.xml iginx-thrift IGinX Thrift - UTF-8 0.16.0 diff --git a/thrift/src/main/proto/filesystem.thrift b/thrift/src/main/proto/filesystem.thrift index 51a6a5f180..171f4f1139 100644 --- a/thrift/src/main/proto/filesystem.thrift +++ b/thrift/src/main/proto/filesystem.thrift @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + include "rpc.thrift" namespace java cn.edu.tsinghua.iginx.filesystem.thrift @@ -157,7 +175,7 @@ service FileSystemService { Status executeDelete(1: DeleteReq req); - GetColumnsOfStorageUnitResp getColumnsOfStorageUnit(1: string storageUnit); + GetColumnsOfStorageUnitResp getColumnsOfStorageUnit(1: string storageUnit, 2: set patterns, 3: RawTagFilter tagFilter); GetBoundaryOfStorageResp getBoundaryOfStorage(1: string dataPrefix); diff --git a/thrift/src/main/proto/parquet.thrift b/thrift/src/main/proto/parquet.thrift index da88565a8f..f2ce834266 100644 --- a/thrift/src/main/proto/parquet.thrift +++ b/thrift/src/main/proto/parquet.thrift @@ -1,3 +1,21 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + namespace java cn.edu.tsinghua.iginx.parquet.thrift struct Status { @@ -72,12 +90,26 @@ struct RawFilter { 9: optional RawValue value } +struct RawFunction { + 1: required string id +} + +struct RawFunctionParams { + 1: required list patterns +} + +struct RawFunctionCall { + 1: required RawFunction func + 2: required RawFunctionParams params +} + struct ProjectReq { 1: required string storageUnit 2: required bool isDummyStorageUnit 3: required list paths 4: optional RawTagFilter tagFilter 5: optional RawFilter filter + 6: optional list aggregations } struct ParquetHeader { @@ -155,7 +187,7 @@ service ParquetService { Status executeDelete(1: DeleteReq req); - GetColumnsOfStorageUnitResp getColumnsOfStorageUnit(1: string storageUnit); + GetColumnsOfStorageUnitResp getColumnsOfStorageUnit(1: string storageUnit, 2: set patterns, 3: RawTagFilter tagFilter); GetStorageBoundaryResp getBoundaryOfStorage(1: string dataPrefix); diff --git a/thrift/src/main/proto/rpc.thrift b/thrift/src/main/proto/rpc.thrift index f4bae6553e..5d64ad21c7 100644 --- a/thrift/src/main/proto/rpc.thrift +++ b/thrift/src/main/proto/rpc.thrift @@ -1,20 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University * - * http://www.apache.org/licenses/LICENSE-2.0 + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ namespace java cn.edu.tsinghua.iginx.thrift namespace py iginx.thrift.rpc @@ -60,6 +59,7 @@ enum SqlType { Query, GetReplicaNum, AddStorageEngines, + AlterStorageEngine, CountPoints, ClearData, DeleteColumns, @@ -116,6 +116,7 @@ enum JobState { JOB_UNKNOWN, JOB_FINISHED, JOB_CREATED, + JOB_IDLE, JOB_RUNNING, JOB_FAILING, JOB_FAILED, @@ -267,6 +268,12 @@ struct AddStorageEnginesReq { 2: required list storageEngines } +struct AlterStorageEngineReq { + 1: required i64 sessionId + 2: required i64 engineId + 3: required map newParams +} + struct StorageEngine { 1: required string ip 2: required i32 port @@ -549,6 +556,7 @@ struct CommitTransformJobReq { 2: required list taskList 3: required ExportType exportType 4: optional string fileName + 5: optional string schedule } struct CommitTransformJobResp { @@ -729,6 +737,8 @@ service IService { Status addStorageEngines(1: AddStorageEnginesReq req); + Status alterStorageEngine(1: AlterStorageEngineReq req); + Status removeHistoryDataSource(1: RemoveHistoryDataSourceReq req); AggregateQueryResp aggregateQuery(1: AggregateQueryReq req); diff --git a/udf_funcs/python_scripts/class_loader.py b/udf_funcs/python_scripts/class_loader.py index c306484d9c..98c4891589 100644 --- a/udf_funcs/python_scripts/class_loader.py +++ b/udf_funcs/python_scripts/class_loader.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + from transformer import BaseTransformer diff --git a/udf_funcs/python_scripts/constant.py b/udf_funcs/python_scripts/constant.py index a2097214de..504ad3c7cf 100644 --- a/udf_funcs/python_scripts/constant.py +++ b/udf_funcs/python_scripts/constant.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + from enum import IntEnum diff --git a/udf_funcs/python_scripts/py_worker.py b/udf_funcs/python_scripts/py_worker.py index e7caddeba3..0ed6b570db 100644 --- a/udf_funcs/python_scripts/py_worker.py +++ b/udf_funcs/python_scripts/py_worker.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + import os import socket import sys diff --git a/udf_funcs/python_scripts/transformer_apply.py b/udf_funcs/python_scripts/transformer_apply.py index e1d74a7133..1d46645596 100644 --- a/udf_funcs/python_scripts/transformer_apply.py +++ b/udf_funcs/python_scripts/transformer_apply.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + """ Apply a function along an axis of the DataFrame. diff --git a/udf_funcs/python_scripts/transformer_between_time.py b/udf_funcs/python_scripts/transformer_between_time.py index 9545bbb171..7132d0341a 100644 --- a/udf_funcs/python_scripts/transformer_between_time.py +++ b/udf_funcs/python_scripts/transformer_between_time.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + """ Select values between particular times of the day (e.g., 9:00-9:30 AM). """ diff --git a/udf_funcs/python_scripts/transformer_bool.py b/udf_funcs/python_scripts/transformer_bool.py index c14288253c..8937ee46b2 100644 --- a/udf_funcs/python_scripts/transformer_bool.py +++ b/udf_funcs/python_scripts/transformer_bool.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + """ Return the bool of a single element Series or DataFrame. This must be a boolean scalar value, either True or False. diff --git a/udf_funcs/python_scripts/transformer_bottom.py b/udf_funcs/python_scripts/transformer_bottom.py index b1cdb04a24..f059d2dea6 100644 --- a/udf_funcs/python_scripts/transformer_bottom.py +++ b/udf_funcs/python_scripts/transformer_bottom.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + """ The bottom() function sorts a table by columns and keeps only the bottom n records. bottom() is a selector function. bottom() 函数按列对表进行排序并仅保留底部的 n 条记录。 bottom() 是一个选择器函数。 diff --git a/udf_funcs/python_scripts/transformer_distinct.py b/udf_funcs/python_scripts/transformer_distinct.py index ceb33a71cf..668fadae7b 100644 --- a/udf_funcs/python_scripts/transformer_distinct.py +++ b/udf_funcs/python_scripts/transformer_distinct.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + """ Writing a distinct() function for IginX: Count the distinct field values associated with a field key. diff --git a/udf_funcs/python_scripts/transformer_filter.py b/udf_funcs/python_scripts/transformer_filter.py index bf8957ce0f..9dfa7be692 100644 --- a/udf_funcs/python_scripts/transformer_filter.py +++ b/udf_funcs/python_scripts/transformer_filter.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + """ The filter() function is used to subset rows or columns of dataframe according to labels in the specified index. """ diff --git a/udf_funcs/python_scripts/transformer_head.py b/udf_funcs/python_scripts/transformer_head.py index 57d0874cc0..4a12f09b0b 100644 --- a/udf_funcs/python_scripts/transformer_head.py +++ b/udf_funcs/python_scripts/transformer_head.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + """ Return the first n rows. This function returns the first n rows for the object based on position. diff --git a/udf_funcs/python_scripts/transformer_loc.py b/udf_funcs/python_scripts/transformer_loc.py index f96dfa4c89..1fd3c720a8 100644 --- a/udf_funcs/python_scripts/transformer_loc.py +++ b/udf_funcs/python_scripts/transformer_loc.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + """ Access a group of rows and columns by label(s) or a boolean array. """ diff --git a/udf_funcs/python_scripts/transformer_mean.py b/udf_funcs/python_scripts/transformer_mean.py index b639921862..722d289bac 100644 --- a/udf_funcs/python_scripts/transformer_mean.py +++ b/udf_funcs/python_scripts/transformer_mean.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + """ mean() returns the average of non-null values in a specified column from each input table. Mean function returns the average by dividing the sum of the values in the set by their number. diff --git a/udf_funcs/python_scripts/transformer_skew.py b/udf_funcs/python_scripts/transformer_skew.py index e1e7061e8b..d8813767e0 100644 --- a/udf_funcs/python_scripts/transformer_skew.py +++ b/udf_funcs/python_scripts/transformer_skew.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + """ Return unbiased skew over requested axis. Normalized by N-1. diff --git a/udf_funcs/python_scripts/transformer_sortindex.py b/udf_funcs/python_scripts/transformer_sortindex.py index 0579441296..3e50de5c56 100644 --- a/udf_funcs/python_scripts/transformer_sortindex.py +++ b/udf_funcs/python_scripts/transformer_sortindex.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + """ Sorts index in descending order: diff --git a/udf_funcs/python_scripts/transformer_spread.py b/udf_funcs/python_scripts/transformer_spread.py index 57b896e2a0..3361816d2b 100644 --- a/udf_funcs/python_scripts/transformer_spread.py +++ b/udf_funcs/python_scripts/transformer_spread.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + import pandas as pd diff --git a/udf_funcs/python_scripts/udf_avg.py b/udf_funcs/python_scripts/udf_avg.py index 2cb5dc817b..3dda56303f 100644 --- a/udf_funcs/python_scripts/udf_avg.py +++ b/udf_funcs/python_scripts/udf_avg.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + class UDFAvg: def __init__(self): pass diff --git a/udf_funcs/python_scripts/udf_count.py b/udf_funcs/python_scripts/udf_count.py index a6bf93bf75..375843d6f5 100644 --- a/udf_funcs/python_scripts/udf_count.py +++ b/udf_funcs/python_scripts/udf_count.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + class UDFCount: def __init__(self): pass diff --git a/udf_funcs/python_scripts/udf_max.py b/udf_funcs/python_scripts/udf_max.py index 156db6974c..58201b76e0 100644 --- a/udf_funcs/python_scripts/udf_max.py +++ b/udf_funcs/python_scripts/udf_max.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + class UDFMax: def __init__(self): pass @@ -7,9 +25,9 @@ def transform(self, data, args, kvargs): maxRow = [] rows = data[2:] - for row in list(zip(*rows))[1:]: + for col in list(zip(*rows))[1:]: max = None - for num in row: + for num in col: if num is not None: if max is None: max = num diff --git a/udf_funcs/python_scripts/udf_max_with_key.py b/udf_funcs/python_scripts/udf_max_with_key.py index c503ae63de..1003915b2e 100644 --- a/udf_funcs/python_scripts/udf_max_with_key.py +++ b/udf_funcs/python_scripts/udf_max_with_key.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + class UDFMaxWithKey: def __init__(self): pass diff --git a/udf_funcs/python_scripts/udf_min.py b/udf_funcs/python_scripts/udf_min.py index 6c896505d8..bd1b2a4977 100644 --- a/udf_funcs/python_scripts/udf_min.py +++ b/udf_funcs/python_scripts/udf_min.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + class UDFMin: def __init__(self): pass diff --git a/udf_funcs/python_scripts/udf_sum.py b/udf_funcs/python_scripts/udf_sum.py index 7b8375cc5b..617e5070dc 100644 --- a/udf_funcs/python_scripts/udf_sum.py +++ b/udf_funcs/python_scripts/udf_sum.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + class UDFSum: def __init__(self): pass diff --git a/udf_funcs/python_scripts/udsf_reverse_rows.py b/udf_funcs/python_scripts/udsf_reverse_rows.py index 8f1dbbe365..7c05e0a738 100644 --- a/udf_funcs/python_scripts/udsf_reverse_rows.py +++ b/udf_funcs/python_scripts/udsf_reverse_rows.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + class UDFReverseRows: def __init__(self): pass diff --git a/udf_funcs/python_scripts/udsf_transpose.py b/udf_funcs/python_scripts/udsf_transpose.py index 7c787249b6..90be6ad228 100644 --- a/udf_funcs/python_scripts/udsf_transpose.py +++ b/udf_funcs/python_scripts/udsf_transpose.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + class UDFTranspose: def __init__(self): pass diff --git a/udf_funcs/python_scripts/udtf_column_expand.py b/udf_funcs/python_scripts/udtf_column_expand.py index 66dd008bc3..29b1a3e4ba 100644 --- a/udf_funcs/python_scripts/udtf_column_expand.py +++ b/udf_funcs/python_scripts/udtf_column_expand.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + class UDFColumnExpand: def __init__(self): pass diff --git a/udf_funcs/python_scripts/udtf_cos.py b/udf_funcs/python_scripts/udtf_cos.py index f3ba7cdbc2..bd4160df9a 100644 --- a/udf_funcs/python_scripts/udtf_cos.py +++ b/udf_funcs/python_scripts/udtf_cos.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + import math diff --git a/udf_funcs/python_scripts/udtf_key_add_one.py b/udf_funcs/python_scripts/udtf_key_add_one.py index b5b9afeaf2..6a39d2b8de 100644 --- a/udf_funcs/python_scripts/udtf_key_add_one.py +++ b/udf_funcs/python_scripts/udtf_key_add_one.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + class UDFKeyAddOne: def __init__(self): pass diff --git a/udf_funcs/python_scripts/udtf_multiply.py b/udf_funcs/python_scripts/udtf_multiply.py index 959a0f7e65..f28fa6a94a 100644 --- a/udf_funcs/python_scripts/udtf_multiply.py +++ b/udf_funcs/python_scripts/udtf_multiply.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + class UDFMultiply: def __init__(self): pass diff --git a/udf_funcs/python_scripts/udtf_pow.py b/udf_funcs/python_scripts/udtf_pow.py index 93e6e04093..7f02151b24 100644 --- a/udf_funcs/python_scripts/udtf_pow.py +++ b/udf_funcs/python_scripts/udtf_pow.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + class UDFPow: def __init__(self): self._n = 1