diff --git a/.github/cppcheck-problem-matcher.json b/.github/cppcheck-problem-matcher.json new file mode 100644 index 00000000..3b18fecc --- /dev/null +++ b/.github/cppcheck-problem-matcher.json @@ -0,0 +1,16 @@ +{ + "problemMatcher": [ + { + "owner": "cppcheck", + "pattern": [ + { + "regexp": "^([^:]+):(\\d+):(\\d*):\\s(style|portability|performance|warning|error):\\s(.*)$", + "file": 1, + "line": 2, + "column": 3, + "message": 5 + } + ] + } + ] +} diff --git a/.github/workflows/build-and-test-daily.yaml b/.github/workflows/build-and-test-daily.yaml new file mode 100644 index 00000000..63822f8b --- /dev/null +++ b/.github/workflows/build-and-test-daily.yaml @@ -0,0 +1,79 @@ +name: build-and-test-daily + +on: + schedule: + - cron: 0 0 * * * + workflow_dispatch: + +jobs: + build-and-test-daily: + runs-on: [self-hosted, linux, X64, gpu] + container: ${{ matrix.container }}${{ matrix.container-suffix }} + strategy: + fail-fast: false + matrix: + rosdistro: + - humble + container-suffix: + - "" + - -cuda + include: + - rosdistro: humble + container: ghcr.io/autowarefoundation/autoware:universe-devel + build-depends-repos: build_depends.repos + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Show disk space before the tasks + run: df -h + + - name: Remove exec_depend + uses: autowarefoundation/autoware-github-actions/remove-exec-depend@v1 + + - name: Get self packages + id: get-self-packages + uses: autowarefoundation/autoware-github-actions/get-self-packages@v1 + + - name: Export CUDA state as a variable for adding to cache key + run: | + build_type_cuda_state=nocuda + if [[ "${{ matrix.container-suffix }}" == "-cuda" ]]; then + build_type_cuda_state=cuda + fi + echo "BUILD_TYPE_CUDA_STATE=$build_type_cuda_state" >> "${GITHUB_ENV}" + echo "::notice::BUILD_TYPE_CUDA_STATE=$build_type_cuda_state" + shell: bash + + - name: Build + if: ${{ steps.get-self-packages.outputs.self-packages != '' }} + uses: autowarefoundation/autoware-github-actions/colcon-build@v1 + with: + rosdistro: ${{ matrix.rosdistro }} + target-packages: ${{ steps.get-self-packages.outputs.self-packages }} + build-depends-repos: ${{ matrix.build-depends-repos }} + cache-key-element: ${{ env.BUILD_TYPE_CUDA_STATE }} + + - name: Test + if: ${{ steps.get-self-packages.outputs.self-packages != '' }} + id: test + uses: autowarefoundation/autoware-github-actions/colcon-test@v1 + with: + rosdistro: ${{ matrix.rosdistro }} + target-packages: ${{ steps.get-self-packages.outputs.self-packages }} + build-depends-repos: ${{ matrix.build-depends-repos }} + + - name: Upload coverage to CodeCov + if: ${{ steps.test.outputs.coverage-report-files != '' }} + uses: codecov/codecov-action@v4 + with: + files: ${{ steps.test.outputs.coverage-report-files }} + fail_ci_if_error: false + verbose: true + flags: total + token: ${{ secrets.CODECOV_TOKEN }} + + - name: Show disk space after the tasks + run: df -h diff --git a/.github/workflows/build-and-test-differential.yaml b/.github/workflows/build-and-test-differential.yaml index cda16666..e77aae16 100644 --- a/.github/workflows/build-and-test-differential.yaml +++ b/.github/workflows/build-and-test-differential.yaml @@ -42,7 +42,7 @@ jobs: - -cuda include: - rosdistro: humble - container: ghcr.io/autowarefoundation/autoware:latest-autoware-universe + container: ghcr.io/autowarefoundation/autoware:universe-devel build-depends-repos: build_depends.repos - container-suffix: -cuda runner: codebuild-autoware-us-east-1-${{ github.run_id }}-${{ github.run_attempt }}-ubuntu-7.0-large @@ -76,7 +76,7 @@ jobs: clang-tidy-differential: needs: build-and-test-differential runs-on: ubuntu-22.04 - container: ghcr.io/autowarefoundation/autoware:latest-autoware-universe-cuda + container: ghcr.io/autowarefoundation/autoware:universe-devel-cuda steps: - name: Set PR fetch depth run: echo "PR_FETCH_DEPTH=$(( ${{ github.event.pull_request.commits }} + 1 ))" >> "${GITHUB_ENV}" diff --git a/.github/workflows/build-and-test.yaml b/.github/workflows/build-and-test.yaml index 228b547e..a9274804 100644 --- a/.github/workflows/build-and-test.yaml +++ b/.github/workflows/build-and-test.yaml @@ -2,27 +2,44 @@ name: build-and-test on: push: - schedule: - - cron: 0 0 * * * + branches: + - main workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + +env: + CC: /usr/lib/ccache/gcc + CXX: /usr/lib/ccache/g++ + jobs: build-and-test: - if: ${{ github.event_name != 'push' || github.ref_name == github.event.repository.default_branch }} - runs-on: ubuntu-latest - container: ${{ matrix.container }} + runs-on: codebuild-autoware-us-east-1-${{ github.run_id }}-${{ github.run_attempt }}-ubuntu-7.0-large + container: ${{ matrix.container }}${{ matrix.container-suffix }} strategy: fail-fast: false matrix: rosdistro: - humble + container-suffix: + - -cuda include: - rosdistro: humble - container: ros:humble + container: ghcr.io/autowarefoundation/autoware:universe-devel build-depends-repos: build_depends.repos steps: - name: Check out repository uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Show disk space before the tasks + run: df -h + + - name: Show machine specs + run: lscpu && free -h - name: Remove exec_depend uses: autowarefoundation/autoware-github-actions/remove-exec-depend@v1 @@ -31,6 +48,43 @@ jobs: id: get-self-packages uses: autowarefoundation/autoware-github-actions/get-self-packages@v1 + - name: Create ccache directory + run: | + mkdir -p ${CCACHE_DIR} + du -sh ${CCACHE_DIR} && ccache -s + shell: bash + + - name: Attempt to restore ccache + uses: actions/cache/restore@v4 + with: + path: | + /root/.ccache + key: ccache-main-${{ runner.arch }}-${{ matrix.rosdistro }}-${{ github.sha }} + restore-keys: | + ccache-main-${{ runner.arch }}-${{ matrix.rosdistro }}- + + - name: Limit ccache size + run: | + rm -f "${CCACHE_DIR}/ccache.conf" + echo -e "# Set maximum cache size\nmax_size = 600MB" >> "${CCACHE_DIR}/ccache.conf" + shell: bash + + - name: Show ccache stats before build and reset stats + run: | + du -sh ${CCACHE_DIR} && ccache -s + ccache --zero-stats + shell: bash + + - name: Export CUDA state as a variable for adding to cache key + run: | + build_type_cuda_state=nocuda + if [[ "${{ matrix.container-suffix }}" == "-cuda" ]]; then + build_type_cuda_state=cuda + fi + echo "BUILD_TYPE_CUDA_STATE=$build_type_cuda_state" >> "${GITHUB_ENV}" + echo "::notice::BUILD_TYPE_CUDA_STATE=$build_type_cuda_state" + shell: bash + - name: Build if: ${{ steps.get-self-packages.outputs.self-packages != '' }} uses: autowarefoundation/autoware-github-actions/colcon-build@v1 @@ -38,6 +92,21 @@ jobs: rosdistro: ${{ matrix.rosdistro }} target-packages: ${{ steps.get-self-packages.outputs.self-packages }} build-depends-repos: ${{ matrix.build-depends-repos }} + cache-key-element: ${{ env.BUILD_TYPE_CUDA_STATE }} + build-pre-command: taskset --cpu-list 0-6 + + - name: Show ccache stats after build + run: du -sh ${CCACHE_DIR} && ccache -s + shell: bash + + # Only keep save the -cuda version because cuda packages covers non-cuda packages too + - name: Push the ccache cache + if: matrix.container-suffix == '-cuda' + uses: actions/cache/save@v4 + with: + path: | + /root/.ccache + key: ccache-main-${{ runner.arch }}-${{ matrix.rosdistro }}-${{ github.sha }} - name: Test if: ${{ steps.get-self-packages.outputs.self-packages != '' }} @@ -56,3 +125,7 @@ jobs: fail_ci_if_error: false verbose: true flags: total + token: ${{ secrets.CODECOV_TOKEN }} + + - name: Show disk space after the tasks + run: df -h diff --git a/.github/workflows/cppcheck-differential.yaml b/.github/workflows/cppcheck-differential.yaml new file mode 100644 index 00000000..36239577 --- /dev/null +++ b/.github/workflows/cppcheck-differential.yaml @@ -0,0 +1,95 @@ +name: cppcheck-differential + +on: + pull_request: + +jobs: + cppcheck-differential: + runs-on: ubuntu-22.04 + + steps: + - name: Set PR fetch depth + run: echo "PR_FETCH_DEPTH=$(( ${{ github.event.pull_request.commits }} + 1 ))" >> "${GITHUB_ENV}" + + - name: Checkout PR branch and all PR commits + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: ${{ env.PR_FETCH_DEPTH }} + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y git + + - name: Install colcon + run: | + sudo sh -c 'echo "deb [arch=amd64,arm64] http://repo.ros2.org/ubuntu/main `lsb_release -cs` main" > /etc/apt/sources.list.d/ros2-latest.list' + curl -s https://raw.githubusercontent.com/ros/rosdistro/master/ros.asc | sudo apt-key add - + sudo apt update + sudo apt install python3-colcon-common-extensions + + # cppcheck from apt does not yet support --check-level args, and thus install from snap + - name: Install Cppcheck from snap + run: | + sudo snap install cppcheck + + - name: Fetch the base branch with enough history for a common merge-base commit + run: git fetch origin ${{ github.base_ref }} + shell: bash + + - name: Get modified packages + id: get-modified-packages + uses: autowarefoundation/autoware-github-actions/get-modified-packages@v1 + + - name: Get full paths of modified packages + id: get-full-paths + run: | + modified_packages="${{ steps.get-modified-packages.outputs.modified-packages }}" + paths="" + for pkg in $modified_packages; do + path=$(colcon list --packages-select $pkg | awk '{print $2}' | xargs realpath) + paths="$paths $path" + done + echo "full-paths=$paths" >> $GITHUB_OUTPUT + + - name: Filter packages with no cpp/hpp files + id: filter-paths-no-cpp-files + run: | + filtered_paths="" + for dir in ${{ steps.get-full-paths.outputs.full-paths }}; do + if [ -d "$dir" ] && find "$dir" -name "*.cpp" | grep -q .; then + filtered_paths="$filtered_paths $dir" + fi + done + echo "filtered-full-paths=$filtered_paths" >> $GITHUB_OUTPUT + + # cspell: ignore suppr, Dslots + - name: Run Cppcheck on modified packages + if: ${{ steps.filter-paths-no-cpp-files.outputs.filtered-full-paths != '' }} + continue-on-error: true + id: cppcheck + run: | + echo "Running Cppcheck on modified packages: ${{ steps.filter-paths-no-cpp-files.outputs.filtered-full-paths }}" + cppcheck --enable=all --inconclusive --check-level=exhaustive -D'PLUGINLIB_EXPORT_CLASS(class_type, base_class)=' -Dslots= -DQ_SLOTS= --error-exitcode=1 --suppressions-list=.cppcheck_suppressions --inline-suppr ${{ steps.filter-paths-no-cpp-files.outputs.filtered-full-paths }} 2> cppcheck-report.txt + shell: bash + + - name: Setup Problem Matchers for cppcheck + if: ${{ steps.filter-paths-no-cpp-files.outputs.filtered-full-paths != '' }} + run: echo "::add-matcher::.github/cppcheck-problem-matcher.json" + + - name: Show cppcheck-report result + if: ${{ steps.filter-paths-no-cpp-files.outputs.filtered-full-paths != '' }} + run: | + cat cppcheck-report.txt + + - name: Upload Cppcheck report + if: ${{ steps.filter-paths-no-cpp-files.outputs.filtered-full-paths != '' }} + uses: actions/upload-artifact@v4 + with: + name: cppcheck-report + path: cppcheck-report.txt + + - name: Fail the job if Cppcheck failed + if: ${{ steps.filter-paths-no-cpp-files.outputs.filtered-full-paths != '' && steps.cppcheck.outcome == 'failure' }} + run: exit 1 diff --git a/.github/workflows/cppcheck-weekly.yaml b/.github/workflows/cppcheck-weekly.yaml new file mode 100644 index 00000000..c75a72e1 --- /dev/null +++ b/.github/workflows/cppcheck-weekly.yaml @@ -0,0 +1,47 @@ +name: cppcheck-weekly + +on: + schedule: + - cron: 0 0 * * 1 + workflow_dispatch: + +jobs: + cppcheck-weekly: + runs-on: ubuntu-22.04 + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + # cppcheck from apt does not yet support --check-level args, and thus install from snap + - name: Install Cppcheck from snap + run: | + sudo snap install cppcheck + + # cspell: ignore suppr, Dslots + - name: Run Cppcheck on all files + continue-on-error: true + id: cppcheck + run: | + cppcheck --enable=all --inconclusive --check-level=exhaustive -D'PLUGINLIB_EXPORT_CLASS(class_type, base_class)=' -Dslots= -DQ_SLOTS= --suppress=*:*/test/* --error-exitcode=1 --xml --inline-suppr . 2> cppcheck-report.xml + shell: bash + + - name: Count errors by error ID and severity + run: | + #!/bin/bash + temp_file=$(mktemp) + grep -oP '(?<=id=")[^"]+" severity="[^"]+' cppcheck-report.xml | sed 's/" severity="/,/g' > "$temp_file" + echo "Error counts by error ID and severity:" + sort "$temp_file" | uniq -c + rm "$temp_file" + shell: bash + + - name: Upload Cppcheck report + uses: actions/upload-artifact@v4 + with: + name: cppcheck-report + path: cppcheck-report.xml + + - name: Fail the job if Cppcheck failed + if: steps.cppcheck.outcome == 'failure' + run: exit 1 diff --git a/.github/workflows/json-schema-check.yaml b/.github/workflows/json-schema-check.yaml new file mode 100644 index 00000000..be9b8e3c --- /dev/null +++ b/.github/workflows/json-schema-check.yaml @@ -0,0 +1,39 @@ +name: json-schema-check + +on: + pull_request: + workflow_dispatch: + +jobs: + check-if-relevant-files-changed: + runs-on: ubuntu-22.04 + outputs: + run-check: ${{ steps.paths_filter.outputs.json_or_yaml }} + steps: + - uses: actions/checkout@v4 + - uses: dorny/paths-filter@v3 + id: paths_filter + with: + filters: | + json_or_yaml: + - '**/schema/*.schema.json' + - '**/config/*.param.yaml' + + json-schema-check: + needs: check-if-relevant-files-changed + if: needs.check-if-relevant-files-changed.outputs.run-check == 'true' + runs-on: ubuntu-22.04 + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Run json-schema-check + uses: autowarefoundation/autoware-github-actions/json-schema-check@v1 + + no-relevant-changes: + needs: check-if-relevant-files-changed + if: needs.check-if-relevant-files-changed.outputs.run-check == 'false' + runs-on: ubuntu-22.04 + steps: + - name: Dummy step + run: echo "No relevant changes, passing check"