diff --git a/.github/workflows/docs_pr_push.yml b/.github/workflows/docs_pr_push.yml index 58280e53e..8907c3ea7 100644 --- a/.github/workflows/docs_pr_push.yml +++ b/.github/workflows/docs_pr_push.yml @@ -2,7 +2,7 @@ # to documentation or specific workflows are edited, ensuring that the job runs only for # relevant changes. -name: Documentation PR/push +name: Documentation PR/push(testing) on: push: @@ -33,4 +33,5 @@ permissions: jobs: DocsBuild: + if: false uses: ./.github/workflows/reusable_docs_build.yml diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 76e6bef16..7f0a17502 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -3,210 +3,23 @@ name: Nightly # This job is run at 04:00 UTC every day or on demand. on: - workflow_dispatch: - schedule: - - cron: '0 4 * * *' - -permissions: - contents: read + push: + branches-ignore: + - 'dependabot/**' + pull_request: env: BUILD_DIR : "${{github.workspace}}/build" INSTALL_DIR: "${{github.workspace}}/build/install" jobs: - fuzz-test: - name: Fuzz test - strategy: - fail-fast: false - matrix: - build_type: [Debug, Release] - compiler: [{c: clang, cxx: clang++}] - - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - - name: Install apt packages - run: | - sudo apt-get update - sudo apt-get install -y cmake hwloc libhwloc-dev libnuma-dev libtbb-dev - - - name: Find Clang fuzzer lib - run: | - CLANG_LIBS_DIR=$(find /usr/lib -name "libclang_rt.fuzzer_no_main-x86_64.a" -exec dirname {} \; | head -n 1) - echo "CLANG_LIBS_DIR=${CLANG_LIBS_DIR}" >> $GITHUB_ENV - - - name: Configure CMake - run: > - cmake - -B ${{github.workspace}}/build - -DCMAKE_PREFIX_PATH=${{env.CLANG_LIBS_DIR}} - -DCMAKE_BUILD_TYPE=${{matrix.build_type}} - -DCMAKE_C_COMPILER=${{matrix.compiler.c}} - -DCMAKE_CXX_COMPILER=${{matrix.compiler.cxx}} - -DUMF_BUILD_SHARED_LIBRARY=ON - -DUMF_TESTS_FAIL_ON_SKIP=ON - -DUMF_DEVELOPER_MODE=ON - -DUMF_BUILD_FUZZTESTS=ON - - - name: Build - run: cmake --build ${{github.workspace}}/build --config ${{matrix.build_type}} --verbose -j$(nproc) - - - name: Run regular tests - working-directory: ${{github.workspace}}/build - run: ctest -C ${{matrix.build_type}} --output-on-failure -E "fuzz|test_init_teardown" - - - name: Run regular tests with proxy library - working-directory: ${{env.BUILD_DIR}} - run: LD_PRELOAD=./lib/libumf_proxy.so ctest -C ${{matrix.build_type}} --output-on-failure -E "fuzz|test_init_teardown" - - - name: Fuzz long test - working-directory: ${{github.workspace}}/build - run: ctest -C ${{matrix.build_type}} --output-on-failure --verbose -L "fuzz-long" - - valgrind: - name: Valgrind - strategy: - fail-fast: false - matrix: - tool: ['memcheck', 'drd', 'helgrind'] - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - - name: Install apt packages - run: | - sudo apt-get update - sudo apt-get install -y cmake hwloc libhwloc-dev libnuma-dev libtbb-dev valgrind - - - name: Configure CMake - run: > - cmake - -B ${{github.workspace}}/build - -DCMAKE_BUILD_TYPE=Debug - -DUMF_FORMAT_CODE_STYLE=OFF - -DUMF_DEVELOPER_MODE=ON - -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON - -DUMF_BUILD_LEVEL_ZERO_PROVIDER=OFF - -DUMF_BUILD_CUDA_PROVIDER=OFF - -DUMF_USE_VALGRIND=1 - -DUMF_TESTS_FAIL_ON_SKIP=ON - - - name: Build - run: cmake --build ${{github.workspace}}/build --config Debug -j$(nproc) - - - name: Run tests under valgrind - run: ${{github.workspace}}/test/test_valgrind.sh ${{github.workspace}} ${{github.workspace}}/build ${{matrix.tool}} - - Windows-generators: - name: Windows ${{matrix.generator}} generator - strategy: - matrix: - build_type: [Debug, Release] - compiler: [{c: cl, cxx: cl}] - shared_library: ['ON', 'OFF'] - static_hwloc: ['ON', 'OFF'] - generator: ['Ninja', 'NMake Makefiles'] - umfd_lib: ['ON', 'OFF'] - - runs-on: windows-latest - - steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - - name: Set VCPKG_PATH with hwloc - if: matrix.static_hwloc == 'OFF' - run: echo "VCPKG_PATH=${{github.workspace}}/build/vcpkg/packages/hwloc_x64-windows;${{github.workspace}}/build/vcpkg/packages/tbb_x64-windows;${{github.workspace}}/build/vcpkg/packages/jemalloc_x64-windows" >> $env:GITHUB_ENV - - - name: Set VCPKG_PATH without hwloc - if: matrix.static_hwloc == 'ON' - run: echo "VCPKG_PATH=${{github.workspace}}/build/vcpkg/packages/tbb_x64-windows;${{github.workspace}}/build/vcpkg/packages/jemalloc_x64-windows" >> $env:GITHUB_ENV - - - name: Initialize vcpkg - uses: lukka/run-vcpkg@5e0cab206a5ea620130caf672fce3e4a6b5666a1 # v11.5 - env: - VCPKG_PATH: ${{env.VCPKG_PATH}} - with: - vcpkgGitCommitId: ea2a964f9303270322cf3f2d51c265ba146c422d # 1.04.2025 - vcpkgDirectory: ${{env.BUILD_DIR}}/vcpkg - vcpkgJsonGlob: '**/vcpkg.json' - - - name: Install dependencies - run: vcpkg install --triplet x64-windows - - - name: Install Ninja - if: matrix.generator == 'Ninja' - uses: seanmiddleditch/gha-setup-ninja@3b1f8f94a2f8254bd26914c4ab9474d4f0015f67 # v6 - - - name: Configure MSVC environment - uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 - - - name: Configure build - run: > - cmake - -B ${{env.BUILD_DIR}} - -DCMAKE_BUILD_TYPE=${{matrix.build_type}} - -DCMAKE_PREFIX_PATH="${{env.VCPKG_PATH}}" - -DCMAKE_C_COMPILER=${{matrix.compiler.c}} - -DCMAKE_CXX_COMPILER=${{matrix.compiler.cxx}} - -G "${{matrix.generator}}" - -DUMF_BUILD_SHARED_LIBRARY=${{matrix.shared_library}} - -DUMF_LINK_HWLOC_STATICALLY=${{matrix.static_hwloc}} - -DUMF_FORMAT_CODE_STYLE=OFF - -DUMF_DEVELOPER_MODE=ON - -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON - -DUMF_BUILD_LEVEL_ZERO_PROVIDER=ON - -DUMF_BUILD_CUDA_PROVIDER=ON - -DUMF_TESTS_FAIL_ON_SKIP=ON - ${{ matrix.umfd_lib == 'ON' && '-DUMF_USE_DEBUG_POSTFIX=ON' || '' }} - - - name: Build UMF - shell: cmd - run: cmake --build ${{env.BUILD_DIR}} --config ${{matrix.build_type}} -j %NUMBER_OF_PROCESSORS% - - - name: Run tests - shell: cmd - working-directory: ${{env.BUILD_DIR}} - run: ctest -C ${{matrix.build_type}} --output-on-failure --test-dir test - - - name: Get UMF version - run: | - $version = (git describe --tags --abbrev=0 | Select-String -Pattern '\d+\.\d+\.\d+').Matches.Value - echo "UMF_VERSION=$version" >> $env:GITHUB_ENV - shell: pwsh - - - name: Test UMF installation and uninstallation - # The '--shared-library' parameter is added to the installation test when the UMF is built as a shared library - # The '--umfd-lib' parameter is added when the UMF is built with the umfd library - run: > - python3 ${{github.workspace}}/test/test_installation.py - --build-dir ${{env.BUILD_DIR}} - --install-dir ${{env.INSTALL_DIR}} - --build-type ${{matrix.build_type}} - --umf-version ${{env.UMF_VERSION}} - ${{ matrix.shared_library == 'ON' && '--proxy --shared-library' || '' }} - ${{ matrix.umfd_lib == 'ON' && '--umfd-lib' || ''}} - ${{ matrix.static_hwloc == 'ON' && '--hwloc' || '' }} - icx: name: ICX env: VCPKG_PATH: "${{github.workspace}}/build/vcpkg/packages/hwloc_x64-windows;${{github.workspace}}/build/vcpkg/packages/tbb_x64-windows;${{github.workspace}}/build/vcpkg/packages/jemalloc_x64-windows" strategy: matrix: - os: ['windows-2019', 'windows-2022'] + os: ['windows-2022'] build_type: [Debug] compiler: [{c: icx, cxx: icx}] shared_library: ['ON', 'OFF'] @@ -233,6 +46,24 @@ jobs: - name: Install dependencies run: vcpkg install --triplet x64-windows + - name: Install llvm-mingw + run: | + curl -LO https://github.com/mstorsjo/llvm-mingw/releases/download/20250430/llvm-mingw-20250430-ucrt-x86_64.zip + powershell Expand-Archive llvm-mingw*.zip -DestinationPath . + del llvm-mingw*.zip + mv llvm-mingw* c:\llvm-mingw + echo "c:\llvm-mingw\bin" | Out-File -FilePath $Env:GITHUB_PATH -Encoding utf8 -Append + # - name: Download win64devkit + # run: | + # Invoke-RestMethod "https://github.com/skeeto/w64devkit/archive/refs/tags/v2.2.0.zip" -OutFile 'w64devkit.zip' + # - name: install win64devkit + # run: | + # Expand-Archive ./w64devkit.zip -DestinationPath . + + # - name: Add w64devkit to PATH + # run: | + # echo "$(Get-Location)\w64devkit\bin" | Out-File -Append -FilePath $env:GITHUB_PATH -Encoding utf8 + - name: Install Ninja uses: seanmiddleditch/gha-setup-ninja@3b1f8f94a2f8254bd26914c4ab9474d4f0015f67 # v6 @@ -254,6 +85,7 @@ jobs: - name: Configure build shell: cmd run: | + rmdir ${{env.BUILD_DIR}} /s /q call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" call "C:\Program Files (x86)\Intel\oneAPI\setvars-vcvarsall.bat" cmake ^ @@ -261,7 +93,7 @@ jobs: -DCMAKE_PREFIX_PATH="${{env.VCPKG_PATH}}" ^ -DCMAKE_C_COMPILER=${{matrix.compiler.c}} ^ -DCMAKE_CXX_COMPILER=${{matrix.compiler.cxx}} ^ - -G Ninja ^ + -G "Unix Makefiles" ^ -DUMF_BUILD_SHARED_LIBRARY=${{matrix.shared_library}} ^ -DUMF_FORMAT_CODE_STYLE=OFF ^ -DUMF_DEVELOPER_MODE=ON ^ @@ -285,88 +117,3 @@ jobs: call "C:\Program Files (x86)\Intel\oneAPI\setvars-vcvarsall.bat" ctest -C ${{matrix.build_type}} --output-on-failure --test-dir test - # Scenarios where UMF_LINK_HWLOC_STATICALLY is set to OFF and hwloc is not installed in the system - # The hwloc library is fetched implicitly - hwloc-fallback: - name: "Fallback to static hwloc build" - strategy: - matrix: - include: - - os: 'ubuntu-latest' - build_type: Release - number_of_processors: '$(nproc)' - - os: 'windows-latest' - build_type: Release - number_of_processors: '$Env:NUMBER_OF_PROCESSORS' - - runs-on: ${{matrix.os}} - - steps: - - name: Install dependencies - if: matrix.os == 'ubuntu-latest' - run: sudo apt-get install -y libnuma-dev - - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - - name: Configure build - run: > - cmake - -B ${{env.BUILD_DIR}} - -DCMAKE_BUILD_TYPE=${{matrix.build_type}} - -DUMF_BUILD_SHARED_LIBRARY=ON - -DUMF_BUILD_EXAMPLES=OFF - -DUMF_DEVELOPER_MODE=ON - -DUMF_LINK_HWLOC_STATICALLY=OFF - -DUMF_TESTS_FAIL_ON_SKIP=ON - - - name: Build UMF - run: > - cmake - --build ${{env.BUILD_DIR}} - --config ${{matrix.build_type}} - -j ${{matrix.number_of_processors}} - - - name: Run tests - working-directory: ${{env.BUILD_DIR}} - run: ctest -C ${{matrix.build_type}} --output-on-failure --test-dir test - - L0: - uses: ./.github/workflows/reusable_gpu.yml - with: - provider: "LEVEL_ZERO" - runner: "L0" - L0-BMG: - uses: ./.github/workflows/reusable_gpu.yml - with: - provider: "LEVEL_ZERO" - runner: "L0-BMG" - os: "['Ubuntu']" - CUDA: - uses: ./.github/workflows/reusable_gpu.yml - with: - provider: "CUDA" - runner: "CUDA" - - # Full execution of QEMU tests - QEMU: - uses: ./.github/workflows/reusable_qemu.yml - with: - short_run: false - # Beside the 2 LTS Ubuntu, we also test this on the latest Ubuntu - to be updated - # every 6 months, so we verify the latest version of packages (compilers, etc.). - os: "['ubuntu-22.04', 'ubuntu-24.04', 'ubuntu-24.10']" - - Benchmarks: - uses: ./.github/workflows/reusable_benchmarks.yml - permissions: - contents: write - pull-requests: write - with: - pr_no: '0' - bench_script_params: '--save Baseline_PVC' - - SYCL: - uses: ./.github/workflows/reusable_sycl.yml diff --git a/.github/workflows/pr_push.yml b/.github/workflows/pr_push.yml index 5d6155a37..b53fa7278 100644 --- a/.github/workflows/pr_push.yml +++ b/.github/workflows/pr_push.yml @@ -2,6 +2,9 @@ # This workflow only call other workflows. name: PR/push +env: + ENABLE_TESTING_JOBS: false + on: push: branches-ignore: @@ -17,22 +20,28 @@ permissions: jobs: CodeChecks: + if: github.repository == 'octo-org/octo-repo-prod' uses: ./.github/workflows/reusable_checks.yml FastBuild: + if: github.repository == 'octo-org/octo-repo-prod' name: Fast builds needs: [CodeChecks] uses: ./.github/workflows/reusable_fast.yml Build: + if: github.repository == 'octo-org/octo-repo-prod' name: Basic builds needs: [FastBuild] uses: ./.github/workflows/reusable_basic.yml DevDax: + if: github.repository == 'octo-org/octo-repo-prod' needs: [FastBuild] uses: ./.github/workflows/reusable_dax.yml MultiNuma: + if: github.repository == 'octo-org/octo-repo-prod' needs: [FastBuild] uses: ./.github/workflows/reusable_multi_numa.yml L0: + if: github.repository == 'octo-org/octo-repo-prod' needs: [Build] uses: ./.github/workflows/reusable_gpu.yml with: @@ -40,6 +49,7 @@ jobs: runner: "L0" shared_lib: "['ON']" L0-BMG: + if: github.repository == 'octo-org/octo-repo-prod' needs: [Build] uses: ./.github/workflows/reusable_gpu.yml with: @@ -48,6 +58,7 @@ jobs: shared_lib: "['ON']" os: "['Ubuntu']" CUDA: + if: github.repository == 'octo-org/octo-repo-prod' needs: [Build] uses: ./.github/workflows/reusable_gpu.yml with: @@ -55,33 +66,38 @@ jobs: runner: "CUDA" shared_lib: "['ON']" Sanitizers: + if: github.repository == 'octo-org/octo-repo-prod' needs: [FastBuild] uses: ./.github/workflows/reusable_sanitizers.yml QEMU: + if: github.repository == 'octo-org/octo-repo-prod' needs: [FastBuild] uses: ./.github/workflows/reusable_qemu.yml with: short_run: true ProxyLib: + if: github.repository == 'octo-org/octo-repo-prod' needs: [Build] uses: ./.github/workflows/reusable_proxy_lib.yml Valgrind: + if: github.repository == 'octo-org/octo-repo-prod' needs: [Build] uses: ./.github/workflows/reusable_valgrind.yml Coverage: + # total coverage (on upstream only) - if: github.repository == 'oneapi-src/unified-memory-framework' + if: github.repository == 'octo-org/octo-repo-prod' needs: [Build, DevDax, L0, CUDA, MultiNuma, QEMU, ProxyLib] uses: ./.github/workflows/reusable_coverage.yml secrets: inherit with: trigger: "${{github.event_name}}" Coverage_partial: - # partial coverage (on forks) - if: github.repository != 'oneapi-src/unified-memory-framework' + if: github.repository == 'octo-org/octo-repo-prod' needs: [Build, QEMU, ProxyLib] uses: ./.github/workflows/reusable_coverage.yml CodeQL: + if: github.repository == 'octo-org/octo-repo-prod' needs: [Build] permissions: contents: read