From b668842128b05d6e13e1458bbe12e6152c000e5d Mon Sep 17 00:00:00 2001 From: rbanka1 Date: Mon, 5 May 2025 15:58:54 +0200 Subject: [PATCH 01/14] add dockers --- .github/docker/ubuntu-20.04.Dockerfile | 31 +++-- .github/docker/ubuntu-22.04.Dockerfile | 16 +-- .github/docker/ubuntu-24.04.Dockerfile | 65 ++++++++++ .github/workflows/pr_push.yml | 49 +++++++- .github/workflows/reusable_basic.yml | 85 ++++++------- .github/workflows/reusable_dockers_build.yml | 42 +++++++ .github/workflows/reusable_fast.yml | 124 +++++++++++++------ 7 files changed, 303 insertions(+), 109 deletions(-) create mode 100644 .github/docker/ubuntu-24.04.Dockerfile create mode 100644 .github/workflows/reusable_dockers_build.yml diff --git a/.github/docker/ubuntu-20.04.Dockerfile b/.github/docker/ubuntu-20.04.Dockerfile index a6a45a8c1..9c13d741c 100644 --- a/.github/docker/ubuntu-20.04.Dockerfile +++ b/.github/docker/ubuntu-20.04.Dockerfile @@ -1,10 +1,10 @@ -# Copyright (C) 2024 Intel Corporation +# Copyright (C) 2024-2025 Intel Corporation # Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # # Dockerfile - a 'recipe' for Docker to build an image of ubuntu-based -# environment for building the Unified Memory Framework project. +# environment for building the Unified Memory Framework project. # # Pull base image ("20.04") @@ -22,14 +22,22 @@ ARG BASE_DEPS="\ cmake \ git" +# Hwloc installation dependencies +ARG HWLOC_DEPS="\ + dos2unix \ + libtool" + +# Copy hwloc +COPY .github/scripts/install_hwloc.sh /opt/umf/install_hwloc.sh + # UMF's dependencies -ARG UMF_DEPS="\ - libhwloc-dev \ - libtbb-dev" +# libhwloc-dev is required - installed via script because hwloc verion is to old on this OS # Dependencies for tests (optional) ARG TEST_DEPS="\ - libnuma-dev" + libnuma-dev \ + libtbb-dev\ + valgrind" # Miscellaneous for our builds/CI (optional) ARG MISC_DEPS="\ @@ -38,24 +46,27 @@ ARG MISC_DEPS="\ g++-7 \ python3-pip \ sudo \ - whois" + whois \ + lcov" # Update and install required packages RUN apt-get update \ && apt-get install -y --no-install-recommends \ ${BASE_DEPS} \ - ${UMF_DEPS} \ ${TEST_DEPS} \ ${MISC_DEPS} \ + ${HWLOC_DEPS} \ + && /opt/umf/install_hwloc.sh \ && rm -rf /var/lib/apt/lists/* \ && apt-get clean all # Prepare a dir (accessible by anyone) -RUN mkdir --mode 777 /opt/umf/ +RUN mkdir -p --mode 777 /opt/umf/ # Additional dependencies (installed via pip) +# It's actively used and tested only on selected distros. Be aware +# they may not work, because pip packages list differ from OS to OS. COPY third_party/requirements.txt /opt/umf/requirements.txt -RUN pip3 install --no-cache-dir -r /opt/umf/requirements.txt # Add a new (non-root) 'test_user' ENV USER test_user diff --git a/.github/docker/ubuntu-22.04.Dockerfile b/.github/docker/ubuntu-22.04.Dockerfile index 75c71c526..8d63d2347 100644 --- a/.github/docker/ubuntu-22.04.Dockerfile +++ b/.github/docker/ubuntu-22.04.Dockerfile @@ -1,4 +1,4 @@ -# Copyright (C) 2024 Intel Corporation +# Copyright (C) 2024-2025 Intel Corporation # Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -24,12 +24,13 @@ ARG BASE_DEPS="\ # UMF's dependencies ARG UMF_DEPS="\ - libhwloc-dev \ - libtbb-dev" + libhwloc-dev" # Dependencies for tests (optional) ARG TEST_DEPS="\ - libnuma-dev" + libnuma-dev \ + libtbb-dev\ + valgrind" # Miscellaneous for our builds/CI (optional) ARG MISC_DEPS="\ @@ -37,20 +38,21 @@ ARG MISC_DEPS="\ clang \ python3-pip \ sudo \ - whois" + whois \ + lcov" # Update and install required packages RUN apt-get update \ && apt-get install -y --no-install-recommends \ ${BASE_DEPS} \ - ${UMF_DEPS} \ ${TEST_DEPS} \ ${MISC_DEPS} \ + ${UMF_DEPS} \ && rm -rf /var/lib/apt/lists/* \ && apt-get clean all # Prepare a dir (accessible by anyone) -RUN mkdir --mode 777 /opt/umf/ +RUN mkdir -p --mode 777 /opt/umf/ # Additional dependencies (installed via pip) COPY third_party/requirements.txt /opt/umf/requirements.txt diff --git a/.github/docker/ubuntu-24.04.Dockerfile b/.github/docker/ubuntu-24.04.Dockerfile new file mode 100644 index 000000000..cd45cc52c --- /dev/null +++ b/.github/docker/ubuntu-24.04.Dockerfile @@ -0,0 +1,65 @@ +# Copyright (C) 2025 Intel Corporation +# Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +# +# Dockerfile - a 'recipe' for Docker to build an image of ubuntu-based +# environment for building the Unified Memory Framework project. +# + +# Pull base image ("24.04") +FROM registry.hub.docker.com/library/ubuntu@sha256:72297848456d5d37d1262630108ab308d3e9ec7ed1c3286a32fe09856619a782 + +# Set environment variables +ENV OS ubuntu +ENV OS_VER 24.04 +ENV NOTTY 1 +ENV DEBIAN_FRONTEND noninteractive + +# Base development packages +ARG BASE_DEPS="\ + build-essential \ + cmake \ + git" + +# UMF's dependencies +ARG UMF_DEPS="\ + libhwloc-dev" + +# Dependencies for tests (optional) +ARG TEST_DEPS="\ + libnuma-dev \ + libtbb-dev\ + valgrind" + +# Miscellaneous for our builds/CI (optional) +ARG MISC_DEPS="\ + automake \ + clang \ + python3-pip \ + sudo \ + whois \ + lcov" + +# Update and install required packages +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ${BASE_DEPS} \ + ${TEST_DEPS} \ + ${MISC_DEPS} \ + ${UMF_DEPS} \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get clean all + +# Prepare a dir (accessible by anyone) +RUN mkdir -p --mode 777 /opt/umf/ + +# Additional dependencies (installed via pip) +COPY third_party/requirements.txt /opt/umf/requirements.txt +RUN pip3 install --no-cache-dir --break-system-packages -r /opt/umf/requirements.txt + +# Add a new (non-root) 'test_user' +ENV USER test_user +ENV USERPASS pass +RUN useradd -m "${USER}" -g sudo -p "$(mkpasswd ${USERPASS})" +USER test_user diff --git a/.github/workflows/pr_push.yml b/.github/workflows/pr_push.yml index 5d6155a37..174af58c8 100644 --- a/.github/workflows/pr_push.yml +++ b/.github/workflows/pr_push.yml @@ -14,25 +14,59 @@ concurrency: permissions: contents: read + packages: read jobs: CodeChecks: uses: ./.github/workflows/reusable_checks.yml + DocsBuild: + uses: ./.github/workflows/reusable_docs_build.yml + DetectChanges: + runs-on: ubuntu-latest + outputs: + changed_files: ${{ steps.changed-files.outputs.all_changed_files }} + steps: + - name: Checkout code + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v45.0.3 + + - name: List all changed files + env: + ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} + run: | + echo "Changed files: ${{ steps.changed-files.outputs.all_changed_files }}" + BuildDockers: + if: ${{ contains(join(needs.DetectChanges.outputs.changed_files, ' '), '.github/docker/') }} + needs: [DetectChanges] + permissions: + contents: read + packages: write + secrets: inherit + uses: ./.github/workflows/reusable_dockers_build.yml FastBuild: - name: Fast builds - needs: [CodeChecks] + if: always() && (needs.BuildDockers.result == 'skipped' || needs.BuildDockers.result == 'success') + needs: [CodeChecks, DocsBuild, BuildDockers] uses: ./.github/workflows/reusable_fast.yml Build: name: Basic builds + if: always() && (needs.FastBuild.result == 'success') needs: [FastBuild] uses: ./.github/workflows/reusable_basic.yml DevDax: + if: always() && (needs.FastBuild.result == 'success') needs: [FastBuild] uses: ./.github/workflows/reusable_dax.yml MultiNuma: + if: always() && (needs.FastBuild.result == 'success') needs: [FastBuild] uses: ./.github/workflows/reusable_multi_numa.yml L0: + if: always() && (needs.Build.result == 'success') needs: [Build] uses: ./.github/workflows/reusable_gpu.yml with: @@ -40,6 +74,7 @@ jobs: runner: "L0" shared_lib: "['ON']" L0-BMG: + if: always() && (needs.Build.result == 'success') needs: [Build] uses: ./.github/workflows/reusable_gpu.yml with: @@ -48,6 +83,7 @@ jobs: shared_lib: "['ON']" os: "['Ubuntu']" CUDA: + if: always() && (needs.Build.result == 'success') needs: [Build] uses: ./.github/workflows/reusable_gpu.yml with: @@ -55,17 +91,21 @@ jobs: runner: "CUDA" shared_lib: "['ON']" Sanitizers: + if: always() && (needs.FastBuild.result == 'success') needs: [FastBuild] uses: ./.github/workflows/reusable_sanitizers.yml QEMU: + if: always() && (needs.FastBuild.result == 'success') needs: [FastBuild] uses: ./.github/workflows/reusable_qemu.yml with: short_run: true ProxyLib: + if: always() && (needs.Build.result == 'success') needs: [Build] uses: ./.github/workflows/reusable_proxy_lib.yml Valgrind: + if: always() && (needs.Build.result == 'success') needs: [Build] uses: ./.github/workflows/reusable_valgrind.yml Coverage: @@ -78,22 +118,25 @@ jobs: trigger: "${{github.event_name}}" Coverage_partial: # partial coverage (on forks) - if: github.repository != 'oneapi-src/unified-memory-framework' + if: github.repository != 'oneapi-src/unified-memory-framework' && always() && (needs.Build.result == 'success') needs: [Build, QEMU, ProxyLib] uses: ./.github/workflows/reusable_coverage.yml CodeQL: + if: always() && (needs.Build.result == 'success') needs: [Build] permissions: contents: read security-events: write uses: ./.github/workflows/reusable_codeql.yml Trivy: + if: always() && (needs.Build.result == 'success') needs: [Build] permissions: contents: read security-events: write uses: ./.github/workflows/reusable_trivy.yml Compatibility: + if: always() && (needs.Build.result == 'success') needs: [Build] uses: ./.github/workflows/reusable_compatibility.yml strategy: diff --git a/.github/workflows/reusable_basic.yml b/.github/workflows/reusable_basic.yml index 26fbdf6e9..7bb1a1398 100644 --- a/.github/workflows/reusable_basic.yml +++ b/.github/workflows/reusable_basic.yml @@ -5,19 +5,27 @@ on: workflow_call permissions: contents: read + packages: read env: BUILD_DIR : "${{github.workspace}}/build" - INSTL_DIR : "${{github.workspace}}/install-dir" + INSTL_DIR : "${{github.workspace}}/../install-dir" COVERAGE_DIR : "${{github.workspace}}/coverage" COVERAGE_NAME : "exports-coverage-basic" jobs: ubuntu-build: name: Ubuntu + runs-on: ubuntu-latest + container: + # image: ghcr.io/bb-ur/umf-ubuntu-${{ matrix.ubuntu_ver }}:latest + image: ghcr.io/rbanka1/umf-ubuntu-${{ matrix.ubuntu_ver }}:latest # test + options: --user root --privileged + volumes: + - ${{ github.workspace }}:${{ github.workspace }} strategy: matrix: - os: ['ubuntu-22.04', 'ubuntu-24.04'] + ubuntu_ver: ['22.04', '24.04'] build_type: [Debug, Release] compiler: [{c: gcc, cxx: g++}] shared_library: ['OFF'] @@ -27,7 +35,7 @@ jobs: disable_hwloc: ['OFF'] link_hwloc_statically: ['OFF'] include: - - os: 'ubuntu-22.04' + - ubuntu_ver: 22.04 build_type: Release compiler: {c: clang, cxx: clang++} shared_library: 'OFF' @@ -36,7 +44,7 @@ jobs: install_tbb: 'ON' disable_hwloc: 'OFF' link_hwloc_statically: 'OFF' - - os: 'ubuntu-22.04' + - ubuntu_ver: 22.04 build_type: Release compiler: {c: gcc, cxx: g++} shared_library: 'ON' @@ -45,7 +53,7 @@ jobs: install_tbb: 'ON' disable_hwloc: 'OFF' link_hwloc_statically: 'OFF' - - os: 'ubuntu-24.04' + - ubuntu_ver: 24.04 build_type: Debug compiler: {c: gcc, cxx: g++} shared_library: 'ON' @@ -55,7 +63,7 @@ jobs: disable_hwloc: 'OFF' link_hwloc_statically: 'OFF' # test level_zero_provider='OFF' and cuda_provider='OFF' - - os: 'ubuntu-22.04' + - ubuntu_ver: 22.04 build_type: Release compiler: {c: gcc, cxx: g++} shared_library: 'OFF' @@ -65,7 +73,7 @@ jobs: disable_hwloc: 'OFF' link_hwloc_statically: 'OFF' # test icx compiler - - os: 'ubuntu-22.04' + - ubuntu_ver: 22.04 build_type: Release compiler: {c: icx, cxx: icpx} shared_library: 'ON' @@ -75,7 +83,7 @@ jobs: disable_hwloc: 'OFF' link_hwloc_statically: 'OFF' # test lld linker - - os: 'ubuntu-24.04' + - ubuntu_ver: 24.04 build_type: Release compiler: {c: icx, cxx: icpx} shared_library: 'ON' @@ -86,7 +94,7 @@ jobs: link_hwloc_statically: 'OFF' llvm_linker: '-DCMAKE_EXE_LINKER_FLAGS="-fuse-ld=lld" -DCMAKE_MODULE_LINKER_FLAGS="-fuse-ld=lld" -DCMAKE_SHARED_LINKER_FLAGS="-fuse-ld=lld"' # test without installing TBB - - os: 'ubuntu-22.04' + - ubuntu_ver: 22.04 build_type: Release compiler: {c: gcc, cxx: g++} shared_library: 'ON' @@ -95,7 +103,7 @@ jobs: install_tbb: 'OFF' disable_hwloc: 'OFF' link_hwloc_statically: 'OFF' - - os: 'ubuntu-22.04' + - ubuntu_ver: 22.04 build_type: Debug compiler: {c: gcc, cxx: g++} shared_library: 'ON' @@ -104,7 +112,7 @@ jobs: install_tbb: 'ON' disable_hwloc: 'ON' link_hwloc_statically: 'OFF' - - os: 'ubuntu-22.04' + - ubuntu_ver: 22.04 build_type: Release compiler: {c: gcc, cxx: g++} shared_library: 'ON' @@ -113,46 +121,21 @@ jobs: install_tbb: 'ON' disable_hwloc: 'OFF' link_hwloc_statically: 'ON' - runs-on: ${{matrix.os}} - steps: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: fetch-depth: 0 - - name: Install apt packages - run: | - sudo apt-get update - sudo apt-get install -y clang cmake libnuma-dev lcov - - - name: Install hwloc - if: matrix.disable_hwloc == 'OFF' - run: | - sudo apt-get install -y libhwloc-dev - - - name: Install TBB apt package - if: matrix.install_tbb == 'ON' - run: | - sudo apt-get install -y libtbb-dev - - name: Install oneAPI basekit if: matrix.compiler.cxx == 'icpx' run: | + sudo apt-get update sudo apt-get install -y gpg-agent wget wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | sudo tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list sudo apt-get update sudo apt-get install -y intel-oneapi-ippcp-devel intel-oneapi-ipp-devel intel-oneapi-common-oneapi-vars intel-oneapi-compiler-dpcpp-cpp - - - name: Install g++-7 - if: matrix.compiler.cxx == 'g++-7' - run: sudo apt-get install -y ${{matrix.compiler.cxx}} - - - name: Get UMF version - run: | - VERSION=$(git describe --tags --abbrev=0 | grep -oP '\d+\.\d+\.\d+') - echo "UMF_VERSION=$VERSION" >> $GITHUB_ENV - name: Configure build run: > @@ -173,7 +156,6 @@ jobs: -DUMF_DISABLE_HWLOC=${{matrix.disable_hwloc}} -DUMF_LINK_HWLOC_STATICALLY=${{matrix.link_hwloc_statically}} ${{ matrix.build_type == 'Debug' && matrix.compiler.c == 'gcc' && '-DUMF_USE_COVERAGE=ON' || '' }} - ${{ matrix.llvm_linker || '' }} - name: Build UMF run: | @@ -186,11 +168,14 @@ jobs: ${{ matrix.compiler.cxx == 'icpx' && '. /opt/intel/oneapi/setvars.sh' || true }} LD_LIBRARY_PATH="${{env.BUILD_DIR}}/lib/:${LD_LIBRARY_PATH}" ctest --output-on-failure + - name: Set OS name + run: echo "OS_VER=ubuntu-${{ matrix.ubuntu_ver }}" >> $GITHUB_ENV + - name: Check coverage if: ${{ matrix.build_type == 'Debug' && matrix.compiler.c == 'gcc' }} working-directory: ${{env.BUILD_DIR}} run: | - export COVERAGE_FILE_NAME=${{env.COVERAGE_NAME}}-${{matrix.os}}-shared-${{matrix.shared_library}}-no_hwloc-${{matrix.disable_hwloc}} + export COVERAGE_FILE_NAME=${{env.COVERAGE_NAME}}-${{env.OS_VER}}-shared-${{matrix.shared_library}}-no_hwloc-${{matrix.disable_hwloc}} echo "COVERAGE_FILE_NAME: $COVERAGE_FILE_NAME" ../scripts/coverage/coverage_capture.sh $COVERAGE_FILE_NAME mkdir -p ${{env.COVERAGE_DIR}} @@ -199,22 +184,22 @@ jobs: - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 if: ${{ matrix.build_type == 'Debug' && matrix.compiler.c == 'gcc' }} with: - name: ${{env.COVERAGE_NAME}}-${{matrix.os}}-shared-${{matrix.shared_library}}-no_hwloc-${{matrix.disable_hwloc}} + name: ${{env.COVERAGE_NAME}}-${{env.OS_VER}}-shared-${{matrix.shared_library}}-no_hwloc-${{matrix.disable_hwloc}} path: ${{env.COVERAGE_DIR}} - name: Remove the installation directory run: rm -rf ${{env.INSTL_DIR}} - - name: Test UMF installation and uninstallation - # The '--shared-library' parameter is added to the installation test when the UMF is built as a shared library - run: > - python3 ${{github.workspace}}/test/test_installation.py - --build-dir ${{env.BUILD_DIR}} - --install-dir ${{env.INSTL_DIR}} - --build-type ${{matrix.build_type}} - ${{ matrix.install_tbb == 'ON' && matrix.disable_hwloc != 'ON' && matrix.shared_library == 'ON' && '--proxy' || '' }} - --umf-version ${{env.UMF_VERSION}} - ${{ matrix.shared_library == 'ON' && '--shared-library' || '' }} + # - name: Test UMF installation and uninstallation + # # The '--shared-library' parameter is added to the installation test when the UMF is built as a shared library + # run: > + # python3 ${{github.workspace}}/test/test_installation.py + # --build-dir ${{env.BUILD_DIR}} + # --install-dir ${{env.INSTL_DIR}} + # --build-type ${{matrix.build_type}} + # ${{ matrix.install_tbb == 'ON' && matrix.disable_hwloc != 'ON' && matrix.shared_library == 'ON' && '--proxy' || '' }} + # --umf-version ${{env.UMF_VERSION}} + # ${{ matrix.shared_library == 'ON' && '--shared-library' || '' }} windows-build: name: Windows diff --git a/.github/workflows/reusable_dockers_build.yml b/.github/workflows/reusable_dockers_build.yml new file mode 100644 index 000000000..53bb727fe --- /dev/null +++ b/.github/workflows/reusable_dockers_build.yml @@ -0,0 +1,42 @@ +# Build and push Docker images to GHCR +name: BuildDockers + +on: + workflow_call: + workflow_dispatch: + +permissions: + packages: write + contents: read + +jobs: + build-dockers: + runs-on: ubuntu-latest + strategy: + matrix: + ubuntu-version: [20.04, 22.04, 24.04] + outputs: + status: ${{ job.status }} + env: + GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }} # test + IMG: ghcr.io/bb-ur/umf-ubuntu-${{ matrix.ubuntu-version }}:latest + + steps: + - name: Checkout repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + + - name: Login to GitHub Container Registry + uses: docker/login-action@30f019fb76bb54d03ec1e716054622be511a13b2 # v3.2.0 + with: + registry: ghcr.io + username: ${{ github.actor }} # test + password: ${{ env.GHCR_TOKEN }} # test + # username: bb-ur + # password: ${{ secrets.BB_GHCR_TOKEN }} + + - name: Build and push ubuntu-${{ matrix.ubuntu-version }} Docker image + run: | + docker build -f .github/docker/ubuntu-${{ matrix.ubuntu-version }}.Dockerfile -t ${{ env.IMG }} . + docker push ${{ env.IMG }} diff --git a/.github/workflows/reusable_fast.yml b/.github/workflows/reusable_fast.yml index 690cba072..7f8da2e2a 100644 --- a/.github/workflows/reusable_fast.yml +++ b/.github/workflows/reusable_fast.yml @@ -1,54 +1,110 @@ -# Fast builds +# Fast build linux part is working in dockers, Windows is not name: FastBuild on: workflow_call permissions: contents: read + packages: read env: BUILD_DIR : "${{github.workspace}}/build" - INSTL_DIR : "${{github.workspace}}/install-dir" + INSTL_DIR : "${{github.workspace}}/../install-dir" jobs: - FastBuild: - name: Fast builds - env: - VCPKG_PATH: "${{github.workspace}}/build/vcpkg/packages/hwloc_x64-windows;${{github.workspace}}/build/vcpkg/packages/tbb_x64-windows;${{github.workspace}}/build/vcpkg/packages/jemalloc_x64-windows" + ubuntu-build: + name: Linux + runs-on: ${{ github.repository_owner == 'oneapi-src' && 'intel-ubuntu-22.04' || 'ubuntu-22.04' }} + container: + # image: ghcr.io/bb-ur/umf-ubuntu-${{ matrix.ubuntu_ver }}:latest + image: ghcr.io/rbanka1/umf-ubuntu-${{ matrix.ubuntu_ver }}:latest # test + options: --user root --privileged + volumes: + - ${{ github.workspace }}:${{ github.workspace }} strategy: matrix: include: - - os: windows-latest + - ubuntu_ver: 22.04 build_tests: 'ON' + extra_build_options: ' -DUMF_BUILD_BENCHMARKS=ON -DUMF_BUILD_BENCHMARKS_MT=ON' simple_cmake: 'OFF' - # pure C build (Windows) - - os: windows-latest + # pure C build + - ubuntu_ver: 22.04 # Tests' building is off for a pure C build build_tests: 'OFF' + extra_build_options: '-DUMF_BUILD_BENCHMARKS=ON' simple_cmake: 'OFF' - - os: ubuntu-latest + # simplest CMake on Ubuntu 22.04, 24.04, 20.04 + - ubuntu_ver: 22.04 build_tests: 'ON' - # Windows doesn't recognize 'CMAKE_BUILD_TYPE', it uses '--config' param in build command - extra_build_options: '-DCMAKE_BUILD_TYPE=Release -DUMF_BUILD_BENCHMARKS=ON -DUMF_BUILD_BENCHMARKS_MT=ON' - simple_cmake: 'OFF' - # pure C build (Linux) - - os: ubuntu-latest - # Windows doesn't recognize 'CMAKE_BUILD_TYPE', it uses '--config' param in build command - # Tests' building is off for a pure C build - build_tests: 'OFF' - extra_build_options: '-DCMAKE_BUILD_TYPE=Release -DUMF_BUILD_BENCHMARKS=ON' - simple_cmake: 'OFF' - # simplest CMake on ubuntu-latest - - os: ubuntu-latest + simple_cmake: 'ON' + - ubuntu_ver: 24.04 build_tests: 'ON' - extra_build_options: '-DCMAKE_BUILD_TYPE=Release' simple_cmake: 'ON' - # simplest CMake ubuntu-22.04 - - os: ubuntu-22.04 + - ubuntu_ver: 20.04 build_tests: 'ON' - extra_build_options: '-DCMAKE_BUILD_TYPE=Release' simple_cmake: 'ON' - runs-on: ${{ (matrix.os == 'ubuntu-latest' && github.repository_owner == 'oneapi-src') && 'intel-ubuntu-22.04' || matrix.os }} + + steps: + - name: Checkout repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + + - name: Configure CMake + if: matrix.simple_cmake == 'OFF' + run: > + cmake + -B ${{env.BUILD_DIR}} + -DCMAKE_BUILD_TYPE=Release + -DUMF_FORMAT_CODE_STYLE=OFF + -DUMF_DEVELOPER_MODE=ON + -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON + -DUMF_BUILD_TESTS=${{matrix.build_tests}} + -DUMF_BUILD_EXAMPLES=ON + -DUMF_BUILD_LEVEL_ZERO_PROVIDER=ON + -DUMF_BUILD_CUDA_PROVIDER=ON + -DUMF_TESTS_FAIL_ON_SKIP=ON + -DUMF_BUILD_SHARED_LIBRARY=ON + ${{matrix.extra_build_options}} + + - name: Configure CMake (simple) + if: matrix.simple_cmake == 'ON' + run: > + cmake + -B ${{env.BUILD_DIR}} + -DCMAKE_BUILD_TYPE=Release + -DUMF_BUILD_SHARED_LIBRARY=ON + -DUMF_TESTS_FAIL_ON_SKIP=ON + ${{matrix.extra_build_options}} + + - name: Build + run: cmake --build ${{env.BUILD_DIR}} --config Release -j $(nproc) + + - name: Run examples + working-directory: ${{env.BUILD_DIR}} + run: ctest --output-on-failure --test-dir examples -C Release + + - name: Run tests + if: matrix.build_tests == 'ON' + working-directory: ${{env.BUILD_DIR}} + run: ctest --output-on-failure --test-dir test -C Release + + windows-build: + name: Windows + runs-on: windows-latest + + env: + VCPKG_PATH: "${{github.workspace}}/build/vcpkg/packages/hwloc_x64-windows;${{github.workspace}}/build/vcpkg/packages/tbb_x64-windows;${{github.workspace}}/build/vcpkg/packages/jemalloc_x64-windows" + strategy: + matrix: + include: + - build_tests: 'ON' + simple_cmake: 'OFF' + - build_tests: 'OFF' + simple_cmake: 'OFF' + - build_tests: 'OFF' + simple_cmake: 'ON' steps: - name: Checkout repository @@ -57,24 +113,16 @@ jobs: fetch-depth: 0 - name: Initialize vcpkg - if: matrix.os == 'windows-latest' uses: lukka/run-vcpkg@5e0cab206a5ea620130caf672fce3e4a6b5666a1 # v11.5 with: vcpkgGitCommitId: ea2a964f9303270322cf3f2d51c265ba146c422d # 1.04.2025 vcpkgDirectory: ${{env.BUILD_DIR}}/vcpkg vcpkgJsonGlob: '**/vcpkg.json' - - name: Install dependencies (windows-latest) - if: matrix.os == 'windows-latest' + - name: Install dependencies run: vcpkg install --triplet x64-windows shell: pwsh # Specifies PowerShell as the shell for running the script. - - name: Install dependencies - if: matrix.os != 'windows-latest' - run: | - sudo apt-get update - sudo apt-get install -y cmake libhwloc-dev libnuma-dev libtbb-dev - - name: Configure CMake if: matrix.simple_cmake == 'OFF' run: > @@ -91,7 +139,6 @@ jobs: -DUMF_BUILD_CUDA_PROVIDER=ON -DUMF_TESTS_FAIL_ON_SKIP=ON -DUMF_BUILD_SHARED_LIBRARY=ON - ${{matrix.extra_build_options}} - name: Configure CMake (simple) if: matrix.simple_cmake == 'ON' @@ -99,9 +146,9 @@ jobs: cmake -B ${{env.BUILD_DIR}} -DCMAKE_INSTALL_PREFIX="${{env.INSTL_DIR}}" + -DCMAKE_PREFIX_PATH="${{env.VCPKG_PATH}}" -DUMF_BUILD_SHARED_LIBRARY=ON -DUMF_TESTS_FAIL_ON_SKIP=ON - ${{matrix.extra_build_options}} - name: Build run: cmake --build ${{env.BUILD_DIR}} --config Release -j @@ -118,7 +165,6 @@ jobs: # TODO: We could add some script to verify metadata of dll's (selected fields, perhaps) # ref. https://superuser.com/questions/381276/what-are-some-nice-command-line-ways-to-inspect-dll-exe-details - name: Print metadata of our dll's - if: matrix.os == 'windows-latest' run: | get-command ${{github.workspace}}/build/bin/Release/umf.dll | format-list get-command ${{github.workspace}}/build/src/proxy_lib/Release/umf_proxy.dll | format-list From c2d35fc48faffd366e0613f3d4e52493e1fd7b99 Mon Sep 17 00:00:00 2001 From: rbanka1 Date: Tue, 6 May 2025 17:25:01 +0200 Subject: [PATCH 02/14] fix --- .github/docker/ubuntu-20.04.Dockerfile | 2 +- .github/docker/ubuntu-22.04.Dockerfile | 3 ++- .github/docker/ubuntu-24.04.Dockerfile | 3 ++- .github/workflows/reusable_basic.yml | 3 +-- .github/workflows/reusable_dockers_build.yml | 7 ++----- .github/workflows/reusable_fast.yml | 8 ++------ 6 files changed, 10 insertions(+), 16 deletions(-) diff --git a/.github/docker/ubuntu-20.04.Dockerfile b/.github/docker/ubuntu-20.04.Dockerfile index 9c13d741c..ceb4c3f13 100644 --- a/.github/docker/ubuntu-20.04.Dockerfile +++ b/.github/docker/ubuntu-20.04.Dockerfile @@ -31,7 +31,7 @@ ARG HWLOC_DEPS="\ COPY .github/scripts/install_hwloc.sh /opt/umf/install_hwloc.sh # UMF's dependencies -# libhwloc-dev is required - installed via script because hwloc verion is to old on this OS +# libhwloc-dev is required - installed via script because hwloc version is to old on this OS # Dependencies for tests (optional) ARG TEST_DEPS="\ diff --git a/.github/docker/ubuntu-22.04.Dockerfile b/.github/docker/ubuntu-22.04.Dockerfile index 8d63d2347..5fcc2f6d5 100644 --- a/.github/docker/ubuntu-22.04.Dockerfile +++ b/.github/docker/ubuntu-22.04.Dockerfile @@ -20,7 +20,8 @@ ENV DEBIAN_FRONTEND noninteractive ARG BASE_DEPS="\ build-essential \ cmake \ - git" + git \ + libtool" # UMF's dependencies ARG UMF_DEPS="\ diff --git a/.github/docker/ubuntu-24.04.Dockerfile b/.github/docker/ubuntu-24.04.Dockerfile index cd45cc52c..88ec7bb07 100644 --- a/.github/docker/ubuntu-24.04.Dockerfile +++ b/.github/docker/ubuntu-24.04.Dockerfile @@ -20,7 +20,8 @@ ENV DEBIAN_FRONTEND noninteractive ARG BASE_DEPS="\ build-essential \ cmake \ - git" + git \ + gnupg" # UMF's dependencies ARG UMF_DEPS="\ diff --git a/.github/workflows/reusable_basic.yml b/.github/workflows/reusable_basic.yml index 7bb1a1398..5cffcd455 100644 --- a/.github/workflows/reusable_basic.yml +++ b/.github/workflows/reusable_basic.yml @@ -18,8 +18,7 @@ jobs: name: Ubuntu runs-on: ubuntu-latest container: - # image: ghcr.io/bb-ur/umf-ubuntu-${{ matrix.ubuntu_ver }}:latest - image: ghcr.io/rbanka1/umf-ubuntu-${{ matrix.ubuntu_ver }}:latest # test + image: ghcr.io/bb-ur/umf-ubuntu-${{ matrix.ubuntu_ver }}:latest options: --user root --privileged volumes: - ${{ github.workspace }}:${{ github.workspace }} diff --git a/.github/workflows/reusable_dockers_build.yml b/.github/workflows/reusable_dockers_build.yml index 53bb727fe..d4dfadae7 100644 --- a/.github/workflows/reusable_dockers_build.yml +++ b/.github/workflows/reusable_dockers_build.yml @@ -18,7 +18,6 @@ jobs: outputs: status: ${{ job.status }} env: - GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }} # test IMG: ghcr.io/bb-ur/umf-ubuntu-${{ matrix.ubuntu-version }}:latest steps: @@ -31,10 +30,8 @@ jobs: uses: docker/login-action@30f019fb76bb54d03ec1e716054622be511a13b2 # v3.2.0 with: registry: ghcr.io - username: ${{ github.actor }} # test - password: ${{ env.GHCR_TOKEN }} # test - # username: bb-ur - # password: ${{ secrets.BB_GHCR_TOKEN }} + username: bb-ur + password: ${{ secrets.BB_GHCR_TOKEN }} - name: Build and push ubuntu-${{ matrix.ubuntu-version }} Docker image run: | diff --git a/.github/workflows/reusable_fast.yml b/.github/workflows/reusable_fast.yml index 7f8da2e2a..b3ff3a4d0 100644 --- a/.github/workflows/reusable_fast.yml +++ b/.github/workflows/reusable_fast.yml @@ -16,8 +16,7 @@ jobs: name: Linux runs-on: ${{ github.repository_owner == 'oneapi-src' && 'intel-ubuntu-22.04' || 'ubuntu-22.04' }} container: - # image: ghcr.io/bb-ur/umf-ubuntu-${{ matrix.ubuntu_ver }}:latest - image: ghcr.io/rbanka1/umf-ubuntu-${{ matrix.ubuntu_ver }}:latest # test + image: ghcr.io/bb-ur/umf-ubuntu-${{ matrix.ubuntu_ver }}:latest options: --user root --privileged volumes: - ${{ github.workspace }}:${{ github.workspace }} @@ -34,16 +33,13 @@ jobs: build_tests: 'OFF' extra_build_options: '-DUMF_BUILD_BENCHMARKS=ON' simple_cmake: 'OFF' - # simplest CMake on Ubuntu 22.04, 24.04, 20.04 + # simplest CMake on Ubuntu 22.04, 24.04 - ubuntu_ver: 22.04 build_tests: 'ON' simple_cmake: 'ON' - ubuntu_ver: 24.04 build_tests: 'ON' simple_cmake: 'ON' - - ubuntu_ver: 20.04 - build_tests: 'ON' - simple_cmake: 'ON' steps: - name: Checkout repository From ef8733e6009f6fe6dec0311bf91c22005b74b053 Mon Sep 17 00:00:00 2001 From: rbanka1 Date: Thu, 8 May 2025 11:22:38 +0200 Subject: [PATCH 03/14] test --- .github/docker/ubuntu-20.04.Dockerfile | 6 ++++-- .github/docker/ubuntu-22.04.Dockerfile | 2 +- .github/docker/ubuntu-24.04.Dockerfile | 2 +- .github/workflows/reusable_basic.yml | 2 +- .github/workflows/reusable_dockers_build.yml | 12 +++++++++--- 5 files changed, 16 insertions(+), 8 deletions(-) diff --git a/.github/docker/ubuntu-20.04.Dockerfile b/.github/docker/ubuntu-20.04.Dockerfile index ceb4c3f13..044e334e3 100644 --- a/.github/docker/ubuntu-20.04.Dockerfile +++ b/.github/docker/ubuntu-20.04.Dockerfile @@ -28,15 +28,17 @@ ARG HWLOC_DEPS="\ libtool" # Copy hwloc +# libhwloc-dev is required - installed via script because hwloc version is to old on this OS COPY .github/scripts/install_hwloc.sh /opt/umf/install_hwloc.sh # UMF's dependencies -# libhwloc-dev is required - installed via script because hwloc version is to old on this OS +ARG UMF_DEPS="\ + libtbb-dev" # Dependencies for tests (optional) ARG TEST_DEPS="\ libnuma-dev \ - libtbb-dev\ + libtbb-dev \ valgrind" # Miscellaneous for our builds/CI (optional) diff --git a/.github/docker/ubuntu-22.04.Dockerfile b/.github/docker/ubuntu-22.04.Dockerfile index 5fcc2f6d5..ce3c7dd9b 100644 --- a/.github/docker/ubuntu-22.04.Dockerfile +++ b/.github/docker/ubuntu-22.04.Dockerfile @@ -25,12 +25,12 @@ ARG BASE_DEPS="\ # UMF's dependencies ARG UMF_DEPS="\ + libtbb-dev \ libhwloc-dev" # Dependencies for tests (optional) ARG TEST_DEPS="\ libnuma-dev \ - libtbb-dev\ valgrind" # Miscellaneous for our builds/CI (optional) diff --git a/.github/docker/ubuntu-24.04.Dockerfile b/.github/docker/ubuntu-24.04.Dockerfile index 88ec7bb07..7f415bb12 100644 --- a/.github/docker/ubuntu-24.04.Dockerfile +++ b/.github/docker/ubuntu-24.04.Dockerfile @@ -25,12 +25,12 @@ ARG BASE_DEPS="\ # UMF's dependencies ARG UMF_DEPS="\ + libtbb-dev \ libhwloc-dev" # Dependencies for tests (optional) ARG TEST_DEPS="\ libnuma-dev \ - libtbb-dev\ valgrind" # Miscellaneous for our builds/CI (optional) diff --git a/.github/workflows/reusable_basic.yml b/.github/workflows/reusable_basic.yml index 5cffcd455..6bb051bac 100644 --- a/.github/workflows/reusable_basic.yml +++ b/.github/workflows/reusable_basic.yml @@ -9,7 +9,7 @@ permissions: env: BUILD_DIR : "${{github.workspace}}/build" - INSTL_DIR : "${{github.workspace}}/../install-dir" + INSTL_DIR : "${{github.workspace}}/install-dir" COVERAGE_DIR : "${{github.workspace}}/coverage" COVERAGE_NAME : "exports-coverage-basic" diff --git a/.github/workflows/reusable_dockers_build.yml b/.github/workflows/reusable_dockers_build.yml index d4dfadae7..d699ede6f 100644 --- a/.github/workflows/reusable_dockers_build.yml +++ b/.github/workflows/reusable_dockers_build.yml @@ -11,6 +11,7 @@ permissions: jobs: build-dockers: + # if: github.repository == 'oneapi-src/unified-memory-framework' runs-on: ubuntu-latest strategy: matrix: @@ -26,14 +27,19 @@ jobs: with: fetch-depth: 0 + - name: Build ubuntu-${{ matrix.ubuntu-version }} Docker image + run: | + docker build -f .github/docker/ubuntu-${{ matrix.ubuntu-version }}.Dockerfile -t ${{ env.IMG }} . + - name: Login to GitHub Container Registry + if: github.event_name != 'pull_request' uses: docker/login-action@30f019fb76bb54d03ec1e716054622be511a13b2 # v3.2.0 - with: + with: registry: ghcr.io username: bb-ur password: ${{ secrets.BB_GHCR_TOKEN }} - - name: Build and push ubuntu-${{ matrix.ubuntu-version }} Docker image + - name: Push ubuntu-${{ matrix.ubuntu-version }} Docker image + if: github.event_name != 'pull_request' run: | - docker build -f .github/docker/ubuntu-${{ matrix.ubuntu-version }}.Dockerfile -t ${{ env.IMG }} . docker push ${{ env.IMG }} From 5bc7506f4f880e8f61d11d718adf5e21831b44ee Mon Sep 17 00:00:00 2001 From: rbanka1 Date: Tue, 13 May 2025 09:51:01 +0200 Subject: [PATCH 04/14] separated detect changes - test --- .github/docker/ubuntu-24.04.Dockerfile | 1 + .github/workflows/detect_changes.yml | 43 ++++++++++ .github/workflows/pr_push.yml | 83 ++++++++++---------- .github/workflows/reusable_basic.yml | 3 +- .github/workflows/reusable_dockers_build.yml | 10 ++- .github/workflows/reusable_fast.yml | 3 +- 6 files changed, 97 insertions(+), 46 deletions(-) create mode 100644 .github/workflows/detect_changes.yml diff --git a/.github/docker/ubuntu-24.04.Dockerfile b/.github/docker/ubuntu-24.04.Dockerfile index 7f415bb12..4d70351f4 100644 --- a/.github/docker/ubuntu-24.04.Dockerfile +++ b/.github/docker/ubuntu-24.04.Dockerfile @@ -64,3 +64,4 @@ ENV USER test_user ENV USERPASS pass RUN useradd -m "${USER}" -g sudo -p "$(mkpasswd ${USERPASS})" USER test_user +# \ No newline at end of file diff --git a/.github/workflows/detect_changes.yml b/.github/workflows/detect_changes.yml new file mode 100644 index 000000000..83f7458f9 --- /dev/null +++ b/.github/workflows/detect_changes.yml @@ -0,0 +1,43 @@ +name: DetectChanges + +on: + pull_request: + paths: + - '.github/docker/*.Dockerfile' + push: + paths: + - '.github/docker/*.Dockerfile' + +permissions: + contents: read + packages: read + +jobs: + DetectChanges: + runs-on: ubuntu-latest + outputs: + changed_files: ${{ steps.changed-files.outputs.all_changed_files }} + steps: + - name: Checkout code + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v45.0.3 + + - name: List all changed files + env: + ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} + run: | + echo "Changed files: ${{ steps.changed-files.outputs.all_changed_files }}" + + name: Run reusable docker + if: ${{ contains(join(needs.DetectChanges.outputs.changed_files, ' '), '.github/docker/') }} + permissions: + contents: read + packages: write + secrets: inherit + uses: ./.github/workflows/reusable_dockers_build.yml + diff --git a/.github/workflows/pr_push.yml b/.github/workflows/pr_push.yml index 174af58c8..2284430a6 100644 --- a/.github/workflows/pr_push.yml +++ b/.github/workflows/pr_push.yml @@ -21,52 +21,52 @@ jobs: uses: ./.github/workflows/reusable_checks.yml DocsBuild: uses: ./.github/workflows/reusable_docs_build.yml - DetectChanges: - runs-on: ubuntu-latest - outputs: - changed_files: ${{ steps.changed-files.outputs.all_changed_files }} - steps: - - name: Checkout code - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 + # DetectChanges: + # runs-on: ubuntu-latest + # outputs: + # changed_files: ${{ steps.changed-files.outputs.all_changed_files }} + # steps: + # - name: Checkout code + # uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + # with: + # fetch-depth: 0 - - name: Get changed files - id: changed-files - uses: tj-actions/changed-files@v45.0.3 + # - name: Get changed files + # id: changed-files + # uses: tj-actions/changed-files@v45.0.3 - - name: List all changed files - env: - ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} - run: | - echo "Changed files: ${{ steps.changed-files.outputs.all_changed_files }}" - BuildDockers: - if: ${{ contains(join(needs.DetectChanges.outputs.changed_files, ' '), '.github/docker/') }} - needs: [DetectChanges] - permissions: - contents: read - packages: write - secrets: inherit - uses: ./.github/workflows/reusable_dockers_build.yml + # - name: List all changed files + # env: + # ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} + # run: | + # echo "Changed files: ${{ steps.changed-files.outputs.all_changed_files }}" + # BuildDockers: + # if: ${{ contains(join(needs.DetectChanges.outputs.changed_files, ' '), '.github/docker/') }} + # needs: [DetectChanges] + # permissions: + # contents: read + # packages: write + # secrets: inherit + # uses: ./.github/workflows/reusable_dockers_build.yml FastBuild: - if: always() && (needs.BuildDockers.result == 'skipped' || needs.BuildDockers.result == 'success') - needs: [CodeChecks, DocsBuild, BuildDockers] + # if: always() && (needs.BuildDockers.result == 'skipped' || needs.BuildDockers.result == 'success') + needs: [CodeChecks, DocsBuild] uses: ./.github/workflows/reusable_fast.yml Build: name: Basic builds - if: always() && (needs.FastBuild.result == 'success') + # if: always() && (needs.FastBuild.result == 'success') needs: [FastBuild] uses: ./.github/workflows/reusable_basic.yml DevDax: - if: always() && (needs.FastBuild.result == 'success') + # if: always() && (needs.FastBuild.result == 'success') needs: [FastBuild] uses: ./.github/workflows/reusable_dax.yml MultiNuma: - if: always() && (needs.FastBuild.result == 'success') + # if: always() && (needs.FastBuild.result == 'success') needs: [FastBuild] uses: ./.github/workflows/reusable_multi_numa.yml L0: - if: always() && (needs.Build.result == 'success') + # if: always() && (needs.Build.result == 'success') needs: [Build] uses: ./.github/workflows/reusable_gpu.yml with: @@ -74,7 +74,7 @@ jobs: runner: "L0" shared_lib: "['ON']" L0-BMG: - if: always() && (needs.Build.result == 'success') + # if: always() && (needs.Build.result == 'success') needs: [Build] uses: ./.github/workflows/reusable_gpu.yml with: @@ -83,7 +83,7 @@ jobs: shared_lib: "['ON']" os: "['Ubuntu']" CUDA: - if: always() && (needs.Build.result == 'success') + # if: always() && (needs.Build.result == 'success') needs: [Build] uses: ./.github/workflows/reusable_gpu.yml with: @@ -91,21 +91,21 @@ jobs: runner: "CUDA" shared_lib: "['ON']" Sanitizers: - if: always() && (needs.FastBuild.result == 'success') + # if: always() && (needs.FastBuild.result == 'success') needs: [FastBuild] uses: ./.github/workflows/reusable_sanitizers.yml QEMU: - if: always() && (needs.FastBuild.result == 'success') + # if: always() && (needs.FastBuild.result == 'success') needs: [FastBuild] uses: ./.github/workflows/reusable_qemu.yml with: short_run: true ProxyLib: - if: always() && (needs.Build.result == 'success') + # if: always() && (needs.Build.result == 'success') needs: [Build] uses: ./.github/workflows/reusable_proxy_lib.yml Valgrind: - if: always() && (needs.Build.result == 'success') + # if: always() && (needs.Build.result == 'success') needs: [Build] uses: ./.github/workflows/reusable_valgrind.yml Coverage: @@ -118,25 +118,26 @@ jobs: trigger: "${{github.event_name}}" Coverage_partial: # partial coverage (on forks) - if: github.repository != 'oneapi-src/unified-memory-framework' && always() && (needs.Build.result == 'success') + # if: github.repository != 'oneapi-src/unified-memory-framework' && always() && (needs.Build.result == 'success') + if: github.repository != 'oneapi-src/unified-memory-framework' needs: [Build, QEMU, ProxyLib] uses: ./.github/workflows/reusable_coverage.yml CodeQL: - if: always() && (needs.Build.result == 'success') + # if: always() && (needs.Build.result == 'success') needs: [Build] permissions: contents: read security-events: write uses: ./.github/workflows/reusable_codeql.yml Trivy: - if: always() && (needs.Build.result == 'success') + # if: always() && (needs.Build.result == 'success') needs: [Build] permissions: contents: read security-events: write uses: ./.github/workflows/reusable_trivy.yml Compatibility: - if: always() && (needs.Build.result == 'success') + # if: always() && (needs.Build.result == 'success') needs: [Build] uses: ./.github/workflows/reusable_compatibility.yml strategy: diff --git a/.github/workflows/reusable_basic.yml b/.github/workflows/reusable_basic.yml index 6bb051bac..9ae0cad83 100644 --- a/.github/workflows/reusable_basic.yml +++ b/.github/workflows/reusable_basic.yml @@ -18,7 +18,8 @@ jobs: name: Ubuntu runs-on: ubuntu-latest container: - image: ghcr.io/bb-ur/umf-ubuntu-${{ matrix.ubuntu_ver }}:latest + # image: ghcr.io/bb-ur/umf-ubuntu-${{ matrix.ubuntu_ver }}:latest + image: ghcr.io/rbanka1/umf-ubuntu-${{ matrix.ubuntu_ver }}:latest # test options: --user root --privileged volumes: - ${{ github.workspace }}:${{ github.workspace }} diff --git a/.github/workflows/reusable_dockers_build.yml b/.github/workflows/reusable_dockers_build.yml index d699ede6f..3a6e0e29b 100644 --- a/.github/workflows/reusable_dockers_build.yml +++ b/.github/workflows/reusable_dockers_build.yml @@ -19,6 +19,7 @@ jobs: outputs: status: ${{ job.status }} env: + GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }} # test IMG: ghcr.io/bb-ur/umf-ubuntu-${{ matrix.ubuntu-version }}:latest steps: @@ -36,10 +37,13 @@ jobs: uses: docker/login-action@30f019fb76bb54d03ec1e716054622be511a13b2 # v3.2.0 with: registry: ghcr.io - username: bb-ur - password: ${{ secrets.BB_GHCR_TOKEN }} + username: ${{ github.actor }} # test + password: ${{ env.GHCR_TOKEN }} # test + # registry: ghcr.io + # username: bb-ur + # password: ${{ secrets.BB_GHCR_TOKEN }} - name: Push ubuntu-${{ matrix.ubuntu-version }} Docker image - if: github.event_name != 'pull_request' + # if: github.event_name != 'pull_request' run: | docker push ${{ env.IMG }} diff --git a/.github/workflows/reusable_fast.yml b/.github/workflows/reusable_fast.yml index b3ff3a4d0..ee145b015 100644 --- a/.github/workflows/reusable_fast.yml +++ b/.github/workflows/reusable_fast.yml @@ -16,7 +16,8 @@ jobs: name: Linux runs-on: ${{ github.repository_owner == 'oneapi-src' && 'intel-ubuntu-22.04' || 'ubuntu-22.04' }} container: - image: ghcr.io/bb-ur/umf-ubuntu-${{ matrix.ubuntu_ver }}:latest + # image: ghcr.io/bb-ur/umf-ubuntu-${{ matrix.ubuntu_ver }}:latest + image: ghcr.io/rbanka1/umf-ubuntu-${{ matrix.ubuntu_ver }}:latest # test options: --user root --privileged volumes: - ${{ github.workspace }}:${{ github.workspace }} From e14f45ed2dc582d9f254df4a35c8a6c20549c439 Mon Sep 17 00:00:00 2001 From: rbanka1 Date: Tue, 13 May 2025 09:59:13 +0200 Subject: [PATCH 05/14] separated detect changes - test --- .github/workflows/detect_changes.yml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/detect_changes.yml b/.github/workflows/detect_changes.yml index 83f7458f9..6d6a87c54 100644 --- a/.github/workflows/detect_changes.yml +++ b/.github/workflows/detect_changes.yml @@ -31,13 +31,14 @@ jobs: env: ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} run: | - echo "Changed files: ${{ steps.changed-files.outputs.all_changed_files }}" - - name: Run reusable docker - if: ${{ contains(join(needs.DetectChanges.outputs.changed_files, ' '), '.github/docker/') }} - permissions: - contents: read - packages: write - secrets: inherit - uses: ./.github/workflows/reusable_dockers_build.yml + echo "Changed files: $ALL_CHANGED_FILES" + RunReusableDocker: + needs: DetectChanges + runs-on: ubuntu-latest + if: ${{ contains(join(needs.DetectChanges.outputs.changed_files, ' '), '.github/docker/') }} + permissions: + contents: read + packages: write + secrets: inherit + uses: ./.github/workflows/reusable_dockers_build.yml From b1d2ec2bc0d571d99e81fdbc26175351af5c31a1 Mon Sep 17 00:00:00 2001 From: rbanka1 Date: Tue, 13 May 2025 13:57:49 +0200 Subject: [PATCH 06/14] separated detect changes --- .github/docker/ubuntu-20.04.Dockerfile | 8 +--- .github/docker/ubuntu-22.04.Dockerfile | 6 +-- .github/docker/ubuntu-24.04.Dockerfile | 7 ++-- .github/workflows/detect_changes.yml | 8 ++-- .github/workflows/pr_push.yml | 42 -------------------- .github/workflows/reusable_basic.yml | 10 ++++- .github/workflows/reusable_dockers_build.yml | 11 ++--- .github/workflows/reusable_fast.yml | 3 +- 8 files changed, 23 insertions(+), 72 deletions(-) diff --git a/.github/docker/ubuntu-20.04.Dockerfile b/.github/docker/ubuntu-20.04.Dockerfile index 044e334e3..752daaa29 100644 --- a/.github/docker/ubuntu-20.04.Dockerfile +++ b/.github/docker/ubuntu-20.04.Dockerfile @@ -31,10 +31,6 @@ ARG HWLOC_DEPS="\ # libhwloc-dev is required - installed via script because hwloc version is to old on this OS COPY .github/scripts/install_hwloc.sh /opt/umf/install_hwloc.sh -# UMF's dependencies -ARG UMF_DEPS="\ - libtbb-dev" - # Dependencies for tests (optional) ARG TEST_DEPS="\ libnuma-dev \ @@ -46,10 +42,10 @@ ARG MISC_DEPS="\ automake \ clang \ g++-7 \ + lcov \ python3-pip \ sudo \ - whois \ - lcov" + whois" # Update and install required packages RUN apt-get update \ diff --git a/.github/docker/ubuntu-22.04.Dockerfile b/.github/docker/ubuntu-22.04.Dockerfile index ce3c7dd9b..5a6d9e8e0 100644 --- a/.github/docker/ubuntu-22.04.Dockerfile +++ b/.github/docker/ubuntu-22.04.Dockerfile @@ -25,22 +25,22 @@ ARG BASE_DEPS="\ # UMF's dependencies ARG UMF_DEPS="\ - libtbb-dev \ libhwloc-dev" # Dependencies for tests (optional) ARG TEST_DEPS="\ libnuma-dev \ + libtbb-dev \ valgrind" # Miscellaneous for our builds/CI (optional) ARG MISC_DEPS="\ automake \ clang \ + lcov \ python3-pip \ sudo \ - whois \ - lcov" + whois" # Update and install required packages RUN apt-get update \ diff --git a/.github/docker/ubuntu-24.04.Dockerfile b/.github/docker/ubuntu-24.04.Dockerfile index 4d70351f4..98c20247b 100644 --- a/.github/docker/ubuntu-24.04.Dockerfile +++ b/.github/docker/ubuntu-24.04.Dockerfile @@ -25,22 +25,22 @@ ARG BASE_DEPS="\ # UMF's dependencies ARG UMF_DEPS="\ - libtbb-dev \ libhwloc-dev" # Dependencies for tests (optional) ARG TEST_DEPS="\ libnuma-dev \ + libtbb-dev \ valgrind" # Miscellaneous for our builds/CI (optional) ARG MISC_DEPS="\ automake \ clang \ + lcov \ python3-pip \ sudo \ - whois \ - lcov" + whois" # Update and install required packages RUN apt-get update \ @@ -64,4 +64,3 @@ ENV USER test_user ENV USERPASS pass RUN useradd -m "${USER}" -g sudo -p "$(mkpasswd ${USERPASS})" USER test_user -# \ No newline at end of file diff --git a/.github/workflows/detect_changes.yml b/.github/workflows/detect_changes.yml index 6d6a87c54..f8f73cf74 100644 --- a/.github/workflows/detect_changes.yml +++ b/.github/workflows/detect_changes.yml @@ -10,9 +10,8 @@ on: permissions: contents: read - packages: read -jobs: +jobs: DetectChanges: runs-on: ubuntu-latest outputs: @@ -25,7 +24,7 @@ jobs: - name: Get changed files id: changed-files - uses: tj-actions/changed-files@v45.0.3 + uses: tj-actions/changed-files@ed68ef82c095e0d48ec87eccea555d944a631a4c #v46.0.5 - name: List all changed files env: @@ -35,10 +34,9 @@ jobs: RunReusableDocker: needs: DetectChanges - runs-on: ubuntu-latest if: ${{ contains(join(needs.DetectChanges.outputs.changed_files, ' '), '.github/docker/') }} + uses: ./.github/workflows/reusable_dockers_build.yml permissions: contents: read packages: write secrets: inherit - uses: ./.github/workflows/reusable_dockers_build.yml diff --git a/.github/workflows/pr_push.yml b/.github/workflows/pr_push.yml index 2284430a6..99fc19120 100644 --- a/.github/workflows/pr_push.yml +++ b/.github/workflows/pr_push.yml @@ -21,52 +21,20 @@ jobs: uses: ./.github/workflows/reusable_checks.yml DocsBuild: uses: ./.github/workflows/reusable_docs_build.yml - # DetectChanges: - # runs-on: ubuntu-latest - # outputs: - # changed_files: ${{ steps.changed-files.outputs.all_changed_files }} - # steps: - # - name: Checkout code - # uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - # with: - # fetch-depth: 0 - - # - name: Get changed files - # id: changed-files - # uses: tj-actions/changed-files@v45.0.3 - - # - name: List all changed files - # env: - # ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} - # run: | - # echo "Changed files: ${{ steps.changed-files.outputs.all_changed_files }}" - # BuildDockers: - # if: ${{ contains(join(needs.DetectChanges.outputs.changed_files, ' '), '.github/docker/') }} - # needs: [DetectChanges] - # permissions: - # contents: read - # packages: write - # secrets: inherit - # uses: ./.github/workflows/reusable_dockers_build.yml FastBuild: - # if: always() && (needs.BuildDockers.result == 'skipped' || needs.BuildDockers.result == 'success') needs: [CodeChecks, DocsBuild] uses: ./.github/workflows/reusable_fast.yml Build: name: Basic builds - # if: always() && (needs.FastBuild.result == 'success') needs: [FastBuild] uses: ./.github/workflows/reusable_basic.yml DevDax: - # if: always() && (needs.FastBuild.result == 'success') needs: [FastBuild] uses: ./.github/workflows/reusable_dax.yml MultiNuma: - # if: always() && (needs.FastBuild.result == 'success') needs: [FastBuild] uses: ./.github/workflows/reusable_multi_numa.yml L0: - # if: always() && (needs.Build.result == 'success') needs: [Build] uses: ./.github/workflows/reusable_gpu.yml with: @@ -74,7 +42,6 @@ jobs: runner: "L0" shared_lib: "['ON']" L0-BMG: - # if: always() && (needs.Build.result == 'success') needs: [Build] uses: ./.github/workflows/reusable_gpu.yml with: @@ -83,7 +50,6 @@ jobs: shared_lib: "['ON']" os: "['Ubuntu']" CUDA: - # if: always() && (needs.Build.result == 'success') needs: [Build] uses: ./.github/workflows/reusable_gpu.yml with: @@ -91,21 +57,17 @@ jobs: runner: "CUDA" shared_lib: "['ON']" Sanitizers: - # if: always() && (needs.FastBuild.result == 'success') needs: [FastBuild] uses: ./.github/workflows/reusable_sanitizers.yml QEMU: - # if: always() && (needs.FastBuild.result == 'success') needs: [FastBuild] uses: ./.github/workflows/reusable_qemu.yml with: short_run: true ProxyLib: - # if: always() && (needs.Build.result == 'success') needs: [Build] uses: ./.github/workflows/reusable_proxy_lib.yml Valgrind: - # if: always() && (needs.Build.result == 'success') needs: [Build] uses: ./.github/workflows/reusable_valgrind.yml Coverage: @@ -118,26 +80,22 @@ jobs: trigger: "${{github.event_name}}" Coverage_partial: # partial coverage (on forks) - # if: github.repository != 'oneapi-src/unified-memory-framework' && always() && (needs.Build.result == 'success') if: github.repository != 'oneapi-src/unified-memory-framework' needs: [Build, QEMU, ProxyLib] uses: ./.github/workflows/reusable_coverage.yml CodeQL: - # if: always() && (needs.Build.result == 'success') needs: [Build] permissions: contents: read security-events: write uses: ./.github/workflows/reusable_codeql.yml Trivy: - # if: always() && (needs.Build.result == 'success') needs: [Build] permissions: contents: read security-events: write uses: ./.github/workflows/reusable_trivy.yml Compatibility: - # if: always() && (needs.Build.result == 'success') needs: [Build] uses: ./.github/workflows/reusable_compatibility.yml strategy: diff --git a/.github/workflows/reusable_basic.yml b/.github/workflows/reusable_basic.yml index c14b55c16..eefa8cfa4 100644 --- a/.github/workflows/reusable_basic.yml +++ b/.github/workflows/reusable_basic.yml @@ -18,8 +18,7 @@ jobs: name: Ubuntu runs-on: ubuntu-latest container: - # image: ghcr.io/bb-ur/umf-ubuntu-${{ matrix.ubuntu_ver }}:latest - image: ghcr.io/rbanka1/umf-ubuntu-${{ matrix.ubuntu_ver }}:latest # test + image: ghcr.io/bb-ur/umf-ubuntu-${{ matrix.ubuntu_ver }}:latest options: --user root --privileged volumes: - ${{ github.workspace }}:${{ github.workspace }} @@ -45,6 +44,8 @@ jobs: install_tbb: 'ON' disable_hwloc: 'OFF' link_hwloc_statically: 'OFF' + # check minimum supported cmake version + cmake_ver: '3.14.0' - ubuntu_ver: 22.04 build_type: Release compiler: {c: gcc, cxx: g++} @@ -54,6 +55,7 @@ jobs: install_tbb: 'ON' disable_hwloc: 'OFF' link_hwloc_statically: 'OFF' + cmake_ver: '3.28.0' - ubuntu_ver: 24.04 build_type: Debug compiler: {c: gcc, cxx: g++} @@ -97,6 +99,7 @@ jobs: disable_hwloc: 'OFF' link_hwloc_statically: 'OFF' llvm_linker: '-DCMAKE_EXE_LINKER_FLAGS="-fuse-ld=lld" -DCMAKE_MODULE_LINKER_FLAGS="-fuse-ld=lld" -DCMAKE_SHARED_LINKER_FLAGS="-fuse-ld=lld"' + cmake_ver: 'default' # test without installing TBB - ubuntu_ver: 22.04 build_type: Release @@ -107,6 +110,7 @@ jobs: install_tbb: 'OFF' disable_hwloc: 'OFF' link_hwloc_statically: 'OFF' + cmake_ver: 'default' - ubuntu_ver: 22.04 build_type: Debug compiler: {c: gcc, cxx: g++} @@ -116,6 +120,7 @@ jobs: install_tbb: 'ON' disable_hwloc: 'ON' link_hwloc_statically: 'OFF' + cmake_ver: 'default' - ubuntu_ver: 22.04 build_type: Release compiler: {c: gcc, cxx: g++} @@ -125,6 +130,7 @@ jobs: install_tbb: 'ON' disable_hwloc: 'OFF' link_hwloc_statically: 'ON' + cmake_ver: 'default' steps: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 diff --git a/.github/workflows/reusable_dockers_build.yml b/.github/workflows/reusable_dockers_build.yml index 3a6e0e29b..d8322564c 100644 --- a/.github/workflows/reusable_dockers_build.yml +++ b/.github/workflows/reusable_dockers_build.yml @@ -11,7 +11,6 @@ permissions: jobs: build-dockers: - # if: github.repository == 'oneapi-src/unified-memory-framework' runs-on: ubuntu-latest strategy: matrix: @@ -19,7 +18,6 @@ jobs: outputs: status: ${{ job.status }} env: - GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }} # test IMG: ghcr.io/bb-ur/umf-ubuntu-${{ matrix.ubuntu-version }}:latest steps: @@ -37,13 +35,10 @@ jobs: uses: docker/login-action@30f019fb76bb54d03ec1e716054622be511a13b2 # v3.2.0 with: registry: ghcr.io - username: ${{ github.actor }} # test - password: ${{ env.GHCR_TOKEN }} # test - # registry: ghcr.io - # username: bb-ur - # password: ${{ secrets.BB_GHCR_TOKEN }} + username: bb-ur + password: ${{ secrets.BB_GHCR_TOKEN }} - name: Push ubuntu-${{ matrix.ubuntu-version }} Docker image - # if: github.event_name != 'pull_request' + if: github.event_name != 'pull_request' run: | docker push ${{ env.IMG }} diff --git a/.github/workflows/reusable_fast.yml b/.github/workflows/reusable_fast.yml index ee145b015..b3ff3a4d0 100644 --- a/.github/workflows/reusable_fast.yml +++ b/.github/workflows/reusable_fast.yml @@ -16,8 +16,7 @@ jobs: name: Linux runs-on: ${{ github.repository_owner == 'oneapi-src' && 'intel-ubuntu-22.04' || 'ubuntu-22.04' }} container: - # image: ghcr.io/bb-ur/umf-ubuntu-${{ matrix.ubuntu_ver }}:latest - image: ghcr.io/rbanka1/umf-ubuntu-${{ matrix.ubuntu_ver }}:latest # test + image: ghcr.io/bb-ur/umf-ubuntu-${{ matrix.ubuntu_ver }}:latest options: --user root --privileged volumes: - ${{ github.workspace }}:${{ github.workspace }} From e6900b67d5c382a62d66600b75a4254b8bf3b1b5 Mon Sep 17 00:00:00 2001 From: rbanka1 Date: Thu, 15 May 2025 22:04:49 +0200 Subject: [PATCH 07/14] . --- .github/docker/ubuntu-20.04.Dockerfile | 10 ++--- .github/docker/ubuntu-22.04.Dockerfile | 5 +-- .github/docker/ubuntu-24.04.Dockerfile | 2 +- .github/workflows/pr_push.yml | 4 +- .github/workflows/reusable_basic.yml | 43 ++++++++++++++------ .github/workflows/reusable_dockers_build.yml | 13 +++--- .github/workflows/reusable_fast.yml | 7 ++-- 7 files changed, 49 insertions(+), 35 deletions(-) diff --git a/.github/docker/ubuntu-20.04.Dockerfile b/.github/docker/ubuntu-20.04.Dockerfile index 752daaa29..eb7733b6a 100644 --- a/.github/docker/ubuntu-20.04.Dockerfile +++ b/.github/docker/ubuntu-20.04.Dockerfile @@ -24,17 +24,12 @@ ARG BASE_DEPS="\ # Hwloc installation dependencies ARG HWLOC_DEPS="\ - dos2unix \ libtool" -# Copy hwloc -# libhwloc-dev is required - installed via script because hwloc version is to old on this OS -COPY .github/scripts/install_hwloc.sh /opt/umf/install_hwloc.sh - # Dependencies for tests (optional) ARG TEST_DEPS="\ libnuma-dev \ - libtbb-dev \ + # libtbb-dev \ valgrind" # Miscellaneous for our builds/CI (optional) @@ -47,6 +42,9 @@ ARG MISC_DEPS="\ sudo \ whois" +# libhwloc-dev is required - installed via script because hwloc version is too old on this OS +COPY .github/scripts/install_hwloc.sh /opt/umf/install_hwloc.sh + # Update and install required packages RUN apt-get update \ && apt-get install -y --no-install-recommends \ diff --git a/.github/docker/ubuntu-22.04.Dockerfile b/.github/docker/ubuntu-22.04.Dockerfile index 5a6d9e8e0..c566f35de 100644 --- a/.github/docker/ubuntu-22.04.Dockerfile +++ b/.github/docker/ubuntu-22.04.Dockerfile @@ -20,8 +20,7 @@ ENV DEBIAN_FRONTEND noninteractive ARG BASE_DEPS="\ build-essential \ cmake \ - git \ - libtool" + git " # UMF's dependencies ARG UMF_DEPS="\ @@ -30,7 +29,7 @@ ARG UMF_DEPS="\ # Dependencies for tests (optional) ARG TEST_DEPS="\ libnuma-dev \ - libtbb-dev \ + # libtbb-dev \ valgrind" # Miscellaneous for our builds/CI (optional) diff --git a/.github/docker/ubuntu-24.04.Dockerfile b/.github/docker/ubuntu-24.04.Dockerfile index 98c20247b..5f7664f59 100644 --- a/.github/docker/ubuntu-24.04.Dockerfile +++ b/.github/docker/ubuntu-24.04.Dockerfile @@ -30,7 +30,7 @@ ARG UMF_DEPS="\ # Dependencies for tests (optional) ARG TEST_DEPS="\ libnuma-dev \ - libtbb-dev \ + # libtbb-dev \ valgrind" # Miscellaneous for our builds/CI (optional) diff --git a/.github/workflows/pr_push.yml b/.github/workflows/pr_push.yml index 99fc19120..afac430aa 100644 --- a/.github/workflows/pr_push.yml +++ b/.github/workflows/pr_push.yml @@ -19,10 +19,8 @@ permissions: jobs: CodeChecks: uses: ./.github/workflows/reusable_checks.yml - DocsBuild: - uses: ./.github/workflows/reusable_docs_build.yml FastBuild: - needs: [CodeChecks, DocsBuild] + needs: [CodeChecks] uses: ./.github/workflows/reusable_fast.yml Build: name: Basic builds diff --git a/.github/workflows/reusable_basic.yml b/.github/workflows/reusable_basic.yml index eefa8cfa4..215227ad8 100644 --- a/.github/workflows/reusable_basic.yml +++ b/.github/workflows/reusable_basic.yml @@ -18,8 +18,9 @@ jobs: name: Ubuntu runs-on: ubuntu-latest container: - image: ghcr.io/bb-ur/umf-ubuntu-${{ matrix.ubuntu_ver }}:latest - options: --user root --privileged + # image: ghcr.io/bb-ur/umf-ubuntu-${{ matrix.ubuntu_ver }}:latest + image: ghcr.io/rbanka1/umf-ubuntu-${{ matrix.ubuntu_ver }}:latest # test + # options: --user root --privileged volumes: - ${{ github.workspace }}:${{ github.workspace }} strategy: @@ -137,15 +138,33 @@ jobs: with: fetch-depth: 0 + - name: Install cmake (non-default version) + if: matrix.cmake_ver != 'default' + run: | + echo ${USERPASS} | sudo -Sk apt-get remove --purge -y cmake + wget https://github.com/Kitware/CMake/releases/download/v${{matrix.cmake_ver}}/cmake-${{matrix.cmake_ver}}-Linux-x86_64.sh + chmod +x cmake-${{matrix.cmake_ver}}-Linux-x86_64.sh + echo ${USERPASS} | sudo -Sk ./cmake-${{matrix.cmake_ver}}-Linux-x86_64.sh --skip-license --prefix=/usr/local + + - name: Install TBB apt package + if: matrix.install_tbb == 'ON' + run: | + echo ${USERPASS} | sudo -Sk apt-get install -y libtbb-dev + - name: Install oneAPI basekit if: matrix.compiler.cxx == 'icpx' run: | - sudo apt-get update - sudo apt-get install -y gpg-agent wget - wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | sudo tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null - echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list - sudo apt-get update - sudo apt-get install -y intel-oneapi-ippcp-devel intel-oneapi-ipp-devel intel-oneapi-common-oneapi-vars intel-oneapi-compiler-dpcpp-cpp + echo ${USERPASS} | sudo -Sk apt-get update + echo ${USERPASS} | sudo -Sk apt-get install -y gpg-agent wget + wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | echo ${USERPASS} | sudo -Sk tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null + echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | echo ${USERPASS} | sudo -Sk tee /etc/apt/sources.list.d/oneAPI.list + echo ${USERPASS} | sudo -Sk apt-get update + echo ${USERPASS} | sudo -Sk apt-get install -y intel-oneapi-ippcp-devel intel-oneapi-ipp-devel intel-oneapi-common-oneapi-vars intel-oneapi-compiler-dpcpp-cpp + + - name: Get UMF version + run: | + VERSION=$(git describe --tags --abbrev=0 | grep -oP '\d+\.\d+\.\d+') + echo "UMF_VERSION=$VERSION" >> $GITHUB_ENV - name: Configure build run: > @@ -166,6 +185,7 @@ jobs: -DUMF_DISABLE_HWLOC=${{matrix.disable_hwloc}} -DUMF_LINK_HWLOC_STATICALLY=${{matrix.link_hwloc_statically}} ${{ matrix.build_type == 'Debug' && matrix.compiler.c == 'gcc' && '-DUMF_USE_COVERAGE=ON' || '' }} + ${{ matrix.llvm_linker || '' }} - name: Build UMF run: | @@ -178,14 +198,11 @@ jobs: ${{ matrix.compiler.cxx == 'icpx' && '. /opt/intel/oneapi/setvars.sh' || true }} LD_LIBRARY_PATH="${{env.BUILD_DIR}}/lib/:${LD_LIBRARY_PATH}" ctest --output-on-failure - - name: Set OS name - run: echo "OS_VER=ubuntu-${{ matrix.ubuntu_ver }}" >> $GITHUB_ENV - - name: Check coverage if: ${{ matrix.build_type == 'Debug' && matrix.compiler.c == 'gcc' }} working-directory: ${{env.BUILD_DIR}} run: | - export COVERAGE_FILE_NAME=${{env.COVERAGE_NAME}}-${{env.OS_VER}}-shared-${{matrix.shared_library}}-no_hwloc-${{matrix.disable_hwloc}} + export COVERAGE_FILE_NAME=${{env.COVERAGE_NAME}}-ubuntu-${{matrix.ubuntu_ver}}-shared-${{matrix.shared_library}}-no_hwloc-${{matrix.disable_hwloc}} echo "COVERAGE_FILE_NAME: $COVERAGE_FILE_NAME" ../scripts/coverage/coverage_capture.sh $COVERAGE_FILE_NAME mkdir -p ${{env.COVERAGE_DIR}} @@ -194,7 +211,7 @@ jobs: - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 if: ${{ matrix.build_type == 'Debug' && matrix.compiler.c == 'gcc' }} with: - name: ${{env.COVERAGE_NAME}}-${{env.OS_VER}}-shared-${{matrix.shared_library}}-no_hwloc-${{matrix.disable_hwloc}} + name: ${{env.COVERAGE_NAME}}-${{matrix.ubuntu_ver}}-shared-${{matrix.shared_library}}-no_hwloc-${{matrix.disable_hwloc}} path: ${{env.COVERAGE_DIR}} - name: Remove the installation directory diff --git a/.github/workflows/reusable_dockers_build.yml b/.github/workflows/reusable_dockers_build.yml index d8322564c..afd86951e 100644 --- a/.github/workflows/reusable_dockers_build.yml +++ b/.github/workflows/reusable_dockers_build.yml @@ -15,10 +15,9 @@ jobs: strategy: matrix: ubuntu-version: [20.04, 22.04, 24.04] - outputs: - status: ${{ job.status }} env: - IMG: ghcr.io/bb-ur/umf-ubuntu-${{ matrix.ubuntu-version }}:latest + # IMG: ghcr.io/bb-ur/umf-ubuntu-${{ matrix.ubuntu-version }}:latest + IMG: ghcr.io/${{ github.actor }}/umf-ubuntu-${{ matrix.ubuntu-version }}:latest # test steps: - name: Checkout repository @@ -35,10 +34,12 @@ jobs: uses: docker/login-action@30f019fb76bb54d03ec1e716054622be511a13b2 # v3.2.0 with: registry: ghcr.io - username: bb-ur - password: ${{ secrets.BB_GHCR_TOKEN }} + username: ${{ github.actor }} # test + password: ${{ secrets.GHCR_TOKEN }} # test + # username: bb-ur + # password: ${{ secrets.BB_GHCR_TOKEN }} - name: Push ubuntu-${{ matrix.ubuntu-version }} Docker image - if: github.event_name != 'pull_request' + # if: github.event_name != 'pull_request' run: | docker push ${{ env.IMG }} diff --git a/.github/workflows/reusable_fast.yml b/.github/workflows/reusable_fast.yml index b3ff3a4d0..063f8f946 100644 --- a/.github/workflows/reusable_fast.yml +++ b/.github/workflows/reusable_fast.yml @@ -9,15 +9,16 @@ permissions: env: BUILD_DIR : "${{github.workspace}}/build" - INSTL_DIR : "${{github.workspace}}/../install-dir" + INSTL_DIR : "${{github.workspace}}/install-dir" jobs: ubuntu-build: name: Linux runs-on: ${{ github.repository_owner == 'oneapi-src' && 'intel-ubuntu-22.04' || 'ubuntu-22.04' }} container: - image: ghcr.io/bb-ur/umf-ubuntu-${{ matrix.ubuntu_ver }}:latest - options: --user root --privileged + # image: ghcr.io/bb-ur/umf-ubuntu-${{ matrix.ubuntu_ver }}:latest + image: ghcr.io/rbanka1/umf-ubuntu-${{ matrix.ubuntu_ver }}:latest # test + # options: --user root --privileged volumes: - ${{ github.workspace }}:${{ github.workspace }} strategy: From 8bf1b255ac8705a472dcce877d21282207f79952 Mon Sep 17 00:00:00 2001 From: rbanka1 Date: Thu, 15 May 2025 22:16:23 +0200 Subject: [PATCH 08/14] . --- .github/workflows/reusable_basic.yml | 2 +- .github/workflows/reusable_fast.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/reusable_basic.yml b/.github/workflows/reusable_basic.yml index 215227ad8..50b97bcca 100644 --- a/.github/workflows/reusable_basic.yml +++ b/.github/workflows/reusable_basic.yml @@ -20,7 +20,7 @@ jobs: container: # image: ghcr.io/bb-ur/umf-ubuntu-${{ matrix.ubuntu_ver }}:latest image: ghcr.io/rbanka1/umf-ubuntu-${{ matrix.ubuntu_ver }}:latest # test - # options: --user root --privileged + options: --user root --privileged volumes: - ${{ github.workspace }}:${{ github.workspace }} strategy: diff --git a/.github/workflows/reusable_fast.yml b/.github/workflows/reusable_fast.yml index 063f8f946..80d1a6cdf 100644 --- a/.github/workflows/reusable_fast.yml +++ b/.github/workflows/reusable_fast.yml @@ -18,7 +18,7 @@ jobs: container: # image: ghcr.io/bb-ur/umf-ubuntu-${{ matrix.ubuntu_ver }}:latest image: ghcr.io/rbanka1/umf-ubuntu-${{ matrix.ubuntu_ver }}:latest # test - # options: --user root --privileged + options: --user root --privileged volumes: - ${{ github.workspace }}:${{ github.workspace }} strategy: From 19c9daa6762447b59bd15707f659c2bfa2584163 Mon Sep 17 00:00:00 2001 From: rbanka1 Date: Thu, 15 May 2025 22:47:06 +0200 Subject: [PATCH 09/14] . --- .github/workflows/reusable_basic.yml | 2 +- .github/workflows/reusable_fast.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/reusable_basic.yml b/.github/workflows/reusable_basic.yml index 50b97bcca..cf84625d7 100644 --- a/.github/workflows/reusable_basic.yml +++ b/.github/workflows/reusable_basic.yml @@ -20,7 +20,7 @@ jobs: container: # image: ghcr.io/bb-ur/umf-ubuntu-${{ matrix.ubuntu_ver }}:latest image: ghcr.io/rbanka1/umf-ubuntu-${{ matrix.ubuntu_ver }}:latest # test - options: --user root --privileged + options: --user root volumes: - ${{ github.workspace }}:${{ github.workspace }} strategy: diff --git a/.github/workflows/reusable_fast.yml b/.github/workflows/reusable_fast.yml index 80d1a6cdf..0830c3813 100644 --- a/.github/workflows/reusable_fast.yml +++ b/.github/workflows/reusable_fast.yml @@ -18,7 +18,7 @@ jobs: container: # image: ghcr.io/bb-ur/umf-ubuntu-${{ matrix.ubuntu_ver }}:latest image: ghcr.io/rbanka1/umf-ubuntu-${{ matrix.ubuntu_ver }}:latest # test - options: --user root --privileged + options: --user root volumes: - ${{ github.workspace }}:${{ github.workspace }} strategy: From 8c7e08c0ec344cdc8c7fd5b6062a62b69f559e59 Mon Sep 17 00:00:00 2001 From: rbanka1 Date: Thu, 15 May 2025 22:51:57 +0200 Subject: [PATCH 10/14] . --- .github/workflows/reusable_basic.yml | 20 ++++++++++---------- .github/workflows/reusable_fast.yml | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/reusable_basic.yml b/.github/workflows/reusable_basic.yml index cf84625d7..1a5c163cf 100644 --- a/.github/workflows/reusable_basic.yml +++ b/.github/workflows/reusable_basic.yml @@ -20,7 +20,7 @@ jobs: container: # image: ghcr.io/bb-ur/umf-ubuntu-${{ matrix.ubuntu_ver }}:latest image: ghcr.io/rbanka1/umf-ubuntu-${{ matrix.ubuntu_ver }}:latest # test - options: --user root + options: --user root --privileged volumes: - ${{ github.workspace }}:${{ github.workspace }} strategy: @@ -141,25 +141,25 @@ jobs: - name: Install cmake (non-default version) if: matrix.cmake_ver != 'default' run: | - echo ${USERPASS} | sudo -Sk apt-get remove --purge -y cmake + sudo apt-get remove --purge -y cmake wget https://github.com/Kitware/CMake/releases/download/v${{matrix.cmake_ver}}/cmake-${{matrix.cmake_ver}}-Linux-x86_64.sh chmod +x cmake-${{matrix.cmake_ver}}-Linux-x86_64.sh - echo ${USERPASS} | sudo -Sk ./cmake-${{matrix.cmake_ver}}-Linux-x86_64.sh --skip-license --prefix=/usr/local + sudo ./cmake-${{matrix.cmake_ver}}-Linux-x86_64.sh --skip-license --prefix=/usr/local - name: Install TBB apt package if: matrix.install_tbb == 'ON' run: | - echo ${USERPASS} | sudo -Sk apt-get install -y libtbb-dev + sudo apt-get install -y libtbb-dev - name: Install oneAPI basekit if: matrix.compiler.cxx == 'icpx' run: | - echo ${USERPASS} | sudo -Sk apt-get update - echo ${USERPASS} | sudo -Sk apt-get install -y gpg-agent wget - wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | echo ${USERPASS} | sudo -Sk tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null - echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | echo ${USERPASS} | sudo -Sk tee /etc/apt/sources.list.d/oneAPI.list - echo ${USERPASS} | sudo -Sk apt-get update - echo ${USERPASS} | sudo -Sk apt-get install -y intel-oneapi-ippcp-devel intel-oneapi-ipp-devel intel-oneapi-common-oneapi-vars intel-oneapi-compiler-dpcpp-cpp + sudo apt-get update + sudo apt-get install -y gpg-agent wget + wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | sudo tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null + echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list + sudo apt-get update + sudo apt-get install -y intel-oneapi-ippcp-devel intel-oneapi-ipp-devel intel-oneapi-common-oneapi-vars intel-oneapi-compiler-dpcpp-cpp - name: Get UMF version run: | diff --git a/.github/workflows/reusable_fast.yml b/.github/workflows/reusable_fast.yml index 0830c3813..80d1a6cdf 100644 --- a/.github/workflows/reusable_fast.yml +++ b/.github/workflows/reusable_fast.yml @@ -18,7 +18,7 @@ jobs: container: # image: ghcr.io/bb-ur/umf-ubuntu-${{ matrix.ubuntu_ver }}:latest image: ghcr.io/rbanka1/umf-ubuntu-${{ matrix.ubuntu_ver }}:latest # test - options: --user root + options: --user root --privileged volumes: - ${{ github.workspace }}:${{ github.workspace }} strategy: From 511b6af2bdd94758aa29c5ee71b52af825e06464 Mon Sep 17 00:00:00 2001 From: rbanka1 Date: Thu, 15 May 2025 23:22:19 +0200 Subject: [PATCH 11/14] . --- .github/workflows/pr_push.yml | 3 ++- .github/workflows/reusable_basic.yml | 15 ++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pr_push.yml b/.github/workflows/pr_push.yml index afac430aa..10257d305 100644 --- a/.github/workflows/pr_push.yml +++ b/.github/workflows/pr_push.yml @@ -24,7 +24,8 @@ jobs: uses: ./.github/workflows/reusable_fast.yml Build: name: Basic builds - needs: [FastBuild] + needs: [CodeChecks] + # needs: [FastBuild] uses: ./.github/workflows/reusable_basic.yml DevDax: needs: [FastBuild] diff --git a/.github/workflows/reusable_basic.yml b/.github/workflows/reusable_basic.yml index 1a5c163cf..05fc5eb52 100644 --- a/.github/workflows/reusable_basic.yml +++ b/.github/workflows/reusable_basic.yml @@ -138,13 +138,13 @@ jobs: with: fetch-depth: 0 - - name: Install cmake (non-default version) - if: matrix.cmake_ver != 'default' - run: | - sudo apt-get remove --purge -y cmake - wget https://github.com/Kitware/CMake/releases/download/v${{matrix.cmake_ver}}/cmake-${{matrix.cmake_ver}}-Linux-x86_64.sh - chmod +x cmake-${{matrix.cmake_ver}}-Linux-x86_64.sh - sudo ./cmake-${{matrix.cmake_ver}}-Linux-x86_64.sh --skip-license --prefix=/usr/local + # - name: Install cmake (non-default version) + # if: matrix.cmake_ver != 'default' + # run: | + # sudo apt-get remove --purge -y cmake + # wget https://github.com/Kitware/CMake/releases/download/v${{matrix.cmake_ver}}/cmake-${{matrix.cmake_ver}}-Linux-x86_64.sh + # chmod +x cmake-${{matrix.cmake_ver}}-Linux-x86_64.sh + # sudo ./cmake-${{matrix.cmake_ver}}-Linux-x86_64.sh --skip-license --prefix=/usr/local - name: Install TBB apt package if: matrix.install_tbb == 'ON' @@ -163,6 +163,7 @@ jobs: - name: Get UMF version run: | + git config --global --add safe.directory $GITHUB_WORKSPACE VERSION=$(git describe --tags --abbrev=0 | grep -oP '\d+\.\d+\.\d+') echo "UMF_VERSION=$VERSION" >> $GITHUB_ENV From 253b428afae4fd1008fa7d0d00a8adb2e28b107b Mon Sep 17 00:00:00 2001 From: rbanka1 Date: Thu, 15 May 2025 23:28:28 +0200 Subject: [PATCH 12/14] . --- .github/docker/ubuntu-20.04.Dockerfile | 2 +- .github/docker/ubuntu-22.04.Dockerfile | 2 +- .github/docker/ubuntu-24.04.Dockerfile | 2 +- .github/workflows/reusable_basic.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/docker/ubuntu-20.04.Dockerfile b/.github/docker/ubuntu-20.04.Dockerfile index eb7733b6a..4c7dbae9e 100644 --- a/.github/docker/ubuntu-20.04.Dockerfile +++ b/.github/docker/ubuntu-20.04.Dockerfile @@ -29,7 +29,7 @@ ARG HWLOC_DEPS="\ # Dependencies for tests (optional) ARG TEST_DEPS="\ libnuma-dev \ - # libtbb-dev \ + libtbb-dev \ valgrind" # Miscellaneous for our builds/CI (optional) diff --git a/.github/docker/ubuntu-22.04.Dockerfile b/.github/docker/ubuntu-22.04.Dockerfile index c566f35de..1febafbf8 100644 --- a/.github/docker/ubuntu-22.04.Dockerfile +++ b/.github/docker/ubuntu-22.04.Dockerfile @@ -29,7 +29,7 @@ ARG UMF_DEPS="\ # Dependencies for tests (optional) ARG TEST_DEPS="\ libnuma-dev \ - # libtbb-dev \ + libtbb-dev \ valgrind" # Miscellaneous for our builds/CI (optional) diff --git a/.github/docker/ubuntu-24.04.Dockerfile b/.github/docker/ubuntu-24.04.Dockerfile index 5f7664f59..98c20247b 100644 --- a/.github/docker/ubuntu-24.04.Dockerfile +++ b/.github/docker/ubuntu-24.04.Dockerfile @@ -30,7 +30,7 @@ ARG UMF_DEPS="\ # Dependencies for tests (optional) ARG TEST_DEPS="\ libnuma-dev \ - # libtbb-dev \ + libtbb-dev \ valgrind" # Miscellaneous for our builds/CI (optional) diff --git a/.github/workflows/reusable_basic.yml b/.github/workflows/reusable_basic.yml index 05fc5eb52..5813976ef 100644 --- a/.github/workflows/reusable_basic.yml +++ b/.github/workflows/reusable_basic.yml @@ -147,9 +147,9 @@ jobs: # sudo ./cmake-${{matrix.cmake_ver}}-Linux-x86_64.sh --skip-license --prefix=/usr/local - name: Install TBB apt package - if: matrix.install_tbb == 'ON' + if: matrix.install_tbb == 'OFF' run: | - sudo apt-get install -y libtbb-dev + sudo apt-get remove -y libtbb-dev - name: Install oneAPI basekit if: matrix.compiler.cxx == 'icpx' From 156d4ed9b918ac82ee51bff2cb49133b75fb6dc5 Mon Sep 17 00:00:00 2001 From: rbanka1 Date: Thu, 15 May 2025 23:43:53 +0200 Subject: [PATCH 13/14] . --- .github/workflows/reusable_basic.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/reusable_basic.yml b/.github/workflows/reusable_basic.yml index 5813976ef..522fe66de 100644 --- a/.github/workflows/reusable_basic.yml +++ b/.github/workflows/reusable_basic.yml @@ -138,18 +138,18 @@ jobs: with: fetch-depth: 0 - # - name: Install cmake (non-default version) - # if: matrix.cmake_ver != 'default' - # run: | - # sudo apt-get remove --purge -y cmake - # wget https://github.com/Kitware/CMake/releases/download/v${{matrix.cmake_ver}}/cmake-${{matrix.cmake_ver}}-Linux-x86_64.sh - # chmod +x cmake-${{matrix.cmake_ver}}-Linux-x86_64.sh - # sudo ./cmake-${{matrix.cmake_ver}}-Linux-x86_64.sh --skip-license --prefix=/usr/local + - name: Install cmake (non-default version) + if: matrix.cmake_ver != 'default' + run: | + sudo apt-get remove --purge -y cmake + wget https://github.com/Kitware/CMake/releases/download/v${{matrix.cmake_ver}}/cmake-${{matrix.cmake_ver}}-Linux-x86_64.sh + chmod +x cmake-${{matrix.cmake_ver}}-Linux-x86_64.sh + sudo ./cmake-${{matrix.cmake_ver}}-Linux-x86_64.sh --skip-license --prefix=/usr/local - name: Install TBB apt package if: matrix.install_tbb == 'OFF' run: | - sudo apt-get remove -y libtbb-dev + sudo apt-get remove --purge -y libtbb-dev - name: Install oneAPI basekit if: matrix.compiler.cxx == 'icpx' From 69cf7439f89f6d45b5e29861663731ea355142d7 Mon Sep 17 00:00:00 2001 From: rbanka1 Date: Thu, 15 May 2025 23:59:31 +0200 Subject: [PATCH 14/14] . --- .github/workflows/reusable_basic.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reusable_basic.yml b/.github/workflows/reusable_basic.yml index 522fe66de..775293fe1 100644 --- a/.github/workflows/reusable_basic.yml +++ b/.github/workflows/reusable_basic.yml @@ -142,7 +142,7 @@ jobs: if: matrix.cmake_ver != 'default' run: | sudo apt-get remove --purge -y cmake - wget https://github.com/Kitware/CMake/releases/download/v${{matrix.cmake_ver}}/cmake-${{matrix.cmake_ver}}-Linux-x86_64.sh + curl -L -o https://github.com/Kitware/CMake/releases/download/v${{matrix.cmake_ver}}/cmake-${{matrix.cmake_ver}}-Linux-x86_64.sh chmod +x cmake-${{matrix.cmake_ver}}-Linux-x86_64.sh sudo ./cmake-${{matrix.cmake_ver}}-Linux-x86_64.sh --skip-license --prefix=/usr/local