hipobject-hardware-test-gpu-direct.yml

# Copyright (c) Advanced Micro Devices, Inc. All rights reserved.
#
# SPDX-License-Identifier: MIT
#
# Single-VM GPU-buffer/host-buffer S3-over-RDMA integration test
# (two vfio-user functions, one VM).
# The long comment above the job is the important one: this topology has
# requirements that are not obvious and were expensive to find.

name: hipObject - Emulated HW Test - AIS and Host Buffers

on:
  # Pull requests, pushes to develop, a nightly run, and manual runs. The
  # push run catches a develop that no single PR was tested against. The
  # nightly run is only for the known-failing GPU buffer (see the
  # gpu-direct-gpu-buffer job).
  #
  # The paths are what this lane builds and runs, so a change it cannot
  # observe doesn't spend VM time. Unit, header and install tests,
  # examples, the BNXT provider (built OFF here), docs, Clang options
  # (this lane builds with GCC), and the CMake modules for packaging or
  # opt-in tooling never reach this lane's binaries; other lanes cover
  # them. Negations must come last: the last pattern a file matches
  # decides whether it counts.
  pull_request:
    branches: [develop]
    paths: &paths
      - "CMakeLists.txt"
      - "cmake/HIPOBJAddExecutable.cmake"
      - "cmake/HIPOBJAddLibraries.cmake"
      - "cmake/HIPOBJCompilerOptions.cmake"
      - "cmake/HIPOBJFortifySource.cmake"
      - "cmake/HIPOBJGNUCompilerOptions.cmake"
      - "cmake/HIPOBJMinioCppDeps.cmake"
      - "cmake/HIPOBJUseROCmCMake.cmake"
      - "cmake/minio-deps/**"
      - "include/**"
      - "integrations/minio-cpp/**"
      - "shared/**"
      - "src/**"
      - "ci/ernic/assert-transfer.sh"
      - "ci/ernic/background-task.sh"
      - ".github/actions/fetch-guest-vm/**"
      - ".github/actions/fetch-ionic-sources/**"
      - ".github/workflows/hipobject-hardware-test-gpu-direct.yml"
      - "!src/rdma/bnxt/**"
      - "!**/*.md"
  push:
    branches: [develop]
    paths: *paths
  # Every day at 06:41 UTC
  schedule:
    - cron: "41 6 * * *"
  workflow_dispatch:

# A newer commit on a pull request cancels the run for the older one.
# Every other run (a push, a manual run) gets a group of its own: in a
# shared group, a newer run replaces a pending one even without
# cancel-in-progress, which would leave a commit untested.
concurrency:
  group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
  cancel-in-progress: true

permissions:
  contents: read

env:
  # Bump these intentionally when the ernic or rocjitsu images are updated.
  # ERNIC_IMAGE: ernic.0b48aa1, the build whose rocm-ernic carries the s3
  #   backend. Every tag here is the date-prefixed, content-versioned one
  #   (<date>.<repo-sha>-<component>.<sha>-vfu.<sha>): immutable, so a rebuild
  #   cannot change what this lane tests underneath it.
  ERNIC_IMAGE: sbates130272/batesste-ci-images-ubuntu-rocm-ernic:20260922.g555f601-ernic.0b48aa1-vfu.8039244
  # ROCJITSU_IMAGE_GPU: the tag qemu-minimal's ernic-rocjitsu lane runs.
  #   ip_discovery.bin must come from whichever rocjitsu build serves the
  #   socket, and this lane generates it from this image, so the two cannot
  #   drift apart.
  ROCJITSU_IMAGE_GPU: sbates130272/batesste-ci-images-ubuntu-rocm-rocjitsu:20260924.gad7a357-rocjitsu.c85bb75
  ROCM_IMAGE: rocm/dev-ubuntu-24.04:10.0.0-full
  # rocm-ernic's Ansible needs community.general >=13.3.0. Pinned, like the
  # images, because the installed collections are cached under this version:
  # fetching them from Galaxy took ~3 minutes of every run.
  COMMUNITY_GENERAL_VERSION: 13.5.0

# Every run step uses bash, including in container jobs, where the runner
# would otherwise pick sh. "bash -e {0}" and not "bash": the bare name adds
# -o pipefail, which these scripts are not written for.
defaults:
  run:
    shell: bash -e {0}

jobs:
  # ---------------------------------------------------------------------
  # gpu-capable: the GPU-capable topology, in a VM.
  #
  # The other three emulated-hardware lanes run the client on host buffers
  # (hipobject-hardware-test-two-vm-protocol.yml, hipobject-hardware-test-minio-bridge.yml,
  # hipobject-hardware-test-s3-backend.yml). This one attaches a
  # SECOND vfio-user function -- rocjitsu's emulated gfx1250 -- beside the
  # ernic NIC, so the same S3-over-RDMA transfer can be driven from either a
  # real hipMalloc'd device buffer or an explicit host buffer on the same
  # topology. The GPU buffer (the gpu-direct-gpu-buffer job, at the end of
  # this file) exercises AMD Infinity Storage (AIS); the host buffer (this
  # job) keeps that topology covered without depending on VRAM.
  #
  # Modelled on qemu-minimal's ansible-playbook-test-ernic-rocjitsu lane,
  # with one deliberate difference: rather than baking a combined image with
  # qemu-tool gen-vm (~50 min), it starts from the published ernic-rocjitsu
  # guest artifact.
  #
  # That guest bakes everything this lane used to derive at job time:
  # mainline v7.2.4 (ionic_rdma calls ib_umem_get_va, absent before 7.2), the
  # three amdgpu DKMS patches this repo used to carry (see
  # ci/ernic/guest-image-recipe.md), the generated gfx1250 gap firmware, therock
  # ROCm 10.0 userspace, render/video group membership and an amdgpu-probe
  # helper holding the parameter set below. The prep boot -- install a
  # pinned kernel, reboot, patch and rebuild amdgpu DKMS, ~15 min every run
  # -- is gone with it, so the lane boots once, with both vfio-user
  # functions attached.
  #
  # Two hard requirements came out of that work and are reflected below.
  #
  # 1. The vfio-user servers must run in the same container/namespace as
  #    QEMU. Running rocm-ernic on the host against a containerised QEMU is
  #    what produced the handle_dma_map SIGSEGV and the double-errp abort
  #    this comment used to blame on the two-device combination. It is not a
  #    two-device bug: both functions enumerate in one guest, PORT_ACTIVE
  #    holds with the GPU attached, and amdgpu binds the device.
  #
  # 2. amdgpu needs the full emulation parameter set -- fw_load_type=0 to
  #    skip PSP, discovery=2, vm_update_mode=3, gpu_recovery=0, vramlimit
  #    and ip_block_mask=0x7f. Not 0x3f: this DKMS build enumerates an extra
  #    ras_v1_0 at index 5, so MES sits at 6, and dropping it makes
  #    gfx_v12_1_xcc_cp_resume dereference a NULL ring. The guest's
  #    /usr/local/bin/amdgpu-probe carries that set, so the lane calls it
  #    rather than restating the parameters. The gap firmware stubs are
  #    baked into the guest; ip_discovery.bin is NOT, and must come from the
  #    SAME rocjitsu build that serves the socket (ROCJITSU_IMAGE_GPU) or
  #    probe fails at -22 or -2.
  #
  # The guest also cannot be warm-rebooted with the rocjitsu function
  # attached -- the device reset wedges it. Nothing below reboots, but a
  # step that starts to is a bug, not a slow path.
  #
  # What previously parked this lane -- "amdgpu probe hangs in
  # amdgpu_ras_init -> amdgpu_atomfirmware_mem_ecc_supported" -- was neither
  # a hang nor a kernel problem. rocjitsu serves no option ROM, so
  # adev->mode_info.atom_context is NULL and the RAS VBIOS query oopses on
  # it. 0003-amdgpu-ras-guard-vbios-query fixes that; with it, plus
  # ip_block_mask=0x7f, modprobe returns 0 and KFD reports gfx_target_version
  # 120500. The probe step is still bounded at all three layers -- guest
  # command, ssh session and step -- so a regression reports instead of
  # eating the job timeout.
  # ---------------------------------------------------------------------
  gpu-direct:
    # Not from the matrix: a skipped job shows its name unexpanded
    name: GPU-Capable RDMA (Host Buffer)
    # Pushes and pull requests already run the host buffer, so the nightly
    # run would only repeat it.
    if: github.event_name != 'schedule'
    strategy:
      fail-fast: false
      matrix:
        include:
          - buffer_label: Host Buffer
            gpu_mode: nogpu
            allow_failure: false
    runs-on: ubuntu-24.04
    timeout-minutes: 120
    container: &container
      image: sbates130272/batesste-ci-images-ubuntu-qemu-libvfio-user:20260924.ge1cd640-qemu11.1.1-vfu.8039244
      options: --device /dev/kvm -v /var/run/docker.sock:/var/run/docker.sock
    # GUEST_ARTIFACT_*: the ernic-rocjitsu guest: mainline 7.2.4 (the
    # ionic_rdma floor), the patched amdgpu-dkms, the generated gfx1250 gap
    # firmware and therock ROCm 10.0 userspace, with render/video group
    # membership. Only ip_discovery.bin is still the consumer's to stage, and
    # it has to come from the rocjitsu build serving the socket
    # (ROCJITSU_IMAGE_GPU).
    #
    # Nothing may come between the anchor and the first key: CodeQL's YAML
    # parser fails on a comment there ("expected <block end>, but found
    # '<block mapping start>'") and skips the whole file.
    env: &job_env
      GUEST_ARTIFACT_REPO: docker.io/sbates130272/batesste-ci-images-ubuntu-qcow2-gen-ernic-rocjitsu
      GUEST_ARTIFACT_TAG: 20260925.g4868a2e-vm.resolute-ernic-rocjitsu-qm.e73a1e6-qcow2
      S3_ADDR: 192.168.200.1
      S3_PORT: 9000
      S3_BUCKET: hipobj-test
      SSH_PORT: 2222
      # Where the guest role looks for the ionic driver sources. Passed to it
      # explicitly, so the copy staged there cannot drift from its default.
      IONIC_GUEST_DIR: /var/tmp/ionic-src

    # The gpu-direct-gpu-buffer job runs these same steps
    steps: &steps
      - name: Checkout
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false

      - name: Checkout rocm-ernic (Ansible collection + guest driver sources)
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          repository: ROCm/rocm-ernic
          # Pinned to the last develop commit known to work here; later commits
          # break these lanes. Bump intentionally, like the images.
          ref: f7e3ecb18fca02fe9d1e6685952ee10d4a78efe3
          path: rocm-ernic
          persist-credentials: false

      - name: Start building minio-getput-rdma (ROCm container)
        run: |
          # In the background: pulling the ROCm image is most of the build's
          # ~3 minutes, and nothing needs the binary until it is copied into
          # the guest. "Wait for the build" joins it, and prints its log.
          #
          # Stage what the binary loads at run time in rocm-libs, which
          # travels to the guest VM with it: the ROCm libraries, pugixml,
          # curlpp when it's the system's, and the shared libhipobj. In
          # this container the binary's build-tree run path still finds
          # libhipobj in /hipobject-build/src, so the check treats that as
          # unstaged.
          bash ci/ernic/background-task.sh start build << 'EOF'
          docker run --rm --name hipobj-build --user root \
            -v "${{ github.workspace }}:/hipobject:ro" \
            -v "${{ github.workspace }}/ernic-build-gpu:/hipobject-build" \
            -e ROCM_PATH=/opt/rocm \
            -e CC=/opt/rocm/llvm/bin/amdclang \
            -e CXX=/opt/rocm/llvm/bin/amdclang++ \
            "$ROCM_IMAGE" \
            bash -c "
              apt-get update -qq && \
              apt-get install -y -qq git cmake ninja-build libibverbs-dev \
                libcurl4-openssl-dev libssl-dev nlohmann-json3-dev libpugixml-dev && \
              cmake -B /hipobject-build -G Ninja -S /hipobject \
                -DCMAKE_BUILD_TYPE=Release \
                -DCMAKE_PREFIX_PATH=/opt/rocm \
                -DBUILD_TESTING=ON \
                -DHIPOBJ_IONIC=ON \
                -DHIPOBJ_BNXT=OFF \
                -DHIPOBJ_INTEGRATION_TESTS=ON \
                -DHIPOBJ_MINIO_CLIENT=ON \
                -DHIPOBJ_BUILD_DOCS=OFF && \
              cmake --build /hipobject-build --parallel \$(nproc) \
                --target minio-getput-rdma && \
              mkdir -p /hipobject-build/rocm-libs && \
              export LD_LIBRARY_PATH=/opt/rocm/lib && \
              ldd /hipobject-build/integrations/minio-cpp/minio-getput-rdma \
                | awk '/=> \//{print \$3}' \
                | grep -E '^/opt/(rocm|amdgpu)|/libpugixml\.so|/libcurlpp\.so|/libhipobj\.so' \
                | xargs -r -I{} cp -L {} /hipobject-build/rocm-libs/ && \
              if LD_LIBRARY_PATH=/hipobject-build/rocm-libs \
                   ldd /hipobject-build/integrations/minio-cpp/minio-getput-rdma \
                   | grep -E 'not found|=> /hipobject-build/src/'; then \
                echo 'staged closure incomplete'; exit 1; \
              fi && \
              echo 'staged closure complete'
            "
          EOF

      - name: Restore the cached Ansible collections
        id: ansible_collections
        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: ~/.ansible/collections
          key: ansible-collections-community.general-${{ env.COMMUNITY_GENERAL_VERSION }}

      - name: Start installing deps
        env:
          COLLECTIONS_CACHED: ${{ steps.ansible_collections.outputs.cache-hit }}
        run: |
          # In the background, alongside the guest image fetch. "Wait for
          # deps" joins it before anything uses what it installs.
          bash ci/ernic/background-task.sh start deps << 'EOF'
          apt-get update -qq
          apt-get install -y -qq iproute2 libibverbs1 libnl-3-200 libnl-route-3-200 \
            ansible python3-jmespath
          if [ "$COLLECTIONS_CACHED" != true ]; then
            # Galaxy drops connections often enough to fail a lane on its own.
            for attempt in 1 2 3; do
              ansible-galaxy collection install \
                "community.general:==${COMMUNITY_GENERAL_VERSION}" && break
              # Fail here: carrying on without the collection fails much
              # later, in provisioning, with an error that doesn't say why.
              if [ "$attempt" -eq 3 ]; then
                echo "::error::galaxy install failed after 3 attempts"
                exit 1
              fi
              echo "::warning::galaxy install failed (attempt ${attempt}/3)"
              sleep $((attempt * 20))
            done
          fi
          EOF

      - name: Start pulling the ernic and rocjitsu images
        run: |
          # In the background, alongside the guest image fetch, so the steps
          # that run them don't wait for the pull. "Wait for the images"
          # joins it before the first of them.
          bash ci/ernic/background-task.sh start images << 'EOF'
          docker pull "$ERNIC_IMAGE"
          docker pull "$ROCJITSU_IMAGE_GPU"
          EOF

      - name: Fetch the ionic driver sources
        id: ionic_sources
        uses: $/.github/actions/fetch-ionic-sources

      - name: Fetch rocjitsu guest VM image
        id: guest_image
        uses: $/.github/actions/fetch-guest-vm
        with:
          repo: ${{ env.GUEST_ARTIFACT_REPO }}
          tag: ${{ env.GUEST_ARTIFACT_TAG }}

      - name: Wait for deps
        run: bash ci/ernic/background-task.sh wait deps

      - name: Wait for the images
        run: bash ci/ernic/background-task.sh wait images

      - name: Extract rocm-ernic binary from ernic image
        run: |
          docker create --name ernic-tmp-gpu "$ERNIC_IMAGE"
          # Not the workspace: it is mounted noexec in the job container, so a
          # 0755 binary there still fails exec with EACCES.
          docker cp ernic-tmp-gpu:/usr/local/bin/rocm-ernic /usr/local/bin/rocm-ernic
          docker rm ernic-tmp-gpu
          chmod +x /usr/local/bin/rocm-ernic
          # No pipe here: under `bash -e` a failure in a non-final pipeline
          # element is ignored, which is how the EACCES above reached the
          # next step disguised as a missing socket.
          rocm-ernic --help > /tmp/ernic-help.txt
          head -3 /tmp/ernic-help.txt

      - name: Select QEMU accelerator and guest size
        run: |
          if [ -c /dev/kvm ] && "${QEMU_PATH:-}qemu-system-x86_64" -accel help 2>/dev/null | grep -qw kvm; then
            echo "QEMU_ACCEL=kvm" >> "$GITHUB_ENV"
            echo "QEMU_CPU=host"  >> "$GITHUB_ENV"
          else
            echo "QEMU_ACCEL=tcg"  >> "$GITHUB_ENV"
            echo "QEMU_CPU=qemu64" >> "$GITHUB_ENV"
          fi
          HOST_MEM_MB=$(awk '/^MemTotal:/ {print int($2/1024)}' /proc/meminfo)
          # ionic_create_rdma_admin() hard-fails below 4 online CPUs
          # (IONIC_EQ_COUNT_MIN), so 4 is a floor, not a preference.
          VM_VCPUS=$(( $(nproc) - 1 )); [ "$VM_VCPUS" -lt 4 ] && VM_VCPUS=4
          VM_MEM_MB=$(( HOST_MEM_MB * 3 / 4 )); [ "$VM_MEM_MB" -gt 6144 ] && VM_MEM_MB=6144
          echo "VM_VCPUS=$VM_VCPUS" >> "$GITHUB_ENV"
          echo "VM_MEM_MB=$VM_MEM_MB" >> "$GITHUB_ENV"

      - name: Generate ip_discovery.bin on the host
        run: |
          # The only firmware the guest does not bake, because it is the only
          # piece that is not static: it describes the device the socket
          # serves, so it must come from the rocjitsu build serving it. The
          # gap stubs are in /lib/firmware/updates/amdgpu already.
          mkdir -p "${GITHUB_WORKSPACE}/rj-fw"
          chmod 777 "${GITHUB_WORKSPACE}/rj-fw"
          docker run --rm -v "${{ github.workspace }}/rj-fw:/out" \
            "$ROCJITSU_IMAGE_GPU" \
            rj-ip-discovery gfx1250 /out/ip_discovery.bin
          ls -l "${GITHUB_WORKSPACE}/rj-fw"

      - name: Start rocm-ernic S3 backend and rocjitsu GPU server
        run: |
          # rocjitsu runs as a sibling container, so its socket has to live on
          # a path the host daemon can bind-mount AND this container can see.
          # ${{ github.workspace }} is the host path; $GITHUB_WORKSPACE is the
          # same directory as mounted in here.
          mkdir -p "${GITHUB_WORKSPACE}/vfu"
          chmod 777 "${GITHUB_WORKSPACE}/vfu"

          rocm-ernic \
            --socket "${GITHUB_WORKSPACE}/vfu/ernic.sock" \
            --backend "s3:ip=${S3_ADDR},port=${S3_PORT},bucket=${S3_BUCKET}" \
            --verbose > /tmp/ernic-s3.log 2>&1 &
          echo "ERNIC_PID=$!" >> "$GITHUB_ENV"

          docker run -d --name rocjitsu-gpu \
            -v "${{ github.workspace }}/vfu:/run/vfu" \
            "$ROCJITSU_IMAGE_GPU" \
            rocjitsu --config /usr/local/share/rocjitsu/configs/gfx1250_mi455x.json \
                     --vfio-socket /run/vfu/rocjitsu.sock

          for _ in $(seq 1 45); do
            [ -S "${GITHUB_WORKSPACE}/vfu/ernic.sock" ] && \
            [ -S "${GITHUB_WORKSPACE}/vfu/rocjitsu.sock" ] && break
            sleep 2
          done
          for s in ernic rocjitsu; do
            [ -S "${GITHUB_WORKSPACE}/vfu/${s}.sock" ] || {
              echo "${s}.sock never appeared"
              cat /tmp/ernic-s3.log || true
              docker logs rocjitsu-gpu || true
              exit 1
            }
            chmod 666 "${GITHUB_WORKSPACE}/vfu/${s}.sock" || true
          done
          echo "both vfio-user sockets present"

      - name: Start QEMU VM with both vfio-user devices
        env:
          VM_DISK: ${{ steps.guest_image.outputs.vm_disk }}
        run: |
          "${QEMU_PATH:-}qemu-system-x86_64" \
            -machine "q35,accel=${QEMU_ACCEL},memory-backend=mem0" \
            -cpu "${QEMU_CPU}" -smp "${VM_VCPUS}" -m "${VM_MEM_MB}M" \
            -object "memory-backend-memfd,id=mem0,share=on,size=${VM_MEM_MB}M" \
            -nographic \
            -drive file="${VM_DISK}",format=qcow2,if=virtio \
            -netdev "user,id=net0,hostfwd=tcp::${SSH_PORT}-:22" \
            -device virtio-net-pci,netdev=net0 \
            -device '{"driver":"vfio-user-pci","rombar":0,"socket":{"type":"unix","path":"'"${GITHUB_WORKSPACE}"'/vfu/ernic.sock"}}' \
            -device '{"driver":"vfio-user-pci","rombar":0,"socket":{"type":"unix","path":"'"${GITHUB_WORKSPACE}"'/vfu/rocjitsu.sock"}}' \
            -serial file:/tmp/qemu-console.log -monitor none \
            -qmp unix:/tmp/qmp.sock,server=on,wait=off > /tmp/qemu.log 2>&1 &
          QEMU_PID=$!
          echo "QEMU_PID=$QEMU_PID" >> "$GITHUB_ENV"
          sleep 5
          kill -0 "$QEMU_PID" 2>/dev/null || { cat /tmp/qemu.log; exit 1; }

      - name: Wait for VM SSH
        env:
          SSH_KEY: ${{ steps.guest_image.outputs.ssh_key }}
          VM_USER: ${{ steps.guest_image.outputs.vm_user }}
        run: |
          chmod 600 "$SSH_KEY"
          for _ in $(seq 1 60); do
            ssh -i "$SSH_KEY" -o BatchMode=yes -o StrictHostKeyChecking=no \
                -o ConnectTimeout=5 -p "$SSH_PORT" "${VM_USER}@localhost" echo VM_READY \
              2>/dev/null && break
            sleep 5
          done
          ssh -i "$SSH_KEY" -o BatchMode=yes -o StrictHostKeyChecking=no \
              -p "$SSH_PORT" "${VM_USER}@localhost" echo VM_READY || {
            cat /tmp/qemu-console.log; exit 1
          }

      - name: Disable the guest's amdgpu driver apt source
        env:
          SSH_KEY: ${{ steps.guest_image.outputs.ssh_key }}
          VM_USER: ${{ steps.guest_image.outputs.vm_user }}
        run: |
          # The guest image's build added repo.radeon.com/amdgpu/31.60 as an
          # apt source to install amdgpu-dkms, and that tree has since
          # disappeared from repo.radeon.com. Nothing here installs from it --
          # the driver is baked into the image -- but every apt-get update in
          # the guest, the ernic Ansible collection's included, fails on its
          # missing Release file. Mark the stanza "Enabled: no" rather than
          # deleting it, so the change is visible from inside the guest. On an
          # image without the source this does nothing; remove it once
          # GUEST_ARTIFACT_TAG names such an image.
          ssh -i "$SSH_KEY" -o BatchMode=yes -o StrictHostKeyChecking=no \
              -p "$SSH_PORT" "${VM_USER}@localhost" bash -s << 'EOFVM'
          set -euo pipefail
          shopt -s nullglob
          for f in /etc/apt/sources.list.d/*.sources; do
            grep -q 'repo\.radeon\.com/amdgpu/' "$f" || continue
            tmp=$(mktemp)
            # deb822 field names are case-insensitive, and the stanza may
            # already carry an Enabled field (possibly "yes"): rewrite that
            # one to "no", and append the field only when there is none.
            awk 'BEGIN { RS = ""; ORS = "\n\n" }
                 /repo\.radeon\.com\/amdgpu\// {
                   n = split($0, line, "\n"); s = ""; found = 0
                   for (i = 1; i <= n; i++) {
                     if (line[i] ~ /^[Ee][Nn][Aa][Bb][Ll][Ee][Dd][ \t]*:/) {
                       line[i] = "Enabled: no"; found = 1
                     }
                     s = s (i > 1 ? "\n" : "") line[i]
                   }
                   $0 = found ? s : s "\nEnabled: no"
                 }
                 { print }' "$f" > "$tmp"
            sudo install -m 0644 "$tmp" "$f"
            rm -f "$tmp"
            echo "Disabled the amdgpu driver apt source in $f"
          done
          EOFVM

      - name: Verify both emulated functions are present
        env:
          SSH_KEY: ${{ steps.guest_image.outputs.ssh_key }}
          VM_USER: ${{ steps.guest_image.outputs.vm_user }}
        run: |
          vm_ssh() {
            ssh -i "$SSH_KEY" -o BatchMode=yes -o StrictHostKeyChecking=no "$@"
          }
          # Count by sysfs vendor/device, never by parsing lspci. lspci needs
          # /usr/share/misc/pci.ids, and a guest this early is still running
          # the boot's apt work -- the two-VM lane caught pciutils mid-rewrite,
          # and lspci bailed with "Parse error at line 28286" after one device,
          # which reads exactly like a function that never attached.
          count() {
            vm_ssh -p "$SSH_PORT" "${VM_USER}@localhost" \
              "grep -lix '$1' /sys/bus/pci/devices/*/vendor 2>/dev/null | wc -l" | tr -d '\r'
          }
          # 0x1dd8 is Pensando (the emulated ionic NIC), 0x1002 is AMD (rocjitsu).
          [ "$(count 0x1dd8)" -ge 1 ] || {
            echo "emulated ionic NIC not enumerated"; exit 1; }
          [ "$(count 0x1002)" -ge 1 ] || {
            echo "emulated rocjitsu GPU not enumerated"; exit 1; }
          echo "both emulated PCI functions enumerated"
          vm_ssh -p "$SSH_PORT" "${VM_USER}@localhost" 'lspci -nn' || true

      - name: Stage the ionic driver sources in the VM
        env:
          SSH_KEY: ${{ steps.guest_image.outputs.ssh_key }}
          VM_USER: ${{ steps.guest_image.outputs.vm_user }}
          IONIC_SRC: ${{ steps.ionic_sources.outputs.source_dir }}
          IONIC_REF: ${{ steps.ionic_sources.outputs.ref }}
        run: |
          # Left to itself, the guest role sparse-clones the kernel from
          # git.kernel.org through user-mode NAT, ~100 s every run. It skips
          # that when .patches-applied-<ref> is already in place, so copy in
          # the runner's cached tree; provisioning is given the same
          # directory and ref below.
          set -o pipefail
          tar -C "$IONIC_SRC" -cf - . | \
            ssh -i "$SSH_KEY" -o BatchMode=yes -o StrictHostKeyChecking=no \
              -p "$SSH_PORT" "${VM_USER}@localhost" \
              "sudo rm -rf '${IONIC_GUEST_DIR}' && sudo mkdir -p '${IONIC_GUEST_DIR}' && \
               sudo tar -C '${IONIC_GUEST_DIR}' -xf - && \
               test -f '${IONIC_GUEST_DIR}/.patches-applied-${IONIC_REF}'"
          echo "ionic sources at ${IONIC_REF} staged in ${IONIC_GUEST_DIR}"

      - name: Provision the VM with the ernic Ansible collection
        env:
          SSH_KEY: ${{ steps.guest_image.outputs.ssh_key }}
          VM_USER: ${{ steps.guest_image.outputs.vm_user }}
          IONIC_REF: ${{ steps.ionic_sources.outputs.ref }}
        run: |
          ERNIC_RUN_DIR=/tmp/ernic-ci
          mkdir -p "$ERNIC_RUN_DIR"
          cat > "$ERNIC_RUN_DIR/instances.json" << EOFJSON
          {
            "instances": [
              {
                "id": 1,
                "vm": {
                  "vm_name": "hipobj-gpu-vm1",
                  "ssh_port": ${SSH_PORT},
                  "ssh_user": "${VM_USER}"
                }
              }
            ]
          }
          EOFJSON
          export ANSIBLE_HOST_KEY_CHECKING=false
          export ANSIBLE_PRIVATE_KEY_FILE="$SSH_KEY"
          PROJECT_ROOT="${GITHUB_WORKSPACE}/rocm-ernic"
          export ANSIBLE_ROLES_PATH="${PROJECT_ROOT}/ansible/roles"
          cd "${PROJECT_ROOT}/ansible"
          ansible-playbook ci-site.yml --tags guest-setup \
            -e ernic_project_root="$PROJECT_ROOT" \
            -e ernic_run_dir="$ERNIC_RUN_DIR" \
            -e ernic_instances=1 \
            -e ernic_vm_ssh_base_port="$SSH_PORT" \
            -e ernic_vm_ssh_user="$VM_USER" \
            -e ernic_ionic_source_dir="$IONIC_GUEST_DIR" \
            -e ernic_ionic_kernel_ref="$IONIC_REF" \
            -e ernic_gpu_passthrough=false \
            -e ernic_guest_agent=false \
            -e ernic_set_hostname=false

      - name: Verify the RDMA device and the object store
        env:
          SSH_KEY: ${{ steps.guest_image.outputs.ssh_key }}
          VM_USER: ${{ steps.guest_image.outputs.vm_user }}
        run: |
          vm_ssh() {
            ssh -i "$SSH_KEY" -o BatchMode=yes -o StrictHostKeyChecking=no "$@"
          }
          kill -0 "$ERNIC_PID" 2>/dev/null || {
            echo "rocm-ernic is no longer running"; tail -30 /tmp/ernic-s3.log; exit 1; }
          grep -q "s3 bucket '${S3_BUCKET}'" /tmp/ernic-s3.log || {
            echo "no s3 store banner for '${S3_BUCKET}'"
            grep -i s3 /tmp/ernic-s3.log || tail -40 /tmp/ernic-s3.log
            exit 1
          }
          DEVICE=$(vm_ssh -p "$SSH_PORT" "${VM_USER}@localhost" 'sh -s' \
            < "${GITHUB_WORKSPACE}/rocm-ernic/scripts/find-rdma-device.sh" | tr -d '\r') || true
          [ -n "$DEVICE" ] || {
            echo "RDMA device not found in the guest"
            vm_ssh -p "$SSH_PORT" "${VM_USER}@localhost" \
              'ibv_devices; lsmod | grep ionic; dmesg | grep -i ionic | tail -30' || true
            exit 1
          }
          vm_ssh -p "$SSH_PORT" "${VM_USER}@localhost" \
            "ibv_devinfo -d $DEVICE" | tee /dev/stderr | grep -q PORT_ACTIVE || {
            echo "$DEVICE never reached PORT_ACTIVE"; exit 1; }
          echo "RDMA_DEVICE=$DEVICE" >> "$GITHUB_ENV"
          echo "$DEVICE is PORT_ACTIVE"

      - name: Stage ip_discovery.bin into the guest
        env:
          SSH_KEY: ${{ steps.guest_image.outputs.ssh_key }}
          VM_USER: ${{ steps.guest_image.outputs.vm_user }}
        run: |
          scp -i "$SSH_KEY" -o BatchMode=yes -o StrictHostKeyChecking=no \
            -P "$SSH_PORT" "${GITHUB_WORKSPACE}/rj-fw/ip_discovery.bin" \
            "${VM_USER}@localhost:/tmp/"
          # updates/ takes precedence over the packaged firmware, and is where
          # the guest's baked gap stubs already live.
          ssh -i "$SSH_KEY" -o BatchMode=yes -o StrictHostKeyChecking=no \
            -p "$SSH_PORT" "${VM_USER}@localhost" \
            'sudo install -D -m 0644 /tmp/ip_discovery.bin \
               /lib/firmware/updates/amdgpu/ip_discovery.bin && \
             ls -l /lib/firmware/updates/amdgpu/'

      - name: Load amdgpu via amdgpu-probe
        timeout-minutes: 8
        env:
          SSH_KEY: ${{ steps.guest_image.outputs.ssh_key }}
          VM_USER: ${{ steps.guest_image.outputs.vm_user }}
        run: |
          # qemu-minimal parks its lane on this step hanging with no output.
          # Bounded at three layers so a recurrence reports instead of eating
          # the job timeout: guest-side timeout, ssh keepalive, step timeout.
          ssh -i "$SSH_KEY" -o BatchMode=yes -o StrictHostKeyChecking=no \
              -o ConnectTimeout=10 -o ServerAliveInterval=15 -o ServerAliveCountMax=4 \
              -p "$SSH_PORT" "${VM_USER}@localhost" \
              "sudo timeout 300 /usr/local/bin/amdgpu-probe"
          echo "amdgpu-probe completed"

      - name: Verify amdgpu loaded and a GPU KFD node is present
        env:
          SSH_KEY: ${{ steps.guest_image.outputs.ssh_key }}
          VM_USER: ${{ steps.guest_image.outputs.vm_user }}
        run: |
          vm_ssh() {
            ssh -i "$SSH_KEY" -o BatchMode=yes -o StrictHostKeyChecking=no "$@"
          }
          vm_ssh -p "$SSH_PORT" "${VM_USER}@localhost" 'lsmod | grep -q amdgpu' || {
            echo "amdgpu module not loaded"; exit 1; }
          # /dev/kfd is not a valid assertion: kfd_chardev_init() creates it
          # before amdgpu_init() registers the PCI driver, so it exists even
          # when every probe failed. Node 0 is the CPU and has an empty name;
          # a bound GPU adds a named node.
          gpus=$(vm_ssh -p "$SSH_PORT" "${VM_USER}@localhost" \
            'cat /sys/class/kfd/kfd/topology/nodes/*/name 2>/dev/null | grep -c .')
          [ "$gpus" -ge 1 ] || {
            echo "amdgpu loaded but never bound the device"
            vm_ssh -p "$SSH_PORT" "${VM_USER}@localhost" \
              'sudo dmesg | grep -i amdgpu | tail -40' || true
            exit 1
          }
          echo "GPU KFD node present ($gpus)"

      - name: Wait for the build
        run: bash ci/ernic/background-task.sh wait build

      - name: Copy minio binary to VM
        env:
          SSH_KEY: ${{ steps.guest_image.outputs.ssh_key }}
          VM_USER: ${{ steps.guest_image.outputs.vm_user }}
        run: |
          scp -i "$SSH_KEY" -o BatchMode=yes -o StrictHostKeyChecking=no -P "$SSH_PORT" \
            "${GITHUB_WORKSPACE}/ernic-build-gpu/integrations/minio-cpp/minio-getput-rdma" \
            "${VM_USER}@localhost:/tmp/minio-getput-rdma"
          scp -i "$SSH_KEY" -o BatchMode=yes -o StrictHostKeyChecking=no \
            -P "$SSH_PORT" \
            ci/ernic/assert-transfer.sh \
            "${VM_USER}@localhost:/tmp/assert-transfer.sh"
          scp -i "$SSH_KEY" -o BatchMode=yes -o StrictHostKeyChecking=no -P "$SSH_PORT" -r \
            "${GITHUB_WORKSPACE}/ernic-build-gpu/rocm-libs" \
            "${VM_USER}@localhost:/tmp/rocm-libs"

      - name: Run ${{ matrix.buffer_label }} S3 PUT+GET test in VM
        id: client
        # Bounded at three layers, as the two-VM lane: guest-side timeout, ssh
        # keepalive, step timeout. Unbounded, this step wedged for the whole
        # 120-minute job budget at 76a300f -- the client printed its banner
        # and then produced nothing for 110 minutes. That is not merely slow:
        # a job-level timeout CANCELS the step rather than failing it, so
        # continue-on-error below does not apply, steps.client.outcome is
        # 'cancelled' rather than 'failure', and the reporting step is
        # skipped. A wedge has to become a non-zero exit for any of the
        # non-fatal handling to engage.
        timeout-minutes: 10
        # TEMPORARY. The GPU buffer passed until HIPOBJ_REQUIRE_GPU_DIRECT
        # stopped the client staging a device buffer through host memory and
        # calling that AIS; the honest run fails. Non-fatal so the nightly run
        # reports the known failure rather than going red every night --
        # every step before this one is still fatal, so the build, the guest,
        # the NIC and the KFD node are all still enforced. Remove this the
        # moment the transfer works.
        continue-on-error: ${{ matrix.allow_failure }}
        env:
          SSH_KEY: ${{ steps.guest_image.outputs.ssh_key }}
          VM_USER: ${{ steps.guest_image.outputs.vm_user }}
        run: |
          GPU_MODE="${{ matrix.gpu_mode }}"
          BUFFER_LABEL="${{ matrix.buffer_label }}"
          ssh -i "$SSH_KEY" -o BatchMode=yes -o StrictHostKeyChecking=no \
              -o ConnectTimeout=10 -o ServerAliveInterval=15 -o ServerAliveCountMax=4 \
              -p "$SSH_PORT" "${VM_USER}@localhost" \
              "S3_ADDR='${S3_ADDR}' S3_PORT='${S3_PORT}' RDMA_DEVICE='${RDMA_DEVICE}' GPU_MODE='${GPU_MODE}' BUFFER_LABEL='${BUFFER_LABEL}' bash -s" << 'EOFVM'
          set -e
          chmod +x /tmp/minio-getput-rdma
          export LD_LIBRARY_PATH=/tmp/rocm-libs
          ldd /tmp/minio-getput-rdma | grep -E 'not found' && {
            echo "ERROR: unresolved shared libraries in guest"; exit 1; } || true
          echo "=== Waiting for the in-band S3 endpoint ==="
          ready=
          for i in $(seq 1 30); do
            code=$(curl -s -o /dev/null -w '%{http_code}' \
              "http://${S3_ADDR}:${S3_PORT}/" 2>/dev/null) || code=
            case "$code" in [234]*) ready=1; break;; esac
            sleep 2
          done
          [ -n "$ready" ] || {
            echo "ERROR: S3 endpoint never answered"; ip -4 -br addr show; exit 1; }
          export HIPOBJ_NIC_HINT="${RDMA_DEVICE}"
          echo "=== KFD access ==="
          id
          ls -l /dev/kfd /dev/dri/ 2>&1 || true
          # Run it as root. The KFD node comes up (this step is gated on that)
          # but /dev/kfd and /dev/dri/render* are root:render 0660 and the
          # guest's login user is in neither group, so the HIP runtime
          # enumerates no agent and hipMalloc returns 100, hipErrorNoDevice.
          echo "=== minio-getput-rdma, ${BUFFER_LABEL} (NIC $HIPOBJ_NIC_HINT) ==="
          rc=0
          if [ "$GPU_MODE" = gpu ]; then
            sudo timeout 300 env LD_LIBRARY_PATH=/tmp/rocm-libs \
                HIPOBJ_NIC_HINT="$HIPOBJ_NIC_HINT" \
                HIPOBJ_REQUIRE_GPU_DIRECT=1 \
                /tmp/minio-getput-rdma "${S3_ADDR}:${S3_PORT}" \
                minioadmin minioadmin 65536 gpu > /tmp/getput.log 2>&1 || rc=$?
          else
            sudo timeout 300 env LD_LIBRARY_PATH=/tmp/rocm-libs \
                HIPOBJ_NIC_HINT="$HIPOBJ_NIC_HINT" \
                /tmp/minio-getput-rdma "${S3_ADDR}:${S3_PORT}" \
                minioadmin minioadmin 65536 "${GPU_MODE}" > /tmp/getput.log 2>&1 || rc=$?
          fi
          cat /tmp/getput.log
          if [ "$rc" -eq 124 ]; then echo "client hung and was killed after 300s"; fi
          if [ "$rc" -ne 0 ]; then
            if [ "$GPU_MODE" = gpu ]; then
              echo "=== HIP saw no device; agent enumeration ==="
              command -v rocminfo > /dev/null && \
                sudo env LD_LIBRARY_PATH=/tmp/rocm-libs rocminfo 2>&1 | head -40
              sudo dmesg | grep -i 'kfd\|amdgpu' | tail -20 || true
            fi
            exit "$rc"
          fi
          # A green exit is not enough: the bridge falls back to plain HTTP on
          # a failed RDMA transfer and still returns success. This asserts the
          # GET returned what the PUT wrote and that both moved over RDMA.
          chmod +x /tmp/assert-transfer.sh
          /tmp/assert-transfer.sh /tmp/getput.log rdma
          EOFVM

      - name: Report the known AIS failure
        if: steps.client.outcome == 'failure' && matrix.gpu_mode == 'gpu'
        run: |
          echo "::warning title=Known failure::AIS GPU-buffer transfer failed. \
          The job is green only because the client step is continue-on-error."
          cat >> "$GITHUB_STEP_SUMMARY" << 'EOF'
          ### AIS GPU Buffer: KNOWN FAILING

          The GPU-buffer transfer failed and the step is `continue-on-error`,
          so this job reports success. **It did not pass.**

          This lane was green until `HIPOBJ_REQUIRE_GPU_DIRECT=1` stopped the
          client falling back to host staging for a device pointer. The green
          runs were not testing AIS; this red one is.

          Two signatures seen so far, and it is not known whether they share a
          cause. `RDMA PUT failed (-1)` at 65536 bytes, immediately; and a hang
          with no output at all after the client acquires a KFD VM, which the
          `timeout 300` in the step above converts into exit 124. Check the
          step log for `client hung and was killed after 300s` to tell them
          apart.
          EOF
      - name: Collect logs
        if: always()
        env:
          SSH_KEY: ${{ steps.guest_image.outputs.ssh_key }}
          VM_USER: ${{ steps.guest_image.outputs.vm_user }}
        run: |
          echo "=== ernic-s3.log ===";  cat /tmp/ernic-s3.log     || true
          echo "=== rocjitsu ===";      docker logs rocjitsu-gpu   || true
          echo "=== qemu.log ===";      cat /tmp/qemu.log          || true
          echo "=== qemu-console ===";  cat /tmp/qemu-console.log  || true
          timeout 60 ssh -i "$SSH_KEY" -o BatchMode=yes -o StrictHostKeyChecking=no \
            -o ConnectTimeout=10 -p "$SSH_PORT" "${VM_USER}@localhost" \
            'sudo dmesg | tail -200' || true

      - name: Teardown
        if: always()
        run: |
          kill "${QEMU_PID:-}"  2>/dev/null || true
          kill "${ERNIC_PID:-}" 2>/dev/null || true
          docker rm -f rocjitsu-gpu 2>/dev/null || true
          # Still running if the job failed before "Wait for the build".
          docker rm -f hipobj-build 2>/dev/null || true
          docker run --rm -v "${{ github.workspace }}/ernic-build-gpu:/d" \
            busybox sh -c "rm -rf /d/*" 2>/dev/null || true
          rm -rf "${{ github.workspace }}/ernic-build-gpu" \
                 "${GITHUB_WORKSPACE}/vfu" "${GITHUB_WORKSPACE}/rj-fw" 2>/dev/null || true

  # ---------------------------------------------------------------------
  # gpu-direct-gpu-buffer: the gpu-direct job above, on a GPU buffer.
  #
  # TEMPORARY. The GPU-buffer transfer is known to fail (see the client
  # step's continue-on-error), so it runs nightly and on manual runs rather
  # than on every pull request and push, where it cost runner time and
  # reported nothing. When the transfer works, move this matrix entry back
  # into the gpu-direct job with allow_failure false, and delete this job.
  # ---------------------------------------------------------------------
  gpu-direct-gpu-buffer:
    name: GPU-Capable RDMA (GPU Buffer)
    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
    strategy:
      matrix:
        include:
          - buffer_label: GPU Buffer
            gpu_mode: gpu
            allow_failure: true
    runs-on: ubuntu-24.04
    timeout-minutes: 120
    container: *container
    env: *job_env
    steps: *steps