hipobject-hardware-test-s3-backend.yml

# Copyright (c) Advanced Micro Devices, Inc. All rights reserved.
#
# SPDX-License-Identifier: MIT
#
# S3-backend integration test (rocm-ernic --backend s3, one guest VM).
#
# rocm-ernic runs on the runner with an in-process S3 object store whose
# HTTP endpoint is terminated in-band on the emulated wire, so one guest VM
# and one server are a complete object fabric -- no tap, no second VM. The
# VM attaches the vfio-user function, is provisioned with the ionic DKMS
# driver by rocm-ernic's own Ansible collection, and then runs
# minio-getput-rdma against the store with HIPOBJ_NIC_HINT set to the
# discovered device.
#
# This is the job that exercises the RDMA data path on host buffers: a
# passing run carries x-amz-rdma-token / x-amz-rdma-reply on the wire.

name: hipObject - Emulated HW Test - S3 Backend

on:
  # Pull requests, pushes to develop, and manual runs. The push run
  # catches a develop that no single PR was tested against.
  #
  # The paths are what this lane builds and runs, so a change it cannot
  # observe doesn't spend VM time. Unit, header and install tests,
  # examples, the BNXT provider (built OFF here), docs, Clang options
  # (this lane builds with GCC), and the CMake modules for packaging or
  # opt-in tooling never reach this lane's binaries; other lanes cover
  # them. Negations must come last: the last pattern a file matches
  # decides whether it counts.
  pull_request:
    branches: [develop]
    paths: &paths
      - "CMakeLists.txt"
      - "cmake/HIPOBJAddExecutable.cmake"
      - "cmake/HIPOBJAddLibraries.cmake"
      - "cmake/HIPOBJCompilerOptions.cmake"
      - "cmake/HIPOBJFortifySource.cmake"
      - "cmake/HIPOBJGNUCompilerOptions.cmake"
      - "cmake/HIPOBJMinioCppDeps.cmake"
      - "cmake/HIPOBJUseROCmCMake.cmake"
      - "cmake/minio-deps/**"
      - "include/**"
      - "integrations/minio-cpp/**"
      - "shared/**"
      - "src/**"
      - "ci/ernic/assert-transfer.sh"
      - "ci/ernic/background-task.sh"
      - ".github/actions/fetch-guest-vm/**"
      - ".github/actions/fetch-ionic-sources/**"
      - ".github/workflows/hipobject-hardware-test-s3-backend.yml"
      - "!src/rdma/bnxt/**"
      - "!**/*.md"
  push:
    branches: [develop]
    paths: *paths
  workflow_dispatch:

# A newer commit on a pull request cancels the run for the older one.
# Every other run (a push, a manual run) gets a group of its own: in a
# shared group, a newer run replaces a pending one even without
# cancel-in-progress, which would leave a commit untested.
concurrency:
  group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
  cancel-in-progress: true

permissions:
  contents: read

env:
  # Bump these intentionally when the ernic or rocjitsu images are updated.
  # ERNIC_IMAGE: ernic.0b48aa1, the build whose rocm-ernic carries the s3
  #   backend. Every tag here is the date-prefixed, content-versioned one
  #   (<date>.<repo-sha>-<component>.<sha>-vfu.<sha>): immutable, so a rebuild
  #   cannot change what this lane tests underneath it.
  ERNIC_IMAGE: sbates130272/batesste-ci-images-ubuntu-rocm-ernic:20260922.g555f601-ernic.0b48aa1-vfu.8039244
  ROCM_IMAGE: rocm/dev-ubuntu-24.04:10.0.0-full
  # rocm-ernic's Ansible needs community.general >=13.3.0. Pinned, like the
  # images, because the installed collections are cached under this version:
  # fetching them from Galaxy took ~3 minutes of every run.
  COMMUNITY_GENERAL_VERSION: 13.5.0

# Every run step uses bash, including in container jobs, where the runner
# would otherwise pick sh. "bash -e {0}" and not "bash": the bare name adds
# -o pipefail, which these scripts are not written for.
defaults:
  run:
    shell: bash -e {0}

jobs:
  s3-backend:
    name: S3 Backend
    runs-on: ubuntu-24.04
    timeout-minutes: 90
    # Run inside the same QEMU+libvfio-user image used by rocm-ernic's
    # s3-lane: has QEMU, oras, zstd, and libvfio-user pre-installed.
    container:
      image: sbates130272/batesste-ci-images-ubuntu-qemu-libvfio-user:20260924.ge1cd640-qemu11.1.1-vfu.8039244
      options: --device /dev/kvm
    env:
      # Matches the rocm-ernic s3-lane guest artifact.
      GUEST_ARTIFACT_REPO: docker.io/sbates130272/batesste-ci-images-ubuntu-qcow2-gen-ernic-rocjitsu
      GUEST_ARTIFACT_TAG: 20260925.g4868a2e-vm.resolute-ernic-rocjitsu-qm.e73a1e6-qcow2
      S3_ADDR: 192.168.200.1
      S3_PORT: 9000
      SSH_PORT: 2222
      # getput-rdma hardcodes this bucket, and the backend defaults to
      # "ernic", so it has to be passed explicitly or every PUT 400s.
      S3_BUCKET: hipobj-test
      # Where the guest role looks for the ionic driver sources. Passed to it
      # explicitly, so the copy staged there cannot drift from its default.
      IONIC_GUEST_DIR: /var/tmp/ionic-src

    steps:
      - name: Checkout
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false

      - name: Checkout rocm-ernic (Ansible collection + guest driver sources)
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          repository: ROCm/rocm-ernic
          # Pinned to the last develop commit known to work here; later commits
          # break these lanes. Bump intentionally, like the images.
          ref: f7e3ecb18fca02fe9d1e6685952ee10d4a78efe3
          path: rocm-ernic
          persist-credentials: false

      - name: Start building minio-getput-rdma (ROCm container)
        run: |
          # In the background: pulling the ROCm image is most of the build's
          # ~3 minutes, and nothing needs the binary until it is copied into
          # the guest. "Wait for the build" joins it, and prints its log.
          #
          # Stage what the binary loads at run time in rocm-libs, which
          # travels to the guest VM with it: the ROCm libraries, pugixml,
          # curlpp when it's the system's, and the shared libhipobj. In
          # this container the binary's build-tree run path still finds
          # libhipobj in /hipobject-build/src, so the check treats that as
          # unstaged.
          bash ci/ernic/background-task.sh start build << 'EOF'
          docker run --rm --name hipobj-build \
            --user root \
            -v "${{ github.workspace }}:/hipobject:ro" \
            -v "${{ github.workspace }}/ernic-build-s3:/hipobject-build" \
            -e ROCM_PATH=/opt/rocm \
            -e CC=/opt/rocm/llvm/bin/amdclang \
            -e CXX=/opt/rocm/llvm/bin/amdclang++ \
            "$ROCM_IMAGE" \
            bash -c "
              apt-get update -qq && \
              apt-get install -y -qq git cmake ninja-build libibverbs-dev \
                libcurl4-openssl-dev libssl-dev nlohmann-json3-dev libpugixml-dev && \
              cmake -B /hipobject-build -G Ninja -S /hipobject \
                -DCMAKE_BUILD_TYPE=Release \
                -DCMAKE_PREFIX_PATH=/opt/rocm \
                -DBUILD_TESTING=ON \
                -DHIPOBJ_IONIC=ON \
                -DHIPOBJ_BNXT=OFF \
                -DHIPOBJ_INTEGRATION_TESTS=ON \
                -DHIPOBJ_MINIO_CLIENT=ON \
                -DHIPOBJ_BUILD_DOCS=OFF && \
              cmake --build /hipobject-build --parallel \$(nproc) \
                --target minio-getput-rdma && \
              mkdir -p /hipobject-build/rocm-libs && \
              export LD_LIBRARY_PATH=/opt/rocm/lib && \
              ldd /hipobject-build/integrations/minio-cpp/minio-getput-rdma \
                | awk '/=> \//{print \$3}' \
                | grep -E '^/opt/(rocm|amdgpu)|/libpugixml\.so|/libcurlpp\.so|/libhipobj\.so' \
                | xargs -r -I{} cp -L {} /hipobject-build/rocm-libs/ && \
              if LD_LIBRARY_PATH=/hipobject-build/rocm-libs \
                   ldd /hipobject-build/integrations/minio-cpp/minio-getput-rdma \
                   | grep -E 'not found|=> /hipobject-build/src/'; then \
                echo 'staged closure incomplete'; exit 1; \
              fi && \
              echo 'staged closure complete'
            "
          EOF

      - name: Restore the cached Ansible collections
        id: ansible_collections
        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: ~/.ansible/collections
          key: ansible-collections-community.general-${{ env.COMMUNITY_GENERAL_VERSION }}

      - name: Start installing deps
        env:
          COLLECTIONS_CACHED: ${{ steps.ansible_collections.outputs.cache-hit }}
        run: |
          # In the background, alongside the guest image fetch. "Wait for
          # deps" joins it before anything uses what it installs.
          bash ci/ernic/background-task.sh start deps << 'EOF'
          apt-get update -qq
          apt-get install -y -qq iproute2 libibverbs1 libnl-3-200 libnl-route-3-200 \
            ansible python3-jmespath
          if [ "$COLLECTIONS_CACHED" != true ]; then
            # Galaxy drops connections often enough to fail a lane on its own.
            for attempt in 1 2 3; do
              ansible-galaxy collection install \
                "community.general:==${COMMUNITY_GENERAL_VERSION}" && break
              # Fail here: carrying on without the collection fails much
              # later, in provisioning, with an error that doesn't say why.
              if [ "$attempt" -eq 3 ]; then
                echo "::error::galaxy install failed after 3 attempts"
                exit 1
              fi
              echo "::warning::galaxy install failed (attempt ${attempt}/3)"
              sleep $((attempt * 20))
            done
          fi
          EOF

      - name: Start pulling the ernic image
        run: |
          # In the background, alongside the guest image fetch, so the step
          # that extracts rocm-ernic from it doesn't wait for the pull. "Wait
          # for the image" joins it before then.
          bash ci/ernic/background-task.sh start images << 'EOF'
          docker pull "$ERNIC_IMAGE"
          EOF

      - name: Fetch the ionic driver sources
        id: ionic_sources
        uses: $/.github/actions/fetch-ionic-sources

      - name: Fetch guest VM image
        id: guest_image
        uses: $/.github/actions/fetch-guest-vm
        with:
          repo: ${{ env.GUEST_ARTIFACT_REPO }}
          tag: ${{ env.GUEST_ARTIFACT_TAG }}

      - name: Wait for deps
        run: bash ci/ernic/background-task.sh wait deps

      - name: Wait for the image
        run: bash ci/ernic/background-task.sh wait images

      - name: Extract rocm-ernic binary from ernic image
        run: |
          docker create --name ernic-tmp "$ERNIC_IMAGE"
          # Not the workspace: it is mounted noexec in the job container, so a
          # 0755 binary there still fails exec with EACCES.
          docker cp ernic-tmp:/usr/local/bin/rocm-ernic /usr/local/bin/rocm-ernic
          docker rm ernic-tmp
          chmod +x /usr/local/bin/rocm-ernic
          # No pipe here: under `bash -e` a failure in a non-final pipeline
          # element is ignored, which is how the EACCES above reached the
          # next step disguised as a missing socket.
          rocm-ernic --help > /tmp/ernic-help.txt
          head -3 /tmp/ernic-help.txt

      - name: Select QEMU accelerator and guest size
        run: |
          if [ -c /dev/kvm ] && "${QEMU_PATH:-}qemu-system-x86_64" -accel help 2>/dev/null | grep -qw kvm; then
            echo "QEMU_ACCEL=kvm"  >> "$GITHUB_ENV"
            echo "QEMU_CPU=host"   >> "$GITHUB_ENV"
          else
            echo "QEMU_ACCEL=tcg"       >> "$GITHUB_ENV"
            echo "QEMU_CPU=qemu64"      >> "$GITHUB_ENV"
          fi
          HOST_MEM_MB=$(awk '/^MemTotal:/ {print int($2/1024)}' /proc/meminfo)
          VM_VCPUS=$(( $(nproc) - 1 )); [ "$VM_VCPUS" -lt 4 ] && VM_VCPUS=4
          VM_MEM_MB=$(( HOST_MEM_MB * 3 / 4 )); [ "$VM_MEM_MB" -gt 4096 ] && VM_MEM_MB=4096
          echo "VM_VCPUS=$VM_VCPUS" >> "$GITHUB_ENV"
          echo "VM_MEM_MB=$VM_MEM_MB" >> "$GITHUB_ENV"

      - name: Start rocm-ernic S3 backend (no tap)
        run: |
          rm -f /tmp/vfio-user-rocm-ernic.sock
          rocm-ernic \
            --socket /tmp/vfio-user-rocm-ernic.sock \
            --backend "s3:ip=${S3_ADDR},port=${S3_PORT},bucket=${S3_BUCKET}" \
            --verbose > /tmp/ernic-s3.log 2>&1 &
          echo "ERNIC_PID=$!" >> "$GITHUB_ENV"
          for _ in $(seq 1 30); do
            [ -S /tmp/vfio-user-rocm-ernic.sock ] && { chmod 666 /tmp/vfio-user-rocm-ernic.sock; echo "Server ready"; break; }
            sleep 1
          done
          [ -S /tmp/vfio-user-rocm-ernic.sock ] || { cat /tmp/ernic-s3.log; exit 1; }

      - name: Start QEMU VM
        env:
          VM_DISK: ${{ steps.guest_image.outputs.vm_disk }}
        run: |
          "${QEMU_PATH:-}qemu-system-x86_64" \
            -machine "q35,accel=${QEMU_ACCEL},memory-backend=mem0" \
            -cpu "${QEMU_CPU}" -smp "${VM_VCPUS}" -m "${VM_MEM_MB}M" \
            -object "memory-backend-memfd,id=mem0,share=on,size=${VM_MEM_MB}M" \
            -nographic \
            -drive file="${VM_DISK}",format=qcow2,if=virtio \
            -netdev "user,id=net0,hostfwd=tcp::${SSH_PORT}-:22" \
            -device virtio-net-pci,netdev=net0 \
            -device '{"driver":"vfio-user-pci","socket":{"type":"unix","path":"/tmp/vfio-user-rocm-ernic.sock"}}' \
            -serial file:/tmp/qemu-console.log -monitor none \
            -qmp unix:/tmp/qmp.sock,server=on,wait=off > /tmp/qemu.log 2>&1 &
          QEMU_PID=$!
          echo "QEMU_PID=$QEMU_PID" >> "$GITHUB_ENV"
          sleep 5
          kill -0 "$QEMU_PID" 2>/dev/null || { cat /tmp/qemu.log; exit 1; }

      - name: Wait for VM SSH
        env:
          SSH_KEY: ${{ steps.guest_image.outputs.ssh_key }}
          VM_USER: ${{ steps.guest_image.outputs.vm_user }}
        run: |
          chmod 600 "$SSH_KEY"
          vm_ssh() {
            ssh -i "$SSH_KEY" -o BatchMode=yes -o StrictHostKeyChecking=no \
                -o ConnectTimeout=5 "$@"
          }
          for _ in $(seq 1 60); do
            vm_ssh -p "$SSH_PORT" "${VM_USER}@localhost" echo VM_READY 2>/dev/null && {
              echo "VM_USER=$VM_USER" >> "$GITHUB_ENV"
              echo "SSH_KEY=$SSH_KEY" >> "$GITHUB_ENV"
              break
            }
            sleep 5
          done
          vm_ssh -p "$SSH_PORT" "${VM_USER}@localhost" echo VM_READY || {
            cat /tmp/qemu-console.log; exit 1
          }

      - name: Disable the guest's amdgpu driver apt source
        env:
          SSH_KEY: ${{ steps.guest_image.outputs.ssh_key }}
          VM_USER: ${{ steps.guest_image.outputs.vm_user }}
        run: |
          # The guest image's build added repo.radeon.com/amdgpu/31.60 as an
          # apt source to install amdgpu-dkms, and that tree has since
          # disappeared from repo.radeon.com. Nothing here installs from it --
          # the driver is baked into the image -- but every apt-get update in
          # the guest, the ernic Ansible collection's included, fails on its
          # missing Release file. Mark the stanza "Enabled: no" rather than
          # deleting it, so the change is visible from inside the guest. On an
          # image without the source this does nothing; remove it once
          # GUEST_ARTIFACT_TAG names such an image.
          ssh -i "$SSH_KEY" -o BatchMode=yes -o StrictHostKeyChecking=no \
              -p "$SSH_PORT" "${VM_USER}@localhost" bash -s << 'EOFVM'
          set -euo pipefail
          shopt -s nullglob
          for f in /etc/apt/sources.list.d/*.sources; do
            grep -q 'repo\.radeon\.com/amdgpu/' "$f" || continue
            tmp=$(mktemp)
            # deb822 field names are case-insensitive, and the stanza may
            # already carry an Enabled field (possibly "yes"): rewrite that
            # one to "no", and append the field only when there is none.
            awk 'BEGIN { RS = ""; ORS = "\n\n" }
                 /repo\.radeon\.com\/amdgpu\// {
                   n = split($0, line, "\n"); s = ""; found = 0
                   for (i = 1; i <= n; i++) {
                     if (line[i] ~ /^[Ee][Nn][Aa][Bb][Ll][Ee][Dd][ \t]*:/) {
                       line[i] = "Enabled: no"; found = 1
                     }
                     s = s (i > 1 ? "\n" : "") line[i]
                   }
                   $0 = found ? s : s "\nEnabled: no"
                 }
                 { print }' "$f" > "$tmp"
            sudo install -m 0644 "$tmp" "$f"
            rm -f "$tmp"
            echo "Disabled the amdgpu driver apt source in $f"
          done
          EOFVM

      - name: Install kernel headers and build tools in VM
        env:
          SSH_KEY: ${{ steps.guest_image.outputs.ssh_key }}
          VM_USER: ${{ steps.guest_image.outputs.vm_user }}
        run: |
          ssh -i "$SSH_KEY" -o BatchMode=yes -o StrictHostKeyChecking=no \
              -p "$SSH_PORT" "${VM_USER}@localhost" bash -s << 'EOFVM'
          set -e
          KVER=$(uname -r)
          # A guest on a mainline kernel already has its headers from the
          # .deb, and kernel.ubuntu.com has no Packages index behind it, so
          # asking apt for linux-headers-$(uname -r) would 404.
          HEADER_PKG=""
          if [ -d "/lib/modules/${KVER}/build" ]; then
            echo "Kernel headers already present for ${KVER}"
          else
            HEADER_PKG="linux-headers-${KVER}"
          fi
          APT_LOCK_WAIT="-o DPkg::Lock::Timeout=300"
          for i in 1 2 3; do
            sudo apt-get $APT_LOCK_WAIT update -qq && break || sleep 5
          done
          for i in 1 2 3 4 5; do
            sudo DEBIAN_FRONTEND=noninteractive apt-get $APT_LOCK_WAIT install -y -qq \
              ${HEADER_PKG} dkms git build-essential cmake ninja-build \
              pkg-config libnl-3-dev libnl-route-3-dev libudev-dev && break
            [ "$i" -eq 5 ] && { echo "apt-get install failed after 5 attempts"; exit 1; }
            sleep 10
          done
          echo "Kernel packages installed for ${KVER}"
          EOFVM

      - name: Stage the ionic driver sources in the VM
        env:
          SSH_KEY: ${{ steps.guest_image.outputs.ssh_key }}
          VM_USER: ${{ steps.guest_image.outputs.vm_user }}
          IONIC_SRC: ${{ steps.ionic_sources.outputs.source_dir }}
          IONIC_REF: ${{ steps.ionic_sources.outputs.ref }}
        run: |
          # Left to itself, the guest role sparse-clones the kernel from
          # git.kernel.org through user-mode NAT, ~100 s every run. It skips
          # that when .patches-applied-<ref> is already in place, so copy in
          # the runner's cached tree; provisioning is given the same
          # directory and ref below.
          set -o pipefail
          tar -C "$IONIC_SRC" -cf - . | \
            ssh -i "$SSH_KEY" -o BatchMode=yes -o StrictHostKeyChecking=no \
              -p "$SSH_PORT" "${VM_USER}@localhost" \
              "sudo rm -rf '${IONIC_GUEST_DIR}' && sudo mkdir -p '${IONIC_GUEST_DIR}' && \
               sudo tar -C '${IONIC_GUEST_DIR}' -xf - && \
               test -f '${IONIC_GUEST_DIR}/.patches-applied-${IONIC_REF}'"
          echo "ionic sources at ${IONIC_REF} staged in ${IONIC_GUEST_DIR}"

      - name: Provision the VM with the ernic Ansible collection
        env:
          SSH_KEY: ${{ steps.guest_image.outputs.ssh_key }}
          VM_USER: ${{ steps.guest_image.outputs.vm_user }}
          IONIC_REF: ${{ steps.ionic_sources.outputs.ref }}
        run: |
          # The guest image bakes the kernel and the rdma-core userspace but
          # NOT the ionic DKMS driver or the udev naming rules -- a pristine
          # image has no ionic module and no /sys/class/infiniband entry at
          # all. Without this step there is no RDMA device to hint at.
          ERNIC_RUN_DIR=/tmp/ernic-ci
          mkdir -p "$ERNIC_RUN_DIR"
          cat > "$ERNIC_RUN_DIR/instances.json" << EOFJSON
          {
            "instances": [
              {
                "id": 1,
                "vm": {
                  "vm_name": "hipobj-s3-vm1",
                  "ssh_port": ${SSH_PORT},
                  "ssh_user": "${VM_USER}"
                }
              }
            ]
          }
          EOFJSON
          export ANSIBLE_HOST_KEY_CHECKING=false
          export ANSIBLE_PRIVATE_KEY_FILE="$SSH_KEY"
          PROJECT_ROOT="${GITHUB_WORKSPACE}/rocm-ernic"
          export ANSIBLE_ROLES_PATH="${PROJECT_ROOT}/ansible/roles"
          cd "${PROJECT_ROOT}/ansible"
          # ernic_configure_nic is left at its default (true): without an
          # IPv4 address on the emulated NIC the guest has no route to the
          # endpoint, and ionic_rdma populates no RoCEv2 GID for the client
          # to mint into its x-amz-rdma-token.
          ansible-playbook ci-site.yml --tags guest-setup \
            -e ernic_project_root="$PROJECT_ROOT" \
            -e ernic_run_dir="$ERNIC_RUN_DIR" \
            -e ernic_instances=1 \
            -e ernic_vm_ssh_base_port="$SSH_PORT" \
            -e ernic_vm_ssh_user="$VM_USER" \
            -e ernic_ionic_source_dir="$IONIC_GUEST_DIR" \
            -e ernic_ionic_kernel_ref="$IONIC_REF" \
            -e ernic_gpu_passthrough=false \
            -e ernic_guest_agent=false \
            -e ernic_set_hostname=false

      - name: Verify the RDMA device and the object store
        env:
          SSH_KEY: ${{ steps.guest_image.outputs.ssh_key }}
          VM_USER: ${{ steps.guest_image.outputs.vm_user }}
        run: |
          vm_ssh() {
            ssh -i "$SSH_KEY" -o BatchMode=yes -o StrictHostKeyChecking=no "$@"
          }

          kill -0 "$ERNIC_PID" 2>/dev/null || {
            echo "rocm-ernic (PID $ERNIC_PID) is no longer running"
            tail -30 /tmp/ernic-s3.log || true
            exit 1
          }
          # The store banner is the earliest proof the backend parsed its
          # options and came up on the address the guest will ARP for.
          grep -q "s3 bucket '${S3_BUCKET}'" /tmp/ernic-s3.log || {
            echo "Server log carries no s3 store banner for '${S3_BUCKET}'"
            grep -i s3 /tmp/ernic-s3.log || tail -40 /tmp/ernic-s3.log
            exit 1
          }
          grep -m1 "s3 bucket" /tmp/ernic-s3.log

          # By PCI vendor ID, never by name: the kernel registers ionic_%d,
          # 60-rdma-persistent-naming.rules rewrites it to rocep<bus>s<slot>,
          # then 99-rocm-ernic.rules rewrites it again to rocm-rdma-ernic<n>.
          DEVICE=$(vm_ssh -p "$SSH_PORT" "${VM_USER}@localhost" 'sh -s' \
            < "${GITHUB_WORKSPACE}/rocm-ernic/scripts/find-rdma-device.sh" | tr -d '\r') || true
          [ -n "$DEVICE" ] || {
            echo "RDMA device not found in the guest"
            vm_ssh -p "$SSH_PORT" "${VM_USER}@localhost" \
              'ibv_devices; lsmod | grep ionic; dmesg | grep -i ionic | tail -30' || true
            exit 1
          }
          vm_ssh -p "$SSH_PORT" "${VM_USER}@localhost" \
            "ibv_devinfo -d $DEVICE" | tee /dev/stderr | grep -q PORT_ACTIVE || {
            echo "$DEVICE never reached PORT_ACTIVE"; exit 1
          }
          echo "RDMA_DEVICE=$DEVICE" >> "$GITHUB_ENV"
          echo "$DEVICE is PORT_ACTIVE and the object store is up"

      - name: Wait for the build
        run: bash ci/ernic/background-task.sh wait build

      - name: Copy minio binary to VM
        env:
          SSH_KEY: ${{ steps.guest_image.outputs.ssh_key }}
          VM_USER: ${{ steps.guest_image.outputs.vm_user }}
        run: |
          # In a container job, ${{ github.workspace }} resolves to the runner
          # host path but inside the container it is mounted at /__w/.
          # The minio binary was built into ernic-build-s3 which is a subdir
          # of the workspace, accessible inside the container via the same
          # relative path from GITHUB_WORKSPACE.
          MINIO_BIN="${GITHUB_WORKSPACE}/ernic-build-s3/integrations/minio-cpp/minio-getput-rdma"
          ROCM_LIBS="${GITHUB_WORKSPACE}/ernic-build-s3/rocm-libs"
          scp -i "$SSH_KEY" -o BatchMode=yes -o StrictHostKeyChecking=no \
            -P "$SSH_PORT" \
            "$MINIO_BIN" \
            "${VM_USER}@localhost:/tmp/minio-getput-rdma"
          scp -i "$SSH_KEY" -o BatchMode=yes -o StrictHostKeyChecking=no \
            -P "$SSH_PORT" \
            ci/ernic/assert-transfer.sh \
            "${VM_USER}@localhost:/tmp/assert-transfer.sh"
          # The guest has no ROCm install, so the binary's HIP runtime
          # closure (~30 MB, staged by the build step) has to travel with it.
          scp -i "$SSH_KEY" -o BatchMode=yes -o StrictHostKeyChecking=no \
            -P "$SSH_PORT" -r \
            "$ROCM_LIBS" \
            "${VM_USER}@localhost:/tmp/rocm-libs"

      - name: Run S3 PUT+GET test in VM
        env:
          SSH_KEY: ${{ steps.guest_image.outputs.ssh_key }}
          VM_USER: ${{ steps.guest_image.outputs.vm_user }}
        run: |
          ssh -i "$SSH_KEY" -o BatchMode=yes -o StrictHostKeyChecking=no \
              -p "$SSH_PORT" "${VM_USER}@localhost" \
              "S3_ADDR='${S3_ADDR}' S3_PORT='${S3_PORT}' RDMA_DEVICE='${RDMA_DEVICE}' bash -s" << 'EOFVM'
          set -e
          chmod +x /tmp/minio-getput-rdma
          export LD_LIBRARY_PATH=/tmp/rocm-libs
          echo "=== Runtime linkage ==="
          ldd /tmp/minio-getput-rdma | grep -E 'not found' && {
            echo "ERROR: unresolved shared libraries in guest"; exit 1
          } || true
          echo "=== Waiting for the in-band S3 endpoint ==="
          ready=
          for i in $(seq 1 30); do
            code=$(curl -s -o /dev/null -w '%{http_code}' \
              "http://${S3_ADDR}:${S3_PORT}/" 2>/dev/null) || code=
            case "$code" in [234]*) ready=1; break;; esac
            sleep 2
          done
          [ -n "$ready" ] || {
            echo "ERROR: S3 endpoint http://${S3_ADDR}:${S3_PORT}/ never answered"
            ip -4 -br addr show; ip route
            exit 1
          }
          echo "Endpoint answered with HTTP ${code}"
          # Without a hint hipObjInit has no GPU topology to walk in this
          # GPU-less guest, so it finds no NIC and the bridge falls back to
          # plain HTTP -- which the emulator rejects for a 64 KiB body.
          export HIPOBJ_NIC_HINT="${RDMA_DEVICE}"
          echo "=== Running minio-getput-rdma (HIPOBJ_NIC_HINT=$HIPOBJ_NIC_HINT) ==="
          rc=0
          /tmp/minio-getput-rdma "${S3_ADDR}:${S3_PORT}" \
            minioadmin minioadmin 65536 nogpu > /tmp/getput.log 2>&1 || rc=$?
          cat /tmp/getput.log
          [ "$rc" -eq 0 ] || exit "$rc"
          # The bridge returns success when it silently falls back to plain
          # HTTP, so the exit code is checked and then the run's own report of
          # what it did -- payload verified, bytes over RDMA, no fallback.
          chmod +x /tmp/assert-transfer.sh
          /tmp/assert-transfer.sh /tmp/getput.log rdma
          EOFVM

      - name: Collect logs
        if: always()
        run: |
          echo "=== ernic-s3.log ===";  cat /tmp/ernic-s3.log    || true
          echo "=== qemu.log ===";      cat /tmp/qemu.log         || true
          echo "=== qemu-console ===";  cat /tmp/qemu-console.log || true

      - name: Teardown
        if: always()
        run: |
          kill "$QEMU_PID"  2>/dev/null || true
          kill "$ERNIC_PID" 2>/dev/null || true
          # Still running if the job failed before "Wait for the build".
          docker rm -f hipobj-build 2>/dev/null || true
          docker run --rm -v "${{ github.workspace }}/ernic-build-s3:/d" \
            busybox sh -c "rm -rf /d/*" 2>/dev/null || true
          rm -rf "${{ github.workspace }}/ernic-build-s3" 2>/dev/null || true