From 7650cd8b2f823e7e1d91e403c6df3231da8e5274 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiri=20Dan=C4=9Bk?= Date: Tue, 26 Nov 2024 11:05:50 +0100 Subject: [PATCH] RHOAIENG-16076: tests(gha): change to using rootful podman, so that we can share containers/storage with cri-o later (#782) --- .../workflows/build-notebooks-TEMPLATE.yaml | 55 ++++++++++++++----- ci/cached-builds/containers.conf | 5 ++ ci/cached-builds/homebrew.podman.service | 11 ---- ci/cached-builds/podman.service | 20 +++++++ ci/cached-builds/podman.socket | 12 ++++ ci/cached-builds/registries.conf | 6 ++ ci/cached-builds/storage.conf | 10 +++- 7 files changed, 94 insertions(+), 25 deletions(-) delete mode 100644 ci/cached-builds/homebrew.podman.service create mode 100644 ci/cached-builds/podman.service create mode 100644 ci/cached-builds/podman.socket create mode 100644 ci/cached-builds/registries.conf diff --git a/.github/workflows/build-notebooks-TEMPLATE.yaml b/.github/workflows/build-notebooks-TEMPLATE.yaml index ee1483fa5..afef1bf6d 100644 --- a/.github/workflows/build-notebooks-TEMPLATE.yaml +++ b/.github/workflows/build-notebooks-TEMPLATE.yaml @@ -20,6 +20,12 @@ jobs: build: runs-on: ubuntu-22.04 env: + # Some pieces of code (image pulls for example) in podman consult TMPDIR or default to /var/tmp + TMPDIR: /home/runner/.local/share/containers/tmpdir + # Use the rootful instance of podman for sharing images with cri-o + # https://podman-desktop.io/blog/sharing-podman-images-with-kubernetes-cluster#introduction + # https://access.redhat.com/solutions/6986565 + CONTAINER_HOST: unix:///var/run/podman/podman.sock # We don't push here when building PRs, so we can use the same IMAGE_REGISTRY in all branches of the workflow IMAGE_REGISTRY: "ghcr.io/${{ github.repository }}/workbench-images" # GitHub image registry used for storing $(CONTAINER_ENGINE)'s cache @@ -101,22 +107,45 @@ jobs: - name: Configure Podman run: | - set -x - mkdir -p $HOME/.config/containers/ - cp ci/cached-builds/containers.conf $HOME/.config/containers/containers.conf - cp ci/cached-builds/storage.conf $HOME/.config/containers/storage.conf - - # should at least reset storage when touching storage.conf - podman system reset --force + set -Eeuxo pipefail + + # podman running as service ignores the TMPDIR env var here, let's give it a bind-mount to /var/tmp + mkdir -p $TMPDIR + sudo mount --bind -o rw,noexec,nosuid,nodev,bind $TMPDIR /var/tmp + + # podman from brew has its own /etc (was giving me Failed to obtain podman configuration: runroot must be set) + # the (default) config location is also where cri-o gets its storage defaults (that can be overriden in crio.conf) + sudo cp ci/cached-builds/containers.conf /etc/containers.conf + sudo cp ci/cached-builds/containers.conf /home/linuxbrew/.linuxbrew/opt/podman/etc/containers.conf + sudo cp ci/cached-builds/storage.conf /etc/containers/storage.conf + sudo cp ci/cached-builds/storage.conf /home/linuxbrew/.linuxbrew/opt/podman/etc/containers/storage.conf + sudo cp ci/cached-builds/registries.conf /etc/containers/registries.conf + sudo cp ci/cached-builds/registries.conf /home/linuxbrew/.linuxbrew/opt/podman/etc/containers/registries.conf + + # should reset storage when changing storage.conf mkdir -p $HOME/.local/share/containers/storage/tmp + # remote (CONTAINER_HOST) podman does not do reset (and refuses --force option) + sudo /home/linuxbrew/.linuxbrew/opt/podman/bin/podman system reset --force - # start systemd user service + # https://github.com/containers/podman/blob/main/docs/tutorials/socket_activation.md # since `brew services start podman` is buggy, let's do our own brew-compatible service - mkdir -p "${HOME}/.config/systemd/user/" - cp ci/cached-builds/homebrew.podman.service "${HOME}/.config/systemd/user/homebrew.podman.service" - systemctl --user daemon-reload - systemctl --user start homebrew.podman.service - echo "PODMAN_SOCK=/run/user/${UID}/podman/podman.sock" >> $GITHUB_ENV + # Regarding directory paths, see https://unix.stackexchange.com/questions/224992/where-do-i-put-my-systemd-unit-file + sudo mkdir -p /usr/local/lib/systemd/system/ + sudo cp ci/cached-builds/podman.service /usr/local/lib/systemd/system/podman.service + sudo cp ci/cached-builds/podman.socket /usr/local/lib/systemd/system/podman.socket + sudo systemctl daemon-reload + sudo systemctl unmask --now podman.service podman.socket + sudo systemctl start podman.socket + + # needed (much) later for trivy + echo "PODMAN_SOCK=/var/run/podman/podman.sock" >> $GITHUB_ENV + + # quick check podman works + podman ps + + - name: Show error logs (on failure) + if: ${{ failure() }} + run: journalctl -xe - name: Calculate image name and tag id: calculated_vars diff --git a/ci/cached-builds/containers.conf b/ci/cached-builds/containers.conf index 6f9a8c43e..d1dad0278 100644 --- a/ci/cached-builds/containers.conf +++ b/ci/cached-builds/containers.conf @@ -8,8 +8,13 @@ retry=100 # supposedly these images are faster to pull compression_format="zstd:chunked" compression_level=6 + # defaults to /var/tmp, which is small image_copy_tmp_dir="storage" +# setting image_copy_tmp_dir is not enough, it still can give me +# Error: creating build container: writing blob: storing blob to file "/var/tmp/container_images_storage2384030476/20": write /var/tmp/container_images_storage2384030476/20: no space left on device +# https://github.com/containers/podman/issues/5411, https://github.com/containers/podman/pull/5412 +# Set the TMPDIR env variable, https://github.com/containers/podman/blob/d85ac938e60938369ff1337dccaf0943b7405f48/cmd/podman/images/load.go#L96 [machine] diff --git a/ci/cached-builds/homebrew.podman.service b/ci/cached-builds/homebrew.podman.service deleted file mode 100644 index b8ef961a6..000000000 --- a/ci/cached-builds/homebrew.podman.service +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=Homebrew generated unit for podman - -[Install] -WantedBy=default.target - -[Service] -Type=simple -ExecStart=/home/linuxbrew/.linuxbrew/opt/podman/bin/podman system service --time=0 -WorkingDirectory=/home/linuxbrew/.linuxbrew -Environment="PATH=/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin:/usr/bin:/bin:/usr/sbin:/sbin" diff --git a/ci/cached-builds/podman.service b/ci/cached-builds/podman.service new file mode 100644 index 000000000..e0b261412 --- /dev/null +++ b/ci/cached-builds/podman.service @@ -0,0 +1,20 @@ +# https://docs.podman.io/en/latest/markdown/podman-system-service.1.html +# cat /usr/lib/systemd/system/podman.socket + +[Unit] +Description=Podman API Service +Requires=podman.socket +After=podman.socket +Documentation=man:podman-system-service(1) +StartLimitIntervalSec=0 + +[Service] +Delegate=true +Type=exec +KillMode=process +Environment="PATH=/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin:/usr/bin:/bin:/usr/sbin:/sbin" +WorkingDirectory=/home/linuxbrew/.linuxbrew +ExecStart=/home/linuxbrew/.linuxbrew/opt/podman/bin/podman --log-level=info system service + +[Install] +WantedBy=default.target diff --git a/ci/cached-builds/podman.socket b/ci/cached-builds/podman.socket new file mode 100644 index 000000000..7e0081213 --- /dev/null +++ b/ci/cached-builds/podman.socket @@ -0,0 +1,12 @@ +# cat /usr/lib/systemd/system/podman.socket + +[Unit] +Description=Podman API Socket +Documentation=man:podman-system-service(1) + +[Socket] +ListenStream=%t/podman/podman.sock +SocketMode=0666 + +[Install] +WantedBy=sockets.target diff --git a/ci/cached-builds/registries.conf b/ci/cached-builds/registries.conf new file mode 100644 index 000000000..6925b8309 --- /dev/null +++ b/ci/cached-builds/registries.conf @@ -0,0 +1,6 @@ +# https://github.com/containers/image/blob/main/docs/containers-registries.conf.5.md + +# prevent this kyverno error +# Failed to pull image "bitnami/kubectl:1.26.4": reading manifest 1.26.4 in quay.io/bitnami/kubectl: unauthorized: access to the requested resource is not authorized +unqualified-search-registries = [ "docker.io" ] +short-name-mode = "enforcing" diff --git a/ci/cached-builds/storage.conf b/ci/cached-builds/storage.conf index eb9bba42a..a3d94f402 100644 --- a/ci/cached-builds/storage.conf +++ b/ci/cached-builds/storage.conf @@ -1,7 +1,15 @@ # https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md +# Multiple users sharing the same containers/storage is not "supported" as it tends to cause +# various permission issues on the host or wrong uid/gids in the containers. C.f. +# https://access.redhat.com/solutions/6986565 [storage] -driver="overlay" +driver = "overlay" + +graphroot = "/home/runner/.local/share/containers/storage" +runroot = "/home/runner/.local/share/containers/storage" + +transient_store = true [storage.options] # https://www.redhat.com/sysadmin/faster-container-image-pulls