RHOAIENG-16076: tests(gha): change to using rootful podman, so that w…

…e can share containers/storage with cri-o later (#782)

.github/workflows/build-notebooks-TEMPLATE.yaml

@@ -20,6 +20,12 @@ jobs:
   build:
     runs-on: ubuntu-22.04
     env:
+      # Some pieces of code (image pulls for example) in podman consult TMPDIR or default to /var/tmp
+      TMPDIR: /home/runner/.local/share/containers/tmpdir
+      # Use the rootful instance of podman for sharing images with cri-o
+      # https://podman-desktop.io/blog/sharing-podman-images-with-kubernetes-cluster#introduction
+      # https://access.redhat.com/solutions/6986565
+      CONTAINER_HOST: unix:///var/run/podman/podman.sock
       # We don't push here when building PRs, so we can use the same IMAGE_REGISTRY in all branches of the workflow
       IMAGE_REGISTRY: "ghcr.io/${{ github.repository }}/workbench-images"
       # GitHub image registry used for storing $(CONTAINER_ENGINE)'s cache
@@ -101,22 +107,45 @@ jobs:
 
       - name: Configure Podman
         run: |
-          set -x
-          mkdir -p $HOME/.config/containers/
-          cp ci/cached-builds/containers.conf $HOME/.config/containers/containers.conf
-          cp ci/cached-builds/storage.conf $HOME/.config/containers/storage.conf
-
-          # should at least reset storage when touching storage.conf
-          podman system reset --force
+          set -Eeuxo pipefail
+
+          # podman running as service ignores the TMPDIR env var here, let's give it a bind-mount to /var/tmp
+          mkdir -p $TMPDIR
+          sudo mount --bind -o rw,noexec,nosuid,nodev,bind $TMPDIR /var/tmp
+
+          # podman from brew has its own /etc (was giving me Failed to obtain podman configuration: runroot must be set)
+          # the (default) config location is also where cri-o gets its storage defaults (that can be overriden in crio.conf)
+          sudo cp ci/cached-builds/containers.conf /etc/containers.conf
+          sudo cp ci/cached-builds/containers.conf /home/linuxbrew/.linuxbrew/opt/podman/etc/containers.conf
+          sudo cp ci/cached-builds/storage.conf /etc/containers/storage.conf
+          sudo cp ci/cached-builds/storage.conf /home/linuxbrew/.linuxbrew/opt/podman/etc/containers/storage.conf
+          sudo cp ci/cached-builds/registries.conf /etc/containers/registries.conf
+          sudo cp ci/cached-builds/registries.conf /home/linuxbrew/.linuxbrew/opt/podman/etc/containers/registries.conf
+
+          # should reset storage when changing storage.conf
           mkdir -p $HOME/.local/share/containers/storage/tmp
+          # remote (CONTAINER_HOST) podman does not do reset (and refuses --force option)
+          sudo /home/linuxbrew/.linuxbrew/opt/podman/bin/podman system reset --force
 
-          # start systemd user service
+          # https://github.com/containers/podman/blob/main/docs/tutorials/socket_activation.md
           # since `brew services start podman` is buggy, let's do our own brew-compatible service
-          mkdir -p "${HOME}/.config/systemd/user/"
-          cp ci/cached-builds/homebrew.podman.service "${HOME}/.config/systemd/user/homebrew.podman.service"
-          systemctl --user daemon-reload
-          systemctl --user start homebrew.podman.service
-          echo "PODMAN_SOCK=/run/user/${UID}/podman/podman.sock" >> $GITHUB_ENV
+          # Regarding directory paths, see https://unix.stackexchange.com/questions/224992/where-do-i-put-my-systemd-unit-file
+          sudo mkdir -p /usr/local/lib/systemd/system/
+          sudo cp ci/cached-builds/podman.service /usr/local/lib/systemd/system/podman.service
+          sudo cp ci/cached-builds/podman.socket /usr/local/lib/systemd/system/podman.socket
+          sudo systemctl daemon-reload
+          sudo systemctl unmask --now podman.service podman.socket
+          sudo systemctl start podman.socket
+
+          # needed (much) later for trivy
+          echo "PODMAN_SOCK=/var/run/podman/podman.sock" >> $GITHUB_ENV
+
+          # quick check podman works
+          podman ps
+
+      - name: Show error logs (on failure)
+        if: ${{ failure() }}
+        run: journalctl -xe
 
       - name: Calculate image name and tag
         id: calculated_vars

ci/cached-builds/containers.conf

@@ -8,8 +8,13 @@ retry=100
 # supposedly these images are faster to pull
 compression_format="zstd:chunked"
 compression_level=6
+
 # defaults to /var/tmp, which is small
 image_copy_tmp_dir="storage"
+# setting image_copy_tmp_dir is not enough, it still can give me
+# Error: creating build container: writing blob: storing blob to file "/var/tmp/container_images_storage2384030476/20": write /var/tmp/container_images_storage2384030476/20: no space left on device
+# https://github.com/containers/podman/issues/5411, https://github.com/containers/podman/pull/5412
+# Set the TMPDIR env variable, https://github.com/containers/podman/blob/d85ac938e60938369ff1337dccaf0943b7405f48/cmd/podman/images/load.go#L96
 
 [machine]
 

ci/cached-builds/podman.service

@@ -0,0 +1,20 @@
+# https://docs.podman.io/en/latest/markdown/podman-system-service.1.html
+# cat /usr/lib/systemd/system/podman.socket
+
+[Unit]
+Description=Podman API Service
+Requires=podman.socket
+After=podman.socket
+Documentation=man:podman-system-service(1)
+StartLimitIntervalSec=0
+
+[Service]
+Delegate=true
+Type=exec
+KillMode=process
+Environment="PATH=/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin:/usr/bin:/bin:/usr/sbin:/sbin"
+WorkingDirectory=/home/linuxbrew/.linuxbrew
+ExecStart=/home/linuxbrew/.linuxbrew/opt/podman/bin/podman --log-level=info system service
+
+[Install]
+WantedBy=default.target

ci/cached-builds/podman.socket

@@ -0,0 +1,12 @@
+# cat /usr/lib/systemd/system/podman.socket
+
+[Unit]
+Description=Podman API Socket
+Documentation=man:podman-system-service(1)
+
+[Socket]
+ListenStream=%t/podman/podman.sock
+SocketMode=0666
+
+[Install]
+WantedBy=sockets.target

ci/cached-builds/registries.conf

@@ -0,0 +1,6 @@
+# https://github.com/containers/image/blob/main/docs/containers-registries.conf.5.md
+
+# prevent this kyverno error
+#  Failed to pull image "bitnami/kubectl:1.26.4": reading manifest 1.26.4 in quay.io/bitnami/kubectl: unauthorized: access to the requested resource is not authorized
+unqualified-search-registries = [ "docker.io" ]
+short-name-mode = "enforcing"

ci/cached-builds/storage.conf

@@ -1,7 +1,15 @@
 # https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md
 
+# Multiple users sharing the same containers/storage is not "supported" as it tends to cause
+# various permission issues on the host or wrong uid/gids in the containers. C.f.
+#  https://access.redhat.com/solutions/6986565
 [storage]
-driver="overlay"
+driver = "overlay"
+
+graphroot = "/home/runner/.local/share/containers/storage"
+runroot = "/home/runner/.local/share/containers/storage"
+
+transient_store = true
 
 [storage.options]
 # https://www.redhat.com/sysadmin/faster-container-image-pulls