check matrix

.gitattributes

@@ -0,0 +1 @@
+ci/secrets/** filter=git-crypt diff=git-crypt

.github/workflows/build-notebooks-TEMPLATE.yaml

@@ -15,6 +15,11 @@ name: Build & Publish Notebook Servers (TEMPLATE)
         required: true
         description: "top workflow's `github`"
         type: string
+      subscription:
+        required: false
+        default: false
+        description: "add RHEL subscription from github secret"
+        type: boolean
 
 jobs:
   build:
@@ -26,6 +31,14 @@ jobs:
     steps:
 
       - uses: actions/checkout@v4
+        if: ${{ !fromJson(inputs.github).event_name == 'pull_request_target' }}
+
+      # we need to checkout the pr branch, not pr target (the default)
+      # user access check is done in calling workflow
+      - uses: actions/checkout@v4
+        if: ${{ fromJson(inputs.github).event_name == 'pull_request_target' }}
+        with:
+          ref: "refs/pull/${{ fromJson(inputs.github).event.number }}/merge"
 
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v3
@@ -43,6 +56,14 @@ jobs:
 
           df -h
 
+          sudo apt-get update
+          sudo apt-get remove -y '^dotnet-.*'
+          sudo apt-get remove -y '^llvm-.*'
+          sudo apt-get remove -y 'php.*'
+          sudo apt-get remove -y '^mongodb-.*'
+          sudo apt-get autoremove -y
+          sudo apt-get clean
+          sudo rm -rf /usr/local/.ghcup &
           sudo rm -rf /usr/local/lib/android &
           sudo rm -rf /usr/local/share/boost &
           sudo rm -rf /usr/local/lib/node_modules &
@@ -95,15 +116,71 @@ jobs:
           podman system reset --force
           mkdir -p $HOME/.local/share/containers/storage/tmp
 
+          # start systemd user service
+          # since `brew services start podman` is buggy, let's do our own brew-compatible service
+          mkdir -p "${HOME}/.config/systemd/user/"
+          cp ci/cached-builds/homebrew.podman.service "${HOME}/.config/systemd/user/homebrew.podman.service"
+          systemctl --user daemon-reload
+          systemctl --user start homebrew.podman.service
+          echo "PODMAN_SOCK=/run/user/${UID}/podman/podman.sock" >> $GITHUB_ENV
+
+      - name: Unlock encrypted secrets with git-crypt
+        if: ${{ inputs.subscription }}
+        run: |
+          sudo apt-get update
+          sudo apt-get install git-crypt
+          echo "${GIT_CRYPT_KEY}" | base64 --decode > ./git-crypt-key
+          git-crypt unlock ./git-crypt-key
+          rm ./git-crypt-key
+        env:
+          GIT_CRYPT_KEY: ${{ secrets.GIT_CRYPT_KEY }}
+
+      - name: Add subscriptions from GitHub secret
+        if: ${{ inputs.subscription }}
+        run: |
+          printf "${PWD}/ci/secrets/pki/consumer:/etc/pki/consumer\n${PWD}/ci/secrets/pki/entitlement:/etc/pki/entitlement" > /usr/share/containers/mounts.conf
+          cp ${PWD}/ci/secrets/pull-secret.txt $HOME/.config/containers/auth.json
+
       # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#push
-      - name: "push: make ${{ inputs.target }}"
+      - name: "push|schedule: make ${{ inputs.target }}"
         run: "make ${{ inputs.target }}"
-        if: "${{ fromJson(inputs.github).event_name == 'push' }}"
+        if: ${{ fromJson(inputs.github).event_name == 'push' || fromJson(inputs.github).event_name == 'schedule' }}
         env:
           IMAGE_TAG: "${{ github.ref_name }}_${{ github.sha }}"
           IMAGE_REGISTRY: "ghcr.io/${{ github.repository }}/workbench-images"
           CONTAINER_BUILD_CACHE_ARGS: "--cache-from ${{ env.CACHE }} --cache-to ${{ env.CACHE }}"
 
+      - name: "schedule: run Trivy vulnerability scanner"
+        if: "${{ fromJson(inputs.github).event_name == 'schedule' }}"
+        run: |
+          TRIVY_VERSION=0.53.0
+          REPORT_FOLDER=${{ github.workspace }}/report
+          REPORT_FILE=trivy-report.md
+          REPORT_TEMPLATE=trivy-markdown.tpl
+
+          mkdir -p $REPORT_FOLDER
+          cp ci/$REPORT_TEMPLATE $REPORT_FOLDER
+
+          IMAGE_NAME=ghcr.io/${{ github.repository }}/workbench-images:${{ inputs.target }}-${{ github.ref_name }}_${{ github.sha }}
+          echo "Scanning $IMAGE_NAME"
+
+          # have trivy access podman socket,
+          # https://github.com/aquasecurity/trivy/issues/580#issuecomment-666423279
+          podman run --rm \
+              -v ${PODMAN_SOCK}:/var/run/podman/podman.sock \
+              -v ${REPORT_FOLDER}:/report \
+              docker.io/aquasec/trivy:$TRIVY_VERSION \
+                image \
+                --image-src podman \
+                --podman-host /var/run/podman/podman.sock \
+                --scanners vuln,secret \
+                --exit-code 0 --timeout 30m \
+                --severity CRITICAL,HIGH \
+                --format template --template "@/report/$REPORT_TEMPLATE" -o /report/$REPORT_FILE \
+                $IMAGE_NAME
+
+          cat $REPORT_FOLDER/$REPORT_FILE >> $GITHUB_STEP_SUMMARY
+
       # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request
       - name: "pull_request: make ${{ inputs.target }}"
         run: |
@@ -113,7 +190,7 @@ jobs:
           go run ci/cached-builds/dev_null_container_registry.go &
           # build and push the image
           make ${{ inputs.target }}
-        if: "${{ fromJson(inputs.github).event_name == 'pull_request' }}"
+        if: "${{ fromJson(inputs.github).event_name == 'pull_request' || fromJson(inputs.github).event_name == 'pull_request_target' }}"
         env:
           IMAGE_TAG: "${{ github.sha }}"
           IMAGE_REGISTRY: "localhost:5000/workbench-images"

.github/workflows/build-notebooks-pr-rhel.yaml

@@ -0,0 +1,59 @@
+---
+"name": "Build Notebooks (pr, RHEL images)"
+"on":
+  "pull_request_target":
+    "types": ["opened", "synchronize", "reopened", "edited"]
+
+# BEWARE: This GitHub Actions workflow runs on pull_request_target, meaning it has access to our secrets
+# see https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-secrets
+# and https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
+
+permissions:
+  contents: read
+  packages: read
+
+env:
+  contributors: '["atheo89", "caponetto", "dibryant", "harshad16", "jiridanek", "jstourac", "paulovmr"]'
+
+jobs:
+  gen:
+    name: Generate job matrix
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.gen.outputs.matrix }}
+      has_jobs: ${{ steps.gen.outputs.has_jobs }}
+    steps:
+      - name: Check permissions (this must be done FIRST, for security, before we checkout)
+        if: ${{ !contains(fromJSON(env.contributors), github.actor) }}
+        run: |
+          echo "GitHub user ${{ github.actor }} is not a registered project contributor, not allowed to run actions on RHEL!"
+          exit 1
+
+      - uses: actions/checkout@v4
+        with:
+          ref: "refs/pull/${{ github.event.number }}/merge"
+
+      - name: Determine targets to build based on changed files
+        run: |
+          set -x
+          git fetch --no-tags origin 'pull/${{ github.event.pull_request.number }}/head:${{ github.event.pull_request.head.ref }}'
+          git fetch --no-tags origin '+refs/heads/${{ github.event.pull_request.base.ref }}:refs/remotes/origin/${{ github.event.pull_request.base.ref }}'
+          python3 ci/cached-builds/gen_gha_matrix_jobs.py \
+            --from-ref 'origin/${{ github.event.pull_request.base.ref }}' \
+            --to-ref '${{ github.event.pull_request.head.ref }}' \
+            --only-rhel
+        id: gen
+        shell: bash
+
+  build:
+    needs: ["gen"]
+    strategy:
+      fail-fast: false
+      matrix: "${{ fromJson(needs.gen.outputs.matrix) }}"
+    uses: ./.github/workflows/build-notebooks-TEMPLATE.yaml
+    if: ${{ fromJson(needs.gen.outputs.has_jobs) }}
+    with:
+      target: "${{ matrix.target }}"
+      github: "${{ toJSON(github) }}"
+      subscription: true
+    secrets: inherit

.github/workflows/build-notebooks-pr.yaml

@@ -1,12 +1,11 @@
 ---
-"name": "Build Notebooks"
+"name": "Build Notebooks (pr)"
 "on":
   "pull_request":
 
 permissions:
   contents: read
   packages: read
-  pull-requests: read
 
 jobs:
   gen:
@@ -18,15 +17,18 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - run: |
+      - name: Determine targets to build based on changed files
+        run: |
+          set -x
+          git fetch --no-tags origin 'pull/${{ github.event.pull_request.number }}/head:${{ github.event.pull_request.head.ref }}'
+          git fetch --no-tags origin '+refs/heads/${{ github.event.pull_request.base.ref }}:refs/remotes/origin/${{ github.event.pull_request.base.ref }}'
           python3 ci/cached-builds/gen_gha_matrix_jobs.py \
-            --owner=${{ github.repository_owner }} \
-            --repo=${{ github.event.pull_request.base.repo.name }} \
-            --pr-number=${{ github.event.pull_request.number }} \
-            --skip-unchanged
+            --from-ref 'origin/${{ github.event.pull_request.base.ref }}' \
+            --to-ref '${{ github.event.pull_request.head.ref }}'
         id: gen
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        shell: bash
 
   build:
     needs: ["gen"]

.github/workflows/build-notebooks.yaml

@@ -1,13 +1,18 @@
 ---
 # This file is autogenerated by ci/cached-builds/gen_gha_matrix_jobs.py
 {
-    "name": "Build Notebooks",
+    "name": "Build Notebooks (push)",
     "permissions": {
         "packages": "write"
     },
     "on": {
         "push": {},
-        "workflow_dispatch": {}
+        "workflow_dispatch": {},
+        "schedule": [
+            {
+                "cron": "0 2 * * *"
+            }
+        ]
     },
     "jobs": {
         "base-ubi8-python-3_8": {

.github/workflows/code-quality.yaml

@@ -19,17 +19,15 @@ jobs:
 
       - name: Check there aren't any modified files present
         run: |
-          if [[ $(git ls-files . -d -m -o --exclude-standard --full-name -v | tee modified.log | wc -l) -gt 0 ]]; then
-            echo "There are changed files"
-            exit 1
+          clean=$(git status --porcelain)
+          if [[ -z "$clean" ]]; then
+              echo "Empty git status --porcelain: $clean"
+          else
+              echo "Uncommitted file changes detected: $clean"
+              git diff
+              exit 1
           fi
 
-      - name: Print modified files
-        if: ${{ failure() }}
-        run: |
-          cat modified.log
-          git diff
-
   code-static-analysis:
     runs-on: ubuntu-latest
     steps:

.github/workflows/purge-ghcr.yaml

@@ -0,0 +1,35 @@
+---
+name: "Purge old ghcr.io test images periodically"
+
+"on":
+  workflow_dispatch:
+    inputs:
+      dry_run:
+        type: boolean
+        default: true
+        description: "Do a dry run?"
+  schedule:
+    - cron: "0 5 * * *"  # at 05:00 every day
+
+permissions:
+  packages: write
+
+jobs:
+  clean:
+    runs-on: ubuntu-latest
+    name: Delete old test images
+    steps:
+      # https://github.com/snok/container-retention-policy?tab=readme-ov-file#parameters
+      - uses: snok/container-retention-policy@4f22ef80902ad409ed55a99dc5133cc1250a0d03  # v3.0.0
+        with:
+          # account must be the gh org name when running for an org, and 'user' when running for a user
+          account: ${{ (github.repository_owner == github.actor) && 'user' || github.repository_owner }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+          image-names: |
+            ${{ github.event.repository.name }}/workbench-images
+            ${{ github.event.repository.name }}/workbench-images/build-cache
+          image-tags: "*"
+          cut-off: "3w"
+          dry-run: ${{ inputs.dry_run || false }}
+        env:
+          RUST_BACKTRACE: 1

Makefile

@@ -275,6 +275,33 @@ base-rhel9-python-3.9:
 	$(call image,$@,base/rhel9-python-3.9)
 
 ####################################### Buildchain for AMD Python 3.9 using C9S #######################################
+
+.PHONY: amd-rhel9-python-3.9
+amd-rhel9-python-3.9: base-rhel9-python-3.9
+	$(call image,$@,amd/rhel9-python-3.9,$<)
+
+# We are only using rhel9 base image here onwards,
+# DON'T be confused due to the ubi9 mention, it's just a directory name.
+.PHONY: amd-jupyter-minimal-rhel9-python-3.9
+amd-jupyter-minimal-rhel9-python-3.9: amd-rhel9-python-3.9
+	$(call image,$@,jupyter/minimal/ubi9-python-3.9,$<)
+
+# Build and push jupyter-datascience-ubi9-python-3.9 image to the registry
+.PHONY: amd-jupyter-datascience-rhel9-python-3.9
+amd-jupyter-datascience-rhel9-python-3.9: amd-jupyter-minimal-rhel9-python-3.9
+	$(call image,$@,jupyter/datascience/ubi9-python-3.9,$<)
+
+# Build and push jupyter-tensorflow-ubi9-python-3.9 image to the registry
+.PHONY: amd-jupyter-tensorflow-rhel9-python-3.9
+amd-jupyter-tensorflow-rhel9-python-3.9: amd-jupyter-datascience-rhel9-python-3.9
+	$(call image,$@,jupyter/amd/tensorflow/ubi9-python-3.9,$<)
+
+# Build and push jupyter-pytorch-ubi9-python-3.9 image to the registry
+.PHONY: amd-jupyter-pytorch-rhel9-python-3.9
+amd-jupyter-pytorch-rhel9-python-3.9: amd-jupyter-datascience-rhel9-python-3.9
+	$(call image,$@,jupyter/amd/pytorch/ubi9-python-3.9,$<)
+
+####################################### Buildchain for AMD Python 3.9 using RHEL 9 #######################################
 .PHONY: amd-c9s-python-3.9
 amd-c9s-python-3.9: base-c9s-python-3.9	
 	$(call image,$@,amd/c9s-python-3.9,$<)

amd/rhel9-python-3.9/Dockerfile

@@ -26,20 +26,9 @@ RUN echo "Installing softwares and packages" && micropipenv install && rm -f ./P
 
 USER 0
 
-# Run the subscription manager command using the provided credentials. Only include --serverurl and --baseurl if they are provided
-RUN SERVERURL=$(cat ${SECRET_DIR}/SERVERURL 2>/dev/null || echo ${SERVERURL_DEFAULT}) && \
-    BASEURL=$(cat ${SECRET_DIR}/BASEURL 2>/dev/null || echo ${BASEURL_DEFAULT}) && \
-    USERNAME=$(cat ${SECRET_DIR}/USERNAME) && \
-    PASSWORD=$(cat ${SECRET_DIR}/PASSWORD) && \
-    subscription-manager register \
-    ${SERVERURL:+--serverurl=$SERVERURL} \
-    ${BASEURL:+--baseurl=$BASEURL} \
-    --username=$USERNAME \
-    --password=$PASSWORD \
-    --force \
-    --auto-attach
-
-# Install required packages 
+# change!!!!
+
+# Install required packages
 RUN yum -y install git java-1.8.0-openjdk && \
     yum clean all && rm -rf /var/cache/yum