Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[GHA] digest updater workflow fixes and updates #566

Merged
merged 3 commits into from
Jun 26, 2024
Merged

[GHA] digest updater workflow fixes and updates #566

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
bc0a54f
[GHA] fix the regexp used for the latest image tag selection
jstourac Jun 15, 2024
e7db0aa
[GHA] fix the missing habana image update record for the digest updater
jstourac Jun 15, 2024
0c6aecc
[GHA] Minor refactoring
jstourac Jun 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
173 changes: 91 additions & 82 deletions .github/workflows/notebooks-digest-updater-upstream.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,57 +67,61 @@ jobs:

- name: Update the param.env file
run: |
echo Latest commit is: ${{ steps.hash-n.outputs.HASH_N }} on ${{ env.RELEASE_VERSION_N}}
IMAGES=("odh-minimal-notebook-image-n"
"odh-minimal-gpu-notebook-image-n"
"odh-pytorch-gpu-notebook-image-n"
"odh-generic-data-science-notebook-image-n"
"odh-tensorflow-gpu-notebook-image-n"
"odh-trustyai-notebook-image-n"
"odh-codeserver-notebook-image-n"
"odh-rstudio-notebook-image-n"
"odh-rstudio-gpu-notebook-image-n")

for ((i=0;i<${#IMAGES[@]};++i)); do
image=${IMAGES[$i]}
echo "CHECKING: " $image
img=$(cat manifests/base/params.env | grep -E "${image}=" | cut -d '=' -f2)
registry=$(echo $img | cut -d '@' -f1)
src_tag=$(skopeo inspect docker://$img | jq '.Env[] | select(startswith("OPENSHIFT_BUILD_NAME=")) | split("=")[1]' | tr -d '"' | sed 's/-amd64$//')
regex="$src_tag-${{ env.RELEASE_VERSION_N}}-\d+-${{ steps.hash-n.outputs.HASH_N }}"
latest_tag=$(skopeo inspect docker://$img | jq -r --arg regex "$regex" '.RepoTags | map(select(. | test($regex))) | .[0]')
digest=$(skopeo inspect docker://$registry:$latest_tag | jq .Digest | tr -d '"')
output=$registry@$digest
echo "NEW:" $output
sed -i "s|${image}=.*|${image}=$output|" manifests/base/params.env
PARAMS_ENV_PATH="manifests/base/params.env"

echo Latest commit is: ${{ steps.hash-n.outputs.HASH_N }} on ${{ env.RELEASE_VERSION_N }}

# Get the complete list of images N-version to update
IMAGES=$(cat "${PARAMS_ENV_PATH}" | grep "\-n=" | cut -d "=" -f 1)

for image in ${IMAGES}; do
echo "CHECKING: '${image}'"
img=$(grep -E "${image}=" "${PARAMS_ENV_PATH}" | cut -d '=' -f2)
registry=$(echo "${img}" | cut -d '@' -f1)

skopeo_metadata=$(skopeo inspect --retry-times 3 "docker://${img}")

src_tag=$(echo "${skopeo_metadata}" | jq '.Env[] | select(startswith("OPENSHIFT_BUILD_NAME=")) | split("=")[1]' | tr -d '"' | sed 's/-amd64$//')
regex="^$src_tag-${{ env.RELEASE_VERSION_N}}-\d+-${{ steps.hash-n.outputs.HASH_N }}\$"
latest_tag=$(echo "${skopeo_metadata}" | jq -r --arg regex "$regex" '.RepoTags | map(select(. | test($regex))) | .[0]')
# use `--no-tags` for skopeo once available in newer version
digest=$(skopeo inspect --retry-times 3 "docker://${registry}:${latest_tag}" | jq .Digest | tr -d '"')
output="${registry}@${digest}"
echo "NEW: ${output}"
sed -i "s|${image}=.*|${image}=${output}|" "${PARAMS_ENV_PATH}"
done

if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then
git fetch origin ${{ env.DIGEST_UPDATER_BRANCH }} && git pull origin ${{ env.DIGEST_UPDATER_BRANCH }} && git add manifests/base/params.env && git commit -m "Update images for release N via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && git push origin ${{ env.DIGEST_UPDATER_BRANCH }}
git fetch origin ${{ env.DIGEST_UPDATER_BRANCH }} && \
git pull origin ${{ env.DIGEST_UPDATER_BRANCH }} && \
git add "${PARAMS_ENV_PATH}" && \
git commit -m "Update images for release N via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && \
git push origin ${{ env.DIGEST_UPDATER_BRANCH }}
else
echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N}}"
echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N }}"
fi

- name: Update the commit.env file
run: |
echo Latest commit is: ${{ steps.hash-n.outputs.HASH_N }} on ${{ env.RELEASE_VERSION_N}}
COMMIT=("odh-minimal-notebook-image-commit-n"
"odh-minimal-gpu-notebook-image-commit-n"
"odh-pytorch-gpu-notebook-image-commit-n"
"odh-generic-data-science-notebook-image-commit-n"
"odh-tensorflow-gpu-notebook-image-commit-n"
"odh-trustyai-notebook-image-commit-n"
"odh-codeserver-notebook-image-commit-n"
"odh-rstudio-notebook-image-commit-n"
"odh-rstudio-gpu-notebook-image-commit-n")

for val in "${COMMIT[@]}"; do
echo $val
sed -i "s|${val}=.*|${val}=${{ steps.hash-n.outputs.HASH_N }}|" manifests/base/commit.env
COMMIT_ENV_PATH="manifests/base/commit.env"

echo Latest commit is: ${{ steps.hash-n.outputs.HASH_N }} on ${{ env.RELEASE_VERSION_N }}
# Get the complete list of images N-1-version to update
COMMIT=$(grep "\-n=" "${COMMIT_ENV_PATH}" | cut -d "=" -f 1)

for val in ${COMMIT}; do
echo "${val}"
sed -i "s|${val}=.*|${val}=${{ steps.hash-n.outputs.HASH_N }}|" "${COMMIT_ENV_PATH}"
done

if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then
git fetch origin ${{ env.DIGEST_UPDATER_BRANCH }} && git pull origin ${{ env.DIGEST_UPDATER_BRANCH }} && git add manifests/base/commit.env && git commit -m "Update image commits for release N via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && git push origin ${{ env.DIGEST_UPDATER_BRANCH }}
git fetch origin ${{ env.DIGEST_UPDATER_BRANCH }} && \
git pull origin ${{ env.DIGEST_UPDATER_BRANCH }} && \
git add "${COMMIT_ENV_PATH}" && \
git commit -m "Update image commits for release N via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && \
git push origin ${{ env.DIGEST_UPDATER_BRANCH }}
else
echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N}}"
echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N }}"
fi

update-n-1-version:
Expand Down Expand Up @@ -147,56 +151,61 @@ jobs:

- name: Update the param.env file
run: |
echo Latest commit is: ${{ steps.hash-n-1.outputs.HASH_N_1 }} on ${{ env.RELEASE_VERSION_N_1}}
IMAGES=("odh-minimal-notebook-image-n-1"
"odh-minimal-gpu-notebook-image-n-1"
"odh-pytorch-gpu-notebook-image-n-1"
"odh-generic-data-science-notebook-image-n-1"
"odh-tensorflow-gpu-notebook-image-n-1"
"odh-trustyai-notebook-image-n-1"
"odh-codeserver-notebook-image-n-1"
"odh-rstudio-notebook-image-n-1"
"odh-rstudio-gpu-notebook-image-n-1")

for ((i=0;i<${#IMAGES[@]};++i)); do
image=${IMAGES[$i]}
echo "CHECKING: " $image
img=$(cat manifests/base/params.env | grep -E "${image}=" | cut -d '=' -f2)
registry=$(echo $img | cut -d '@' -f1)
src_tag=$(skopeo inspect docker://$img | jq '.Env[] | select(startswith("OPENSHIFT_BUILD_NAME=")) | split("=")[1]' | tr -d '"' | sed 's/-amd64$//')
regex="$src_tag-${{ env.RELEASE_VERSION_N_1}}-\d+-${{ steps.hash-n-1.outputs.HASH_N_1 }}"
latest_tag=$(skopeo inspect docker://$img | jq -r --arg regex "$regex" '.RepoTags | map(select(. | test($regex))) | .[0]')
digest=$(skopeo inspect docker://$registry:$latest_tag | jq .Digest | tr -d '"')
output=$registry@$digest
echo "NEW:" $output
sed -i "s|${image}=.*|${image}=$output|" manifests/base/params.env
PARAMS_ENV_PATH="manifests/base/params.env"

echo Latest commit is: ${{ steps.hash-n-1.outputs.HASH_N_1 }} on ${{ env.RELEASE_VERSION_N_1 }}

# Get the complete list of images N-1-version to update
IMAGES=$(cat "${PARAMS_ENV_PATH}" | grep "\-n-1=" | cut -d "=" -f 1)

for image in ${IMAGES}; do
echo "CHECKING: '${image}'"
img=$(grep -E "${image}=" "${PARAMS_ENV_PATH}" | cut -d '=' -f2)
registry=$(echo "${img}" | cut -d '@' -f1)

skopeo_metadata=$(skopeo inspect --retry-times 3 "docker://${img}")

src_tag=$(echo "${skopeo_metadata}" | jq '.Env[] | select(startswith("OPENSHIFT_BUILD_NAME=")) | split("=")[1]' | tr -d '"' | sed 's/-amd64$//')
regex="^$src_tag-${{ env.RELEASE_VERSION_N_1}}-\d+-${{ steps.hash-n-1.outputs.HASH_N_1 }}\$"
latest_tag=$(echo "${skopeo_metadata}" | jq -r --arg regex "$regex" '.RepoTags | map(select(. | test($regex))) | .[0]')
# use `--no-tags` for skopeo once available in newer version
digest=$(skopeo inspect --retry-times 3 "docker://${registry}:${latest_tag}" | jq .Digest | tr -d '"')
output="${registry}@${digest}"
echo "NEW: ${output}"
sed -i "s|${image}=.*|${image}=${output}|" "${PARAMS_ENV_PATH}"
done

if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then
git fetch origin ${{ env.DIGEST_UPDATER_BRANCH }} && git pull origin ${{ env.DIGEST_UPDATER_BRANCH }} && git add manifests/base/params.env && git commit -m "Update images for release N-1 via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && git push origin ${{ env.DIGEST_UPDATER_BRANCH }}
git fetch origin ${{ env.DIGEST_UPDATER_BRANCH }} && \
git pull origin ${{ env.DIGEST_UPDATER_BRANCH }} && \
git add "${PARAMS_ENV_PATH}" && \
git commit -m "Update images for release N-1 via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && \
git push origin ${{ env.DIGEST_UPDATER_BRANCH }}
else
echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N_1}}"
echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N_1 }}"
fi

- name: Update the commit.env file
run: |
echo Latest commit is: ${{ steps.hash-n-1.outputs.HASH_N_1 }} on ${{ env.RELEASE_VERSION_N_1}}
COMMIT=("odh-minimal-notebook-image-commit-n-1"
"odh-minimal-gpu-notebook-image-commit-n-1"
"odh-pytorch-gpu-notebook-image-commit-n-1"
"odh-generic-data-science-notebook-image-commit-n-1"
"odh-tensorflow-gpu-notebook-image-commit-n-1"
"odh-trustyai-notebook-image-commit-n-1"
"odh-codeserver-notebook-image-commit-n-1"
"odh-rstudio-notebook-image-commit-n-1"
"odh-rstudio-gpu-notebook-image-commit-n-1")

for val in "${COMMIT[@]}"; do
echo $val
sed -i "s|${val}=.*|${val}=${{ steps.hash-n-1.outputs.HASH_N_1 }}|" manifests/base/commit.env
COMMIT_ENV_PATH="manifests/base/commit.env"

echo Latest commit is: ${{ steps.hash-n-1.outputs.HASH_N_1 }} on ${{ env.RELEASE_VERSION_N_1 }}
# Get the complete list of images N-1-version to update
COMMIT=$(grep "\-n-1=" "${COMMIT_ENV_PATH}" | cut -d "=" -f 1)

for val in ${COMMIT}; do
echo "${val}"
sed -i "s|${val}=.*|${val}=${{ steps.hash-n-1.outputs.HASH_N_1 }}|" "${COMMIT_ENV_PATH}"
done

if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then
git fetch origin ${{ env.DIGEST_UPDATER_BRANCH }} && git pull origin ${{ env.DIGEST_UPDATER_BRANCH }} && git add manifests/base/commit.env && git commit -m "Update image commits for release N via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && git push origin ${{ env.DIGEST_UPDATER_BRANCH }}
git fetch origin ${{ env.DIGEST_UPDATER_BRANCH }} && \
git pull origin ${{ env.DIGEST_UPDATER_BRANCH }} && \
git add "${COMMIT_ENV_PATH}" && \
git commit -m "Update image commits for release N via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && \
git push origin ${{ env.DIGEST_UPDATER_BRANCH }}
else
echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N}}"
echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N }}"
fi

open-pull-request:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/runtimes-digest-updater-upstream.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ jobs:
name="minimal-$name"
fi
registry=$(echo $img | cut -d '@' -f1)
regex="runtime-$name-$py_version-${{ env.RELEASE_VERSION_N}}-\d+-${{ steps.hash-n.outputs.HASH_N }}"
regex="^runtime-$name-$py_version-${{ env.RELEASE_VERSION_N}}-\d+-${{ steps.hash-n.outputs.HASH_N }}\$"
echo "CHECKING: " $regex
latest_tag=$(skopeo inspect docker://$img | jq -r --arg regex "$regex" '.RepoTags | map(select(. | test($regex))) | .[0]')
digest=$(skopeo inspect docker://$registry:$latest_tag | jq .Digest | tr -d '"')
Expand Down
4 changes: 2 additions & 2 deletions ci/security-scan/quay_security_analysis.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,9 +81,9 @@ def process_image(image, commit_id_path, RELEASE_VERSION_N, HASH_N):
regex = ""

if RELEASE_VERSION_N == "":
regex = f"{src_tag}-(\\d+-)?{HASH_N}"
regex = f"^{src_tag}-(\\d+-)?{HASH_N}$"
else:
regex = f"{src_tag}-{RELEASE_VERSION_N}-\\d+-{HASH_N}"
regex = f"^{src_tag}-{RELEASE_VERSION_N}-\\d+-{HASH_N}$"

latest_tag_cmd = f'skopeo inspect docker://{img} | jq -r --arg regex "{regex}" \'.RepoTags | map(select(. | test($regex))) | .[0]\''
latest_tag = subprocess.check_output(latest_tag_cmd, shell=True, text=True).strip()
Expand Down