From 096cb4bafce91ade4ca0dc0cc39712d74d36c1e8 Mon Sep 17 00:00:00 2001 From: Jiaxun Yang Date: Wed, 7 May 2025 13:12:48 +0100 Subject: [PATCH 1/2] Rework GitHub Release and Docker Actions The universal goal is to make those actions available in forks, remove some cumbersome scripts and unnecessary steps, enable arm64 build and upload to ghcr.io as an alternative container registry. Signed-off-by: Jiaxun Yang --- .github/workflows/docker.yml | 65 ++++++++--- .github/workflows/dockerhub-description.yml | 22 ++++ .github/workflows/release.yml | 28 +---- dev/docker.sh | 96 --------------- dev/dockerhub_readme.py | 122 -------------------- dev/get_upload_url.sh | 14 --- dev/ref2tag.sh | 4 - dev/release.sh | 53 --------- 8 files changed, 76 insertions(+), 328 deletions(-) create mode 100644 .github/workflows/dockerhub-description.yml delete mode 100755 dev/docker.sh delete mode 100755 dev/dockerhub_readme.py delete mode 100755 dev/get_upload_url.sh delete mode 100755 dev/ref2tag.sh delete mode 100755 dev/release.sh diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 47147d982a4..1750a36c0dc 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -9,6 +9,11 @@ on: jobs: ubuntu: runs-on: ubuntu-latest + permissions: + packages: write + contents: read + attestations: write + id-token: write steps: - name: Print environment shell: bash @@ -18,6 +23,8 @@ jobs: - uses: actions/setup-python@v5 with: python-version: '3.10' + cache: 'pip' + - name: Install dependencies run: python3 -m pip install -r docker/requirements.txt - name: Install opengrok-tools so that pylint can perform the checks @@ -33,18 +40,46 @@ jobs: run: black --check docker/*.py - name: Run isort in check mode run: isort --settings-file docker/.isort.cfg docker/*.py --check --diff - - name: Build and optionally push Docker image - env: - DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} - DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} - OPENGROK_REPO_SLUG: ${{ github.repository }} - OPENGROK_REF: ${{ github.ref }} - run: ./dev/docker.sh - - name: Install Python pre-requisites - run: python3 -m pip install requests - - name: Optionally update README on Docker hub - env: - OPENGROK_REPO_SLUG: ${{ github.repository }} - DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} - DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} - run: ./dev/dockerhub_readme.py + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Log in to Docker Hub + uses: docker/login-action@v3 + if: ${{ vars.DOCKER_SLUG != '' }} && ${{ github.event_name != 'pull_request' }} + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: | + ${{ vars.DOCKER_SLUG }} + ghcr.io/${{ github.repository }} + labels: | + maintainer="https://github.com/oracle/opengrok" + org.opencontainers.image.description="OpenGrok Code Search" + tags: | + type=raw,value=latest,enable=${{ github.event_name == 'release' }} + type=raw,value=master,enable=${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + type=pep440,pattern={{version}},enable=${{ github.event_name == 'release' }} + type=pep440,pattern={{major}}.{{minor}},enable=${{ github.event_name == 'release' }} + type=ref,event=branch + type=ref,event=pr + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build and push Docker image + id: push + uses: docker/build-push-action@v6 + with: + context: . + file: ./Dockerfile + platforms: linux/amd64,linux/arm64 + push: ${{ github.event_name == 'release' || (github.event_name == 'push' && github.ref == 'refs/heads/master') }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/dockerhub-description.yml b/.github/workflows/dockerhub-description.yml new file mode 100644 index 00000000000..485ccdbb2fd --- /dev/null +++ b/.github/workflows/dockerhub-description.yml @@ -0,0 +1,22 @@ +name: Update Docker Hub Description +on: + push: + branches: + - master + paths: + - docker/README.md + - .github/workflows/dockerhub-description.yml +jobs: + dockerHubDescription: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Docker Hub Description + if: ${{ vars.DOCKER_SLUG != '' }} + uses: peter-evans/dockerhub-description@v4 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + repository: ${{ vars.DOCKER_SLUG }} + readme-filepath: ./docker/README.md diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7c16a03c020..c4c24053888 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,27 +1,12 @@ name: Release -# TODO: run this only for the oracle/opengrok repository on: release: types: [created] jobs: - get_tag: - name: Get tag name - outputs: - tag: ${{ steps.get_tag.outputs.tag }} - runs-on: ubuntu-latest - steps: - - name: Checkout master branch - uses: actions/checkout@v4 - - name: Get the tag name - id: get_tag - env: - OPENGROK_REF: ${{ github.ref }} - run: ./dev/ref2tag.sh build: runs-on: ubuntu-latest - needs: get_tag steps: - name: Checkout master branch uses: actions/checkout@v4 @@ -47,18 +32,13 @@ jobs: run: ./dev/before - name: Build run: ./mvnw -DskipTests=true -Dmaven.javadoc.skip=false -B -V package - - name: Get upload URL - id: get_upload_url - env: - OPENGROK_TAG: ${{ needs.get_tag.outputs.tag }} - run: dev/get_upload_url.sh - name: Upload release tarball id: upload-release-asset uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - upload_url: ${{ steps.get_upload_url.outputs.upload_url }} - asset_path: ./distribution/target/opengrok-${{ needs.get_tag.outputs.tag }}.tar.gz - asset_name: opengrok-${{ needs.get_tag.outputs.tag }}.tar.gz - asset_content_type: application/octet-stream + upload_url: ${{ github.event.release.upload_url }} + asset_path: ./distribution/target/opengrok-${{ github.event.release.tag_name }}.tar.gz + asset_name: opengrok-${{ github.event.release.tag_name }}.tar.gz + asset_content_type: application/gzip diff --git a/dev/docker.sh b/dev/docker.sh deleted file mode 100755 index bb33db5c3c7..00000000000 --- a/dev/docker.sh +++ /dev/null @@ -1,96 +0,0 @@ -#!/bin/bash - -# -# Build and optionally push new image to Docker hub. -# -# When pushing, this script uses the following secure variables: -# - DOCKER_USERNAME -# - DOCKER_PASSWORD -# -# These are set via https://github.com/oracle/opengrok/settings/secrets -# - -set -e - -echo "Running linter" -docker run --rm -i hadolint/hadolint:2.6.0 < Dockerfile || exit 1 - -API_URL="https://hub.docker.com/v2" -IMAGE="opengrok/docker" - -if [[ -n $OPENGROK_REF && $OPENGROK_REF == refs/tags/* ]]; then - OPENGROK_TAG=${OPENGROK_REF#"refs/tags/"} -fi - -if [[ -n $OPENGROK_TAG ]]; then - VERSION="$OPENGROK_TAG" - VERSION_SHORT=$( echo $VERSION | cut -d. -f1,2 ) - - if [[ -z $VERSION ]]; then - echo "empty VERSION" - exit 1 - fi - - if [[ -z $VERSION_SHORT ]]; then - echo "empty VERSION_SHORT" - exit 1 - fi - - echo "Version: $VERSION" - echo "Short version: $VERSION_SHORT" - - TAGS="$VERSION $VERSION_SHORT latest" - - echo "Building docker image for release ($TAGS)" - docker buildx build \ - -t $IMAGE:$VERSION \ - -t $IMAGE:$VERSION_SHORT \ - -t $IMAGE:latest . -else - TAGS="master" - - echo "Building docker image for master" - docker buildx build -t $IMAGE:master . -fi - -# -# Run the image in a container. This is not strictly needed however -# serves as additional test in automatic builds. -# -echo "Running the image in container" -docker run -d $IMAGE -docker ps -a - -# This can only work on home repository since it needs encrypted variables. -if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then - echo "Not pushing Docker image for pull requests" - exit 0 -fi - -# The push only works on the main repository. -if [[ "$OPENGROK_REPO_SLUG" != "oracle/opengrok" ]]; then - echo "Not pushing Docker image for non main repository" - exit 0 -fi - -if [[ -z $DOCKER_USERNAME ]]; then - echo "DOCKER_USERNAME is empty, exiting" - exit 1 -fi - -if [[ -z $DOCKER_PASSWORD ]]; then - echo "DOCKER_PASSWORD is empty, exiting" - exit 1 -fi - -# Publish the image to Docker hub. -if [ -n "$DOCKER_PASSWORD" -a -n "$DOCKER_USERNAME" -a -n "$TAGS" ]; then - echo "Logging into Docker Hub" - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - - # All the tags need to be pushed individually: - for tag in $TAGS; do - echo "Pushing Docker image for tag $tag" - docker push $IMAGE:$tag - done -fi diff --git a/dev/dockerhub_readme.py b/dev/dockerhub_readme.py deleted file mode 100755 index 098d424a8e9..00000000000 --- a/dev/dockerhub_readme.py +++ /dev/null @@ -1,122 +0,0 @@ -#!/usr/bin/env python3 - -""" - Update the README on Docker hub. - - This script uses the following secure variables: - - DOCKER_USERNAME - - DOCKER_PASSWORD - - These are set via https://github.com/oracle/opengrok/settings/secrets -""" - -import logging -import os -import sys - -import requests - -API_URL = "https://hub.docker.com/v2" -IMAGE = "opengrok/docker" -MAIN_REPO_SLUG = "oracle/opengrok" - - -def get_token(username, password): - """ - :param username: Docker hub username - :param password: Docker hub password - :return JWT token string from Docker hub API response - """ - - logger = logging.getLogger(__name__) - - logger.debug("Getting Docker hub token using username/password") - headers = {"Content-Type": "application/json"} - data = {"username": f"{username}", "password": f"{password}"} - response = requests.post(f"{API_URL}/users/login/", headers=headers, json=data) - response.raise_for_status() - - return response.json()["token"] - - -def update_readme(image, readme_file_path, username, password): - """ - Update README file for given image on Docker hub. - :param image: image path (in the form of "namespace_name/repository_name") - :param readme_file_path path to the README file - :param username: Docker hub username - :param password: Docker hub password - """ - - logger = logging.getLogger(__name__) - - token = get_token(username, password) - headers = {"Content-Type": "application/json", "Authorization": f"JWT {token}"} - with open(readme_file_path, "r") as readme_fp: - readme_data = readme_fp.read() - logger.info("Updating README file on Docker hub") - body_data = {"full_description": readme_data} - response = requests.patch( - f"{API_URL}/repositories/{image}/", - headers=headers, - json=body_data, - ) - response.raise_for_status() - - -def check_push_env(): - """ - Check environment variables. - Will exit the program if the environment is not setup for pushing images to Docker hub. - Specifically, number of environment variables is required: - - DOCKER_USERNAME - - DOCKER_PASSWORD - :return Docker hub username and password - """ - - logger = logging.getLogger(__name__) - - repo_slug = os.environ.get("OPENGROK_REPO_SLUG") - if repo_slug is None: - logger.error("OPENGROK_REPO_SLUG environment variable not set") - sys.exit(1) - - if repo_slug != MAIN_REPO_SLUG: - logger.info("Not updating Docker hub README for non main repo") - sys.exit(0) - - event_type = os.environ.get("GITHUB_EVENT_NAME") - if event_type and event_type == "pull_request": - logger.info("Not updating Docker hub README for pull requests") - sys.exit(0) - - docker_username = os.environ.get("DOCKER_USERNAME") - if docker_username is None or len(docker_username) == 0: - logger.info("DOCKER_USERNAME is empty, exiting") - sys.exit(1) - - docker_password = os.environ.get("DOCKER_PASSWORD") - if docker_password is None or len(docker_password) == 0: - logger.info("DOCKER_PASSWORD is empty, exiting") - sys.exit(1) - - return docker_username, docker_password - - -def main(): - """ - main program - update Docker hub README and exit. - """ - logging.basicConfig(level=logging.INFO) - logger = logging.getLogger(__name__) - - docker_username, docker_password = check_push_env() - try: - update_readme(IMAGE, "docker/README.md", docker_username, docker_password) - except requests.exceptions.HTTPError as exc: - logger.error(exc) - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/dev/get_upload_url.sh b/dev/get_upload_url.sh deleted file mode 100755 index 633bd6355cb..00000000000 --- a/dev/get_upload_url.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -# -# The purpose of this script is to retrieve upload URL for OpenGrok release given by the tag -# stored in the OPENGROK_TAG environment variable. -# The value is stored in a special file consumed by Github action so that it can be used -# to upload assets to the related OpenGrok release on Github. -# - -echo "Getting upload URL for $OPENGROK_TAG" -upload_url=$( curl -s https://api.github.com/repos/oracle/opengrok/releases/tags/$OPENGROK_TAG | jq -r .upload_url ) -echo "Got '$upload_url'" -if [[ -n $GITHUB_OUTPUT ]]; then - echo "upload_url=$upload_url" >> $GITHUB_OUTPUT -fi diff --git a/dev/ref2tag.sh b/dev/ref2tag.sh deleted file mode 100755 index 81d32b6adb0..00000000000 --- a/dev/ref2tag.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -tag=${OPENGROK_REF#"refs/tags/"} -echo "tag=$tag" >> $GITHUB_OUTPUT diff --git a/dev/release.sh b/dev/release.sh deleted file mode 100755 index cfbf8f831f6..00000000000 --- a/dev/release.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash -# -# Query current release or trigger new release creation on Github. -# For the latter, it merely kick-starts the creation of new Release on Github, -# see https://github.com/oracle/opengrok/wiki/Release-process -# -# Assumes working Maven + Git. -# - -set -e - -if (( $# > 1 )); then - echo "usage: `basename $0` [version]" - exit 1 -fi - -# Get the latest version (needs curl + jq). -if (( $# == 0 )); then - curl -s https://api.github.com/repos/oracle/opengrok/releases/latest | \ - jq .tag_name - exit 0 -fi - -VERSION=$1 - -if ! echo "$VERSION" | grep '^[0-9]\+\.[0-9]\+\.[0-9]\+$' >/dev/null; then - echo "version needs to be in the form of .." - exit 1 -fi - -if [[ ! -d $PWD/opengrok-indexer ]]; then - echo "This needs to be run from top-level directory of the repository" - exit 1 -fi - -ver=$( git tag -l "$VERSION" ) -if (( $? != 0 )); then - echo "Cannot determine tag" - exit 1 -fi -if [[ $ver == $VERSION ]]; then - echo "Tag $VERSION already exists" - exit 1 -fi - -git pull --ff-only -git switch -c "release_${VERSION}" -./mvnw versions:set -DgenerateBackupPoms=false "-DnewVersion=$VERSION" -git commit pom.xml '**/pom.xml' -m "$VERSION" -git push -echo -echo "Create PR with the changes. Once it is merged in, create new release." -echo From 491ab146232348d71eff73def8c7342f3bae4c1b Mon Sep 17 00:00:00 2001 From: Jiaxun Yang Date: Wed, 7 May 2025 15:20:16 +0100 Subject: [PATCH 2/2] Dockerfile: Improve portability The base image tomcat:10.1.40-jdk21 is actually based on Ubuntu noble but we are still using Ubuntu jammy builder and perforce apt source. Fix by making Ubuntu version explict everywhere. helix-p4d is only available on 386/amd64, filter it out on other architectures. maintainer/source/description labels are only supposed to be set on official builds, which is already done by docker/metadata-action GitHub action, so remove them here. Signed-off-by: Jiaxun Yang --- Dockerfile | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index a1f15c80f63..8cf7050e269 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # Copyright (c) 2018, 2025 Oracle and/or its affiliates. All rights reserved. # Portions Copyright (c) 2020, Chris Fraire . -FROM ubuntu:jammy AS build +FROM ubuntu:noble AS build # hadolint ignore=DL3008 RUN apt-get update && apt-get install --no-install-recommends -y openjdk-21-jdk python3 python3-venv && \ @@ -41,7 +41,7 @@ RUN cp `ls -t distribution/target/*.tar.gz | head -1` /opengrok.tar.gz # Store the version in a file so that the tools can report it. RUN /mvn/mvnw help:evaluate -Dexpression=project.version -q -DforceStdout > /mvn/VERSION -FROM tomcat:10.1.40-jdk21 +FROM tomcat:10.1.40-jdk21-temurin-noble LABEL maintainer="https://github.com/oracle/opengrok" LABEL org.opencontainers.image.source="https://github.com/oracle/opengrok" LABEL org.opencontainers.image.description="OpenGrok code search" @@ -54,7 +54,7 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] # hadolint ignore=DL3059 RUN curl -sS https://package.perforce.com/perforce.pubkey | gpg --dearmor > /etc/apt/trusted.gpg.d/perforce.gpg # hadolint ignore=DL3059 -RUN echo 'deb https://package.perforce.com/apt/ubuntu jammy release' > /etc/apt/sources.list.d/perforce.list +RUN echo 'deb https://package.perforce.com/apt/ubuntu noble release' > /etc/apt/sources.list.d/perforce.list # install dependencies and Python tools # hadolint ignore=DL3008,DL3009 @@ -63,10 +63,13 @@ RUN apt-get update && \ unzip python3 python3-pip \ python3-venv python3-setuptools openssh-client libyaml-dev +ARG TARGETARCH # hadolint ignore=DL3008,DL3059 -RUN architecture=$(uname -m) && if [[ "$architecture" == "aarch64" ]]; then \ - echo "aarch64: do not install helix-p4d."; else \ - apt-get install --no-install-recommends -y helix-p4d || echo "Failed to install Perforce"; fi +RUN if [ "$TARGETARCH" = "amd64" ] || [ "$TARGETARCH" = "386" ]; then \ + apt-get install --no-install-recommends -y helix-p4d || echo "Failed to install Perforce"; \ + else \ + echo "Architecture $TARGETARCH: skipping helix-p4d installation"; \ + fi # compile and install universal-ctags # hadolint ignore=DL3003,DL3008