diff --git a/.github/workflows/piplock-renewal-2023a.yml b/.github/workflows/piplock-renewal-2023a.yml deleted file mode 100644 index d98646e28..000000000 --- a/.github/workflows/piplock-renewal-2023a.yml +++ /dev/null @@ -1,47 +0,0 @@ ---- -# This GitHub action is meant to be triggered weekly in order to update the pipfile.locks - -name: Weekly Pipfile.locks renewal on [2023a] branch - -on: # yamllint disable-line rule:truthy - # Triggers the workflow every Monday at 22pm UTC am 0 22 * * 1 - schedule: - - cron: "0 22 * * 1" - workflow_dispatch: # for manual trigger workflow from GH Web UI - -jobs: - build: - runs-on: ubuntu-latest - - permissions: - # Give the default GITHUB_TOKEN write permission to commit and push the - # added or changed files to the repository. - contents: write - - steps: - # Checkout the paricular branch - - name: Checkout code from the release branch - uses: actions/checkout@v4 - with: - ref: 2023a - token: ${{ secrets.GH_ACCESS_TOKEN }} - - # Setup Python environment - - name: Setup Python environment - uses: actions/setup-python@v5 - with: - python-version: | - 3.8 - 3.9 - - name: Install pipenv - run: pip install pipenv - - # Runs the makefile recipe `refresh-pipfilelock-files` and push the chances back to the branch - - name: Run make refresh-pipfilelock-files and push the chances back to the branch - run: | - make refresh-pipfilelock-files - git config --global user.email "github-actions[bot]@users.noreply.github.com" - git config --global user.name "GitHub Actions" - git add . - git commit -m "Update the pipfile.lock via the weekly workflow action" - git push diff --git a/.github/workflows/piplock-renewal-2023b.yml b/.github/workflows/piplock-renewal-2023b.yml deleted file mode 100644 index c2c8161bf..000000000 --- a/.github/workflows/piplock-renewal-2023b.yml +++ /dev/null @@ -1,47 +0,0 @@ ---- -# This GitHub action is meant to be triggered weekly in order to update the pipfile.locks - -name: Weekly Pipfile.locks renewal on [2023b] branch - -on: # yamllint disable-line rule:truthy - # Triggers the workflow every Monday at 22pm UTC am 0 22 * * 1 - schedule: - - cron: "0 22 * * 1" - workflow_dispatch: # for manual trigger workflow from GH Web UI - -jobs: - build: - runs-on: ubuntu-latest - - permissions: - # Give the default GITHUB_TOKEN write permission to commit and push the - # added or changed files to the repository. - contents: write - - steps: - # Checkout the paricular branch - - name: Checkout code from the release branch - uses: actions/checkout@v4 - with: - ref: 2023b - token: ${{ secrets.GH_ACCESS_TOKEN }} - - # Setup Python environment - - name: Setup Python environment - uses: actions/setup-python@v5 - with: - python-version: | - 3.8 - 3.9 - - name: Install pipenv - run: pip install pipenv - - # Runs the makefile recipe `refresh-pipfilelock-files` and push the chances back to the branch - - name: Run make refresh-pipfilelock-files and push the chances back to the branch - run: | - make refresh-pipfilelock-files - git config --global user.email "github-actions[bot]@users.noreply.github.com" - git config --global user.name "GitHub Actions" - git add . - git commit -m "Update the pipfile.lock via the weekly workflow action" - git push diff --git a/.github/workflows/piplock-renewal-2024a.yml b/.github/workflows/piplock-renewal-2024a.yml deleted file mode 100644 index b01323954..000000000 --- a/.github/workflows/piplock-renewal-2024a.yml +++ /dev/null @@ -1,47 +0,0 @@ ---- -# This GitHub action is meant to be triggered weekly in order to update the pipfile.locks - -name: Weekly Pipfile.locks renewal on [2024a] branch - -on: # yamllint disable-line rule:truthy - # Triggers the workflow every Monday at 22pm UTC am 0 22 * * 1 - schedule: - - cron: "0 22 * * 1" - workflow_dispatch: # for manual trigger workflow from GH Web UI - -jobs: - build: - runs-on: ubuntu-latest - - permissions: - # Give the default GITHUB_TOKEN write permission to commit and push the - # added or changed files to the repository. - contents: write - - steps: - # Checkout the paricular branch - - name: Checkout code from the release branch - uses: actions/checkout@v4 - with: - ref: 2024a - token: ${{ secrets.GH_ACCESS_TOKEN }} - - # Setup Python environment - - name: Setup Python environment - uses: actions/setup-python@v5 - with: - python-version: | - 3.8 - 3.9 - - name: Install pipenv - run: pip install pipenv - - # Runs the makefile recipe `refresh-pipfilelock-files` and push the chances back to the branch - - name: Run make refresh-pipfilelock-files and push the chances back to the branch - run: | - make refresh-pipfilelock-files - git config --global user.email "github-actions[bot]@users.noreply.github.com" - git config --global user.name "GitHub Actions" - git add . - git commit -m "Update the pipfile.lock via the weekly workflow action" - git push diff --git a/.github/workflows/piplock-renewal-main.yml b/.github/workflows/piplock-renewal-main.yml deleted file mode 100644 index b3e9ca366..000000000 --- a/.github/workflows/piplock-renewal-main.yml +++ /dev/null @@ -1,47 +0,0 @@ ---- -# This GitHub action is meant to be triggered weekly in order to update the pipfile.locks - -name: Weekly Pipfile.locks renewal on [main] branch - -on: # yamllint disable-line rule:truthy - # Triggers the workflow every Monday at 22pm UTC am 0 22 * * 1 - schedule: - - cron: "0 22 * * 1" - workflow_dispatch: # for manual trigger workflow from GH Web UI - -jobs: - build: - runs-on: ubuntu-latest - - permissions: - # Give the default GITHUB_TOKEN write permission to commit and push the - # added or changed files to the repository. - contents: write - - steps: - # Checkout the paricular branch - - name: Checkout code from the release branch - uses: actions/checkout@v4 - with: - ref: main - token: ${{ secrets.GH_ACCESS_TOKEN }} - - # Setup Python environment - - name: Setup Python environment - uses: actions/setup-python@v5 - with: - python-version: | - 3.9 - 3.11 - - name: Install pipenv - run: pip install pipenv - - # Runs the makefile recipe `refresh-pipfilelock-files` and push the chances back to the branch - - name: Run make refresh-pipfilelock-files and push the chances back to the branch - run: | - make refresh-pipfilelock-files - git config --global user.email "github-actions[bot]@users.noreply.github.com" - git config --global user.name "GitHub Actions" - git add . - git commit -m "Update the pipfile.lock via the weekly workflow action" - git push diff --git a/.github/workflows/piplock-renewal.yaml b/.github/workflows/piplock-renewal.yaml new file mode 100644 index 000000000..17e4d5968 --- /dev/null +++ b/.github/workflows/piplock-renewal.yaml @@ -0,0 +1,68 @@ +--- +# This GitHub action is meant to update the pipfile.locks +name: Pipfile.locks Renewal Action + +on: # yamllint disable-line rule:truthy + # Triggers the workflow every Monday at 22pm UTC am 0 22 * * 1 + schedule: + - cron: "0 22 * * 1" + workflow_dispatch: # for manual trigger workflow from GH Web UI + inputs: + branch: + description: 'Specify branch' + required: false + default: 'main' + python_version: + description: 'Select Python version to update Pipfile.lock' + required: false + default: '3.11' + type: choice + options: + - '3.11' + - '3.9' + - '3.8' + update_optional_dirs: + description: 'Include optional directories in update' + required: false + default: 'false' + type: choice + options: + - 'true' + - 'false' + +jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: write + steps: + # Checkout the specified branch from the specified organization + - name: Checkout code from the specified branch + uses: actions/checkout@v4 + with: + ref: ${{ github.event.inputs.branch }} + token: ${{ secrets.GH_ACCESS_TOKEN }} + + # Configure Git + - name: Configure Git + run: | + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git config --global user.name "GitHub Actions" + + # Setup Python environment with the specified version (or default to '3.11') + - name: Setup Python environment + uses: actions/setup-python@v5 + with: + python-version: ${{ github.event.inputs.python_version }} + + # Install pipenv + - name: Install pipenv + run: pip install pipenv + + # Run makefile recipe to refresh Pipfile.lock and push changes back to the branch + - name: Run make refresh-pipfilelock-files and push the changes back to the branch + run: | + make refresh-pipfilelock-files PYTHON_VERSION=${{ github.event.inputs.python_version }} INCLUDE_OPT_DIRS=${{ github.event.inputs.update_optional_dirs }} + git add . + git commit -m "Update Pipfile.lock files by piplock-renewal.yaml action" + git push origin ${{ github.event.inputs.branch }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c864cd0ec..b180a482a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -27,7 +27,6 @@ Pull requests are the best way to propose changes to the notebooks repository: - Create a proper filepath and naming to the corresponding folder - Add the minimum files you have to add: - Pipfile with the additional packages - - Generate the Pipfile lock by running `pipenv lock` to the corresponding pipfile directory - Dockefile with proper instructions - Kustomization objects to deploy the new notebook into an openshift cluster (Kustomization.yaml, service.yaml, statefulset.yaml) - Create instructions into Makefile, for example if you derive the new notebooks from minimal then the recipe should be like the following: @@ -38,6 +37,7 @@ Pull requests are the best way to propose changes to the notebooks repository: $(call image,$@,jupyter/${NOTEBOOK_NAME}/ubi8-python-3.8,$<) ``` - Add the paths of the new pipfiles under `refresh-pipfilelock-files` +- Run the [piplock-renewal.yaml](https://github.com/opendatahub-io/notebooks/blob/main/.github/workflows/piplock-renewal.yaml) against your fork branch, check [here](https://github.com/opendatahub-io/notebooks/blob/main/README.md) for more info. - Test the changes locally, by manually running the `$ make jupyter-${NOTEBOOK_NAME}-ubi8-python-3.8` from the terminal. diff --git a/Makefile b/Makefile index 65c42d803..713493759 100644 --- a/Makefile +++ b/Makefile @@ -650,41 +650,63 @@ validate-rstudio-image: bin/kubectl continue; \ fi; \ -# This is only for the workflow action +# This recipe used mainly from the Pipfile.locks Renewal Action +# Default Python version +PYTHON_VERSION ?= 3.11 +ROOT_DIR := $(shell pwd) +BASE_DIRS := base/c9s-python-$(PYTHON_VERSION) \ + base/ubi9-python-$(PYTHON_VERSION) \ + jupyter/minimal/ubi9-python-$(PYTHON_VERSION) \ + jupyter/datascience/ubi9-python-$(PYTHON_VERSION) \ + jupyter/pytorch/ubi9-python-$(PYTHON_VERSION) \ + jupyter/tensorflow/ubi9-python-$(PYTHON_VERSION) \ + jupyter/trustyai/ubi9-python-$(PYTHON_VERSION) \ + jupyter/rocm/tensorflow/ubi9-python-$(PYTHON_VERSION) \ + jupyter/rocm/pytorch/ubi9-python-$(PYTHON_VERSION) \ + codeserver/ubi9-python-$(PYTHON_VERSION) \ + runtimes/minimal/ubi9-python-$(PYTHON_VERSION) \ + runtimes/datascience/ubi9-python-$(PYTHON_VERSION) \ + runtimes/pytorch/ubi9-python-$(PYTHON_VERSION) \ + runtimes/tensorflow/ubi9-python-$(PYTHON_VERSION) \ + runtimes/rocm-tensorflow/ubi9-python-$(PYTHON_VERSION) \ + runtimes/rocm-pytorch/ubi9-python-$(PYTHON_VERSION) + +# Default value is false, can be overiden +# The below directories are not supported on tier-1 +INCLUDE_OPT_DIRS ?= false +OPT_DIRS := jupyter/intel/ml/ubi9-python-$(PYTHON_VERSION) \ + jupyter/intel/pytorch/ubi9-python-$(PYTHON_VERSION) \ + jupyter/intel/tensorflow/ubi9-python-$(PYTHON_VERSION) \ + intel/runtimes/ml/ubi9-python-$(PYTHON_VERSION) \ + intel/runtimes/pytorch/ubi9-python-$(PYTHON_VERSION) \ + intel/runtimes/tensorflow/ubi9-python-$(PYTHON_VERSION) + +# This recipe gets args, can be used like +# make refresh-pipfilelock-files PYTHON_VERSION=3.11 INCLUDE_OPT_DIRS=false .PHONY: refresh-pipfilelock-files refresh-pipfilelock-files: - cd base/ubi9-python-3.9 && pipenv lock - cd base/c9s-python-3.9 && pipenv lock - cd jupyter/minimal/ubi9-python-3.9 && pipenv lock - cd jupyter/datascience/ubi9-python-3.9 && pipenv lock - cd jupyter/pytorch/ubi9-python-3.9 && pipenv lock - cd jupyter/tensorflow/ubi9-python-3.9 && pipenv lock - cd jupyter/trustyai/ubi9-python-3.9 && pipenv lock - cd runtimes/minimal/ubi9-python-3.9 && pipenv lock - cd runtimes/datascience/ubi9-python-3.9 && pipenv lock - cd runtimes/pytorch/ubi9-python-3.9 && pipenv lock - cd runtimes/tensorflow/ubi9-python-3.9 && pipenv lock - cd runtimes/rocm-tensorflow/ubi9-python-3.9 && pipenv lock - cd runtimes/rocm-pytorch/ubi9-python-3.9 && pipenv lock - cd base/c9s-python-3.11 && pipenv lock - cd base/ubi9-python-3.11 && pipenv lock - cd codeserver/ubi9-python-3.11 && pipenv lock - cd jupyter/minimal/ubi9-python-3.11 && pipenv lock - cd jupyter/datascience/ubi9-python-3.11 && pipenv lock - cd jupyter/pytorch/ubi9-python-3.11 && pipenv lock - cd jupyter/tensorflow/ubi9-python-3.11 && pipenv lock - cd jupyter/trustyai/ubi9-python-3.11 && pipenv lock - cd jupyter/rocm/tensorflow/ubi9-python-3.11 && pipenv lock - cd jupyter/rocm/pytorch/ubi9-python-3.11 && pipenv lock - cd runtimes/minimal/ubi9-python-3.11 && pipenv lock - cd runtimes/datascience/ubi9-python-3.11 && pipenv lock - cd runtimes/pytorch/ubi9-python-3.11 && pipenv lock - cd runtimes/tensorflow/ubi9-python-3.11 && pipenv lock - cd runtimes/rocm-tensorflow/ubi9-python-3.11 && pipenv lock - cd runtimes/rocm-pytorch/ubi9-python-3.11 && pipenv lock - - - + @echo "Updating Pipfile.lock files for Python $(PYTHON_VERSION)" + @if [ "$(INCLUDE_OPT_DIRS)" = "true" ]; then \ + echo "Including optional directories"; \ + DIRS="$(BASE_DIRS) $(OPT_DIRS)"; \ + else \ + DIRS="$(BASE_DIRS)"; \ + fi; \ + for dir in $$DIRS; do \ + echo "Processing directory: $$dir"; \ + cd $(ROOT_DIR); \ + if [ -d "$$dir" ]; then \ + echo "Updating $(PYTHON_VERSION) Pipfile.lock in $$dir"; \ + cd $$dir; \ + if [ -f "Pipfile" ]; then \ + pipenv lock; \ + else \ + echo "No Pipfile found in $$dir, skipping."; \ + fi; \ + else \ + echo "Skipping $$dir as it does not exist"; \ + fi; \ + done # This is only for the workflow action # For running manually, set the required environment variables diff --git a/README.md b/README.md index 7120bb7a4..a54fbe173 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,12 @@ Use podman/docker to execute the workbench images as container. podman run -it -p 8888:8888 quay.io/opendatahub/workbench-images:jupyter-minimal-ubi9-python-3.9-2024a-20240317-6f4c36b ``` +### Pipfile.lock Generation + +Users can update Pipfile.lock files using the [piplock-renewal.yaml](https://github.com/opendatahub-io/notebooks/blob/main/.github/workflows/piplock-renewal.yaml) GitHub Action. This workflow enables users to specify a target branch for updating and automerging Pipfile.lock files, select the desired Python version for the update as well as to choose whether to include optional directories in the update process. After the action completes, the updated files can be retrieved with a simple git pull. + +Note: To ensure the GitHub Action runs successfully, users must add a `GH_ACCESS_TOKEN` secret in their fork. + ### Deploy & Test #### Running Python selftests in Pytest