From d2bbdac78913b9e92d397ca1da29029db0b45772 Mon Sep 17 00:00:00 2001 From: dystewart Date: Thu, 18 Jan 2024 14:53:09 -0500 Subject: [PATCH 1/4] Add docker resources underlying toolkit container image --- docker/Dockerfile | 6 ++++++ docker/src/ope-notebook-culler.sh | 24 ++++++++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 docker/Dockerfile create mode 100644 docker/src/ope-notebook-culler.sh diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..05140b5 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,6 @@ +FROM quay.io/operate-first/opf-toolbox:v0.8.0 +RUN dnf -y update +RUN dnf -y install jq +WORKDIR /shell-scripts +COPY src/* /shell-scripts +CMD ["/bin/sh"] \ No newline at end of file diff --git a/docker/src/ope-notebook-culler.sh b/docker/src/ope-notebook-culler.sh new file mode 100644 index 0000000..db8e676 --- /dev/null +++ b/docker/src/ope-notebook-culler.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +echo "Running ope-notebook-culler.sh..." + +#threshold to stop running notebooks. Currently set to 24 hours +cutoff_time=86400 +current_time=$(date +%s) +notebooks=$(oc get notebooks -n ope-rhods-testing-1fef2f -o jsonpath='{range .items[?(@.status.containerState.running)]}{.metadata.name}{" "}{.metadata.namespace}{" "}{.status.containerState.running.startedAt}{"\n"}{end}') +if [ -z "$notebooks" ]; then + echo "No running notebooks found" + exit 0 +fi + +# Loop through each notebook +while read -r nb ns ts; do + timestamp=$(date -d $ts +%s) + difference=$((current_time - timestamp)) + if [ $difference -gt $cutoff_time ]; then + echo "$nb is more than 24 hours old, stopping the notebook" + oc patch notebook $nb -n $ns --type merge -p '{"metadata":{"annotations":{"kubeflow-resource-stopped":"'$(date -u +"%Y-%m-%dT%H:%M:%SZ")'"}}}' + fi +done <<< "$notebooks" + +echo "ope-notebook-culler.sh run complete." From 8a3499014d8c1f7afa7fe1ce23cd9c7a64985bea Mon Sep 17 00:00:00 2001 From: dystewart Date: Thu, 18 Jan 2024 14:54:07 -0500 Subject: [PATCH 2/4] Add resources for ope-notebook-culler cronjob File formatting to make git happy Formatting --- README.md | 38 ++++++++++++ cronjobs/notebok-culler/cronjob.yaml | 68 +++++++++++++++++++++ cronjobs/notebok-culler/kustomization.yaml | 7 +++ cronjobs/notebok-culler/rolebinding.yaml | 11 ++++ cronjobs/notebok-culler/serviceaccount.yaml | 4 ++ docker/Dockerfile | 2 +- 6 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 cronjobs/notebok-culler/cronjob.yaml create mode 100644 cronjobs/notebok-culler/kustomization.yaml create mode 100644 cronjobs/notebok-culler/rolebinding.yaml create mode 100644 cronjobs/notebok-culler/serviceaccount.yaml diff --git a/README.md b/README.md index ded972a..c9cdec5 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,44 @@ This repository is a collection of useful scripts and tools for TAs and professors to manage students workloads. ## get_url.py +======= +## Cronjobs + +### ope-notebook-culler + +This cronjob runs once every 24 hours at 7am, removing all notebooks & pvcs from the rhods-notebooks namespace. To add resources to the rhods-notebooks namespace: + +1. Ensure you are logged in to your OpenShift account via the CLI and you have access to rhods-notebooks namespace. +2. Switch to rhods-notebooks namespace: +``` + oc project rhods-notebooks +``` + +3. From cronjobs/ope-notebook-culler/ directory run: +``` + oc apply -k . +``` + + This will deploy all the necessary resources for the cronjob to run on the specified schedule. + +Alternatively, to run the script immediately: + +1. Ensure you followed the steps above +2. Verify the cronjob ope-notebook-culler exists +``` + oc get cronjob ope-notebook-culler +``` + +3. Run: +``` + kubectl create -n rhods-notebooks job --from=cronjob/ope-notebook-culler ope-notebook-culler +``` + + This will trigger the cronjob to spawn a job manually. + +## Scripts + +### get_url.py This script is used to retrieve the URL for a particular notebook associated with one student. To execute this script: diff --git a/cronjobs/notebok-culler/cronjob.yaml b/cronjobs/notebok-culler/cronjob.yaml new file mode 100644 index 0000000..f4d74e1 --- /dev/null +++ b/cronjobs/notebok-culler/cronjob.yaml @@ -0,0 +1,68 @@ +kind: CronJob +apiVersion: batch/v1 +metadata: + name: nb-culler + labels: + component.opendatahub.io/name: nb-culler + opendatahub.io/component: 'true' + opendatahub.io/modified: 'false' +spec: + schedule: '0 7 * * *' + startingDeadlineSeconds: 200 + concurrencyPolicy: Replace + suspend: false + jobTemplate: + metadata: + labels: + component.opendatahub.io/name: nb-culler + opendatahub.io/component: 'true' + spec: + template: + metadata: + labels: + component.opendatahub.io/name: nb-culler + opendatahub.io/component: 'true' + parent: nb-culler + spec: + restartPolicy: Never + serviceAccountName: ope-notebook-culler + schedulerName: default-scheduler + terminationGracePeriodSeconds: 30 + securityContext: {} + containers: + - name: oc-cli + image: >- + registry.redhat.io/openshift4/ose-cli@sha256:25fef269ac6e7491cb8340119a9b473acbeb53bc6970ad029fdaae59c3d0ca61 + command: ["/bin/bash", "-c", "--"] + args: + - | + #threshold to stop running notebooks. Currently set to 24 hours + cutoff_time=86400 + current_time=$(date +%s) + notebooks=$(oc get notebooks -n ope-rhods-testing-1fef2f -o jsonpath='{range .items[?(@.status.containerState.running)]}{.metadata.name}{" "}{.metadata.namespace}{" "}{.status.containerState.running.startedAt}{"\n"}{end}') + if [ -z "$notebooks" ]; then + echo "No running notebooks found" + exit 0 + fi + # Loop through each notebook + while read -r nb ns ts; do + timestamp=$(date -d $ts +%s) + difference=$((current_time - timestamp)) + if [ $difference -gt $cutoff_time ]; then + echo "$nb is more than 24 hours old, stopping the notebook" + oc patch notebook $nb -n $ns --type merge -p '{"metadata":{"annotations":{"kubeflow-resource-stopped":"'$(date -u +"%Y-%m-%dT%H:%M:%SZ")'"}}}' + fi + done <<< "$notebooks" + resources: + limits: + memory: 800Mi + requests: + memory: 400Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + imagePullPolicy: IfNotPresent + serviceAccount: ope-notebook-culler + dnsPolicy: ClusterFirst + successfulJobsHistoryLimit: 7 + failedJobsHistoryLimit: 7 + \ No newline at end of file diff --git a/cronjobs/notebok-culler/kustomization.yaml b/cronjobs/notebok-culler/kustomization.yaml new file mode 100644 index 0000000..a534896 --- /dev/null +++ b/cronjobs/notebok-culler/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - cronjob.yaml + - rolebinding.yaml + - serviceaccount.yaml +namespace: rhods-notebooks diff --git a/cronjobs/notebok-culler/rolebinding.yaml b/cronjobs/notebok-culler/rolebinding.yaml new file mode 100644 index 0000000..b5372b8 --- /dev/null +++ b/cronjobs/notebok-culler/rolebinding.yaml @@ -0,0 +1,11 @@ +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: nb-culler +subjects: + - kind: ServiceAccount + name: nb-culler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: edit diff --git a/cronjobs/notebok-culler/serviceaccount.yaml b/cronjobs/notebok-culler/serviceaccount.yaml new file mode 100644 index 0000000..a459299 --- /dev/null +++ b/cronjobs/notebok-culler/serviceaccount.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ope-notebook-culler diff --git a/docker/Dockerfile b/docker/Dockerfile index 05140b5..b9bd454 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -3,4 +3,4 @@ RUN dnf -y update RUN dnf -y install jq WORKDIR /shell-scripts COPY src/* /shell-scripts -CMD ["/bin/sh"] \ No newline at end of file +CMD ["/bin/sh"] From d70cc737efe97207220006e15829b7b66982da2b Mon Sep 17 00:00:00 2001 From: dystewart Date: Fri, 19 Jan 2024 14:47:43 -0500 Subject: [PATCH 3/4] Update README.md to include instructions for running ope-notebook-culler --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index c9cdec5..16de6c9 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,6 @@ This repository is a collection of useful scripts and tools for TAs and professors to manage students workloads. -## get_url.py -======= ## Cronjobs ### ope-notebook-culler From 69e730f01ced98c60951e1822c8d4e71141f79d4 Mon Sep 17 00:00:00 2001 From: dystewart Date: Fri, 26 Jan 2024 14:02:27 -0500 Subject: [PATCH 4/4] Renames cronjob and makes naming consistent across cronjob resources Also removes unused docker resources after switching cronjob logic to mirror redhat internal's Updates README.md to reflect all these changes --- README.md | 12 ++++++----- cronjobs/notebok-culler/cronjob.yaml | 4 ++-- cronjobs/notebok-culler/serviceaccount.yaml | 2 +- docker/Dockerfile | 6 ------ docker/src/ope-notebook-culler.sh | 24 --------------------- 5 files changed, 10 insertions(+), 38 deletions(-) delete mode 100644 docker/Dockerfile delete mode 100644 docker/src/ope-notebook-culler.sh diff --git a/README.md b/README.md index 16de6c9..9af9fe3 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,9 @@ This repository is a collection of useful scripts and tools for TAs and professo ### ope-notebook-culler -This cronjob runs once every 24 hours at 7am, removing all notebooks & pvcs from the rhods-notebooks namespace. To add resources to the rhods-notebooks namespace: +This cronjob runs once every 24 hours at 7am, shutting down all notebooks from the rhods-notebooks namespace. To add resources to the rhods-notebooks namespace: + +NOTE: PVCs persist this shutdown process. 1. Ensure you are logged in to your OpenShift account via the CLI and you have access to rhods-notebooks namespace. 2. Switch to rhods-notebooks namespace: @@ -14,7 +16,7 @@ This cronjob runs once every 24 hours at 7am, removing all notebooks & pvcs from oc project rhods-notebooks ``` -3. From cronjobs/ope-notebook-culler/ directory run: +3. From cronjobs/notebook-culler/ directory run: ``` oc apply -k . ``` @@ -24,14 +26,14 @@ This cronjob runs once every 24 hours at 7am, removing all notebooks & pvcs from Alternatively, to run the script immediately: 1. Ensure you followed the steps above -2. Verify the cronjob ope-notebook-culler exists +2. Verify the cronjob nb-culler exists ``` - oc get cronjob ope-notebook-culler + oc get cronjob nb-culler ``` 3. Run: ``` - kubectl create -n rhods-notebooks job --from=cronjob/ope-notebook-culler ope-notebook-culler + kubectl create -n rhods-notebooks job --from=cronjob/nb-culler nb-culler ``` This will trigger the cronjob to spawn a job manually. diff --git a/cronjobs/notebok-culler/cronjob.yaml b/cronjobs/notebok-culler/cronjob.yaml index f4d74e1..62d21d1 100644 --- a/cronjobs/notebok-culler/cronjob.yaml +++ b/cronjobs/notebok-culler/cronjob.yaml @@ -39,7 +39,7 @@ spec: #threshold to stop running notebooks. Currently set to 24 hours cutoff_time=86400 current_time=$(date +%s) - notebooks=$(oc get notebooks -n ope-rhods-testing-1fef2f -o jsonpath='{range .items[?(@.status.containerState.running)]}{.metadata.name}{" "}{.metadata.namespace}{" "}{.status.containerState.running.startedAt}{"\n"}{end}') + notebooks=$(oc get notebooks -n rhods-notebooks -o jsonpath='{range .items[?(@.status.containerState.running)]}{.metadata.name}{" "}{.metadata.namespace}{" "}{.status.containerState.running.startedAt}{"\n"}{end}') if [ -z "$notebooks" ]; then echo "No running notebooks found" exit 0 @@ -61,7 +61,7 @@ spec: terminationMessagePath: /dev/termination-log terminationMessagePolicy: File imagePullPolicy: IfNotPresent - serviceAccount: ope-notebook-culler + serviceAccount: nb-culler dnsPolicy: ClusterFirst successfulJobsHistoryLimit: 7 failedJobsHistoryLimit: 7 diff --git a/cronjobs/notebok-culler/serviceaccount.yaml b/cronjobs/notebok-culler/serviceaccount.yaml index a459299..7937c8a 100644 --- a/cronjobs/notebok-culler/serviceaccount.yaml +++ b/cronjobs/notebok-culler/serviceaccount.yaml @@ -1,4 +1,4 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: ope-notebook-culler + name: nb-culler diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index b9bd454..0000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -FROM quay.io/operate-first/opf-toolbox:v0.8.0 -RUN dnf -y update -RUN dnf -y install jq -WORKDIR /shell-scripts -COPY src/* /shell-scripts -CMD ["/bin/sh"] diff --git a/docker/src/ope-notebook-culler.sh b/docker/src/ope-notebook-culler.sh deleted file mode 100644 index db8e676..0000000 --- a/docker/src/ope-notebook-culler.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -echo "Running ope-notebook-culler.sh..." - -#threshold to stop running notebooks. Currently set to 24 hours -cutoff_time=86400 -current_time=$(date +%s) -notebooks=$(oc get notebooks -n ope-rhods-testing-1fef2f -o jsonpath='{range .items[?(@.status.containerState.running)]}{.metadata.name}{" "}{.metadata.namespace}{" "}{.status.containerState.running.startedAt}{"\n"}{end}') -if [ -z "$notebooks" ]; then - echo "No running notebooks found" - exit 0 -fi - -# Loop through each notebook -while read -r nb ns ts; do - timestamp=$(date -d $ts +%s) - difference=$((current_time - timestamp)) - if [ $difference -gt $cutoff_time ]; then - echo "$nb is more than 24 hours old, stopping the notebook" - oc patch notebook $nb -n $ns --type merge -p '{"metadata":{"annotations":{"kubeflow-resource-stopped":"'$(date -u +"%Y-%m-%dT%H:%M:%SZ")'"}}}' - fi -done <<< "$notebooks" - -echo "ope-notebook-culler.sh run complete."