diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 0000000..5e34ac9 --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,72 @@ +name: build +on: + push: + branches: + - main + - develop + + paths: + # Only rebuild website when apps have changed + - '.github/**' + - 'advanced-tooling/**' + - 'dask-gateway/**' + - 'how-to/**' + - 'container/**' +jobs: + + + container-coder: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + - name: build & push image + run: | + cd container/coder + version=$(cat codemeta.json | jq -r .version ) + IMAGE_ID=ghcr.io/eoap/dev-platform-eoap/eoap-coder + docker build . --file Dockerfile.coder --tag eoap-coder + docker tag eoap-coder $IMAGE_ID:${version} + docker push $IMAGE_ID:${version} + + container-build: + + #needs: [] + + runs-on: ubuntu-latest + + strategy: + matrix: + step: ["application-package-patterns", "advanced-tooling", "how-to"] + + steps: + - uses: actions/checkout@v2 + - run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + - name: build & push image + run: | + version=$(cat ${{ matrix.step }}/codemeta.json | jq -r .version ) + IMAGE_ID=ghcr.io/eoap/dev-platform-eoap/${{ matrix.step }}-coder + docker build ${{ matrix.step }} --file ${{ matrix.step }}/Dockerfile.coder --tag ${{ matrix.step }} + docker tag ${{ matrix.step }} $IMAGE_ID:${version} + docker push $IMAGE_ID:${version} + + container-dask: + + runs-on: ubuntu-latest + + strategy: + matrix: + step: ["client", "coder", "worker"] + + steps: + - uses: actions/checkout@v2 + - run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + - name: build & push image + run: | + version=$(cat dask-gateway/codemeta.json | jq -r .version ) + IMAGE_ID=ghcr.io/eoap/dev-platform-eoap/dask-gateway-${{ matrix.step }} + docker build dask-gateway --file dask-gateway/Dockerfile.${{ matrix.step }} --tag eoap-dask-gateway-${{ matrix.step }} + docker tag eoap-dask-gateway-${{ matrix.step }} $IMAGE_ID:${version} + docker push $IMAGE_ID:${version} \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..e2102e8 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,35 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.2.0 + hooks: + - id: check-added-large-files + - id: check-merge-conflict + - id: debug-statements + - id: detect-private-key + - id: no-commit-to-branch + args: + - --pattern=master|main + - id: requirements-txt-fixer + + - repo: https://github.com/myint/autoflake + rev: v1.4 + hooks: + - id: autoflake + args: + - --in-place + - --remove-unused-variables + - --remove-all-unused-imports + - repo: https://github.com/PyCQA/flake8 + rev: 4.0.1 + hooks: + - id: flake8 + args: + - --max-line-length=188 + - --max-doc-length=190 + + - repo: https://github.com/psf/black + rev: 22.3.0 + hooks: + - id: black + args: + - --line-length=140 diff --git a/README.md b/README.md index fcb0ba9..01d7125 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ cd mastering-app-package skaffold dev ``` -Wait for the deployment to stablize (1-2 minutes) and the open your browser on the link printed, usually http://127.0.0.1:8000. +Wait for the deployment to stablize (1-2 minutes) and then open your browser on the link printed, usually http://127.0.0.1:8000. The typical output is: diff --git a/advanced-tooling/Dockerfile.coder b/advanced-tooling/Dockerfile.coder new file mode 100644 index 0000000..b9be26d --- /dev/null +++ b/advanced-tooling/Dockerfile.coder @@ -0,0 +1,156 @@ +FROM docker.io/library/python:3.10.12@sha256:bac3a0e0d16125977e351c861e2f4b12ecafeaa6f72431dc970d0b9155103232 + +RUN \ + apt-get update && \ + apt-get install -y \ + build-essential \ + curl \ + gcc \ + vim \ + tree \ + file + +RUN \ + echo "**** install node repo ****" && \ + curl -s https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - && \ + echo 'deb https://deb.nodesource.com/node_14.x jammy main' \ + > /etc/apt/sources.list.d/nodesource.list && \ + echo "**** install build dependencies ****" && \ + apt-get update && \ + apt-get install -y \ + nodejs && \ + apt remove yq jq -y + +RUN \ + echo "**** install runtime dependencies ****" && \ + apt-get install -y \ + git \ + libatomic1 \ + nano \ + net-tools \ + sudo \ + podman \ + wget \ + python3 \ + python3-pip + +RUN \ + echo "**** install code-server ****" && \ + if [ -z ${CODE_RELEASE+x} ]; then \ + CODE_RELEASE=$(curl -sX GET https://api.github.com/repos/coder/code-server/releases/latest \ + | awk '/tag_name/{print $4;exit}' FS='[""]' | sed 's|^v||'); \ + fi && \ + mkdir -p /app/code-server && \ + curl -o \ + /tmp/code-server.tar.gz -L \ + "https://github.com/coder/code-server/releases/download/v${CODE_RELEASE}/code-server-${CODE_RELEASE}-linux-amd64.tar.gz" && \ + tar xf /tmp/code-server.tar.gz -C \ + /app/code-server --strip-components=1 && \ + echo "**** patch 4.0.2 ****" && \ + if [ "${CODE_RELEASE}" = "4.0.2" ] && [ "$(uname -m)" != "x86_64" ]; then \ + cd /app/code-server && \ + npm i --production @node-rs/argon2; \ + fi && \ + echo "**** clean up ****" && \ + apt-get purge --auto-remove -y \ + build-essential \ + nodejs && \ + apt-get clean && \ + rm -rf \ + /config/* \ + /tmp/* \ + /var/lib/apt/lists/* \ + /var/tmp/* \ + /etc/apt/sources.list.d/nodesource.list + +ENV USER=jovyan \ + UID=1001 \ + GID=100 \ + HOME=/workspace \ + PATH=/opt/conda/bin:/app/code-server/bin/:$PATH:/app/code-server/ + + +# RUN \ +# echo "**** install conda ****" && \ +# wget https://repo.anaconda.com/miniconda/Miniconda3-py39_4.12.0-Linux-x86_64.sh -O miniconda.sh -q && \ +# sh miniconda.sh -b -p /opt/conda && \ +# conda install -n base -c conda-forge mamba && \ +# conda config --system --append channels conda-forge && \ +# conda config --system --append channels terradue && \ +# conda config --system --append channels eoepca && \ +# conda config --system --append channels r && \ +# conda config --system --set auto_update_conda false && \ +# conda config --system --set show_channel_urls true && \ +# conda config --system --set channel_priority "flexible" + + +# RUN \ +# mamba install -n base cwltool cwl-wrapper==0.12.2 nodejs && \ +# mamba clean -a + +RUN \ + echo "**** install yq, aws cli ****" && \ + VERSION="v4.12.2" && \ + BINARY="yq_linux_amd64" && \ + wget --quiet https://github.com/mikefarah/yq/releases/download/${VERSION}/${BINARY}.tar.gz -O - |\ + tar xz && mv ${BINARY} /usr/bin/yq && \ + pip install awscli && \ + pip install awscli-plugin-endpoint + +RUN \ + echo "**** install jupyter-hub native proxy ****" && \ + pip install jhsingle-native-proxy>=0.0.9 && \ + echo "**** install bash kernel ****" && \ + pip install bash_kernel && \ + python -m bash_kernel.install + +RUN \ + echo "**** adds user jovyan ****" && \ + useradd -m -s /bin/bash -N -u $UID $USER + +COPY entrypoint.sh /opt/entrypoint.sh + +RUN chmod +x /opt/entrypoint.sh + +# RUN chown -R jovyan:100 /opt/conda + +# RUN \ +# echo "**** required by cwltool docker pull even if running with --podman ****" && \ +# ln -s /usr/bin/podman /usr/bin/docker + +ENTRYPOINT ["/opt/entrypoint.sh"] + +EXPOSE 8888 + +RUN echo "**** install kubectl ****" && \ + curl -s -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" && \ + chmod +x kubectl && \ + mkdir -p /workspace/.venv/bin && \ + mv ./kubectl /usr/local/bin/kubectl + + +RUN curl -s -L https://github.com/pypa/hatch/releases/latest/download/hatch-x86_64-unknown-linux-gnu.tar.gz | tar -xzvf - -C /usr/local/bin/ && \ + chmod +x /usr/local/bin/hatch + +RUN curl -s -L https://github.com/go-task/task/releases/download/v3.41.0/task_linux_amd64.tar.gz | tar -xzvf - -C /usr/local/bin/ && \ + chmod +x /usr/local/bin/task + +RUN curl -s -L https://storage.googleapis.com/skaffold/releases/latest/skaffold-linux-amd64 > /usr/local/bin/skaffold && \ + chmod +x /usr/local/bin/skaffold + +RUN pip3 install tomlq && tomlq --version + +RUN rm -f /usr/bin/yq && \ + rm -f /usr/local/bin/yq && \ + curl -s -LO https://github.com/mikefarah/yq/releases/download/v4.45.1/yq_linux_amd64.tar.gz && \ + tar -xvf yq_linux_amd64.tar.gz && \ + mv yq_linux_amd64 /usr/local/bin/yq && \ + cp -v /usr/local/bin/yq /usr/bin/yq && \ + chmod +x /usr/bin/yq /usr/local/bin/yq + +RUN curl -s -L https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux-amd64 > /usr/bin/jq && \ + chmod +x /usr/bin/jq + +RUN pip install --upgrade uv + +USER jovyan \ No newline at end of file diff --git a/advanced-tooling/codemeta.json b/advanced-tooling/codemeta.json new file mode 100644 index 0000000..dfba51c --- /dev/null +++ b/advanced-tooling/codemeta.json @@ -0,0 +1,3 @@ +{ + "version": "0.1.0" +} \ No newline at end of file diff --git a/advanced-tooling/entrypoint.sh b/advanced-tooling/entrypoint.sh new file mode 100644 index 0000000..f5b1057 --- /dev/null +++ b/advanced-tooling/entrypoint.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +collect_port=0 +port="8888" +delim='=' + +for var in "$@" +do + echo "$var" + + if [ "$collect_port" == "1" ]; then + echo "Collecting external port $var" + port=$var + collect_port=0 + fi + + splitarg=${var%%$delim*} + + if [ "$splitarg" == "--port" ]; then + if [ ${#splitarg} == ${#var} ]; then + collect_port=1 + else + port=${var#*$delim} + echo "Setting external port $port" + fi + fi +done + +destport=$((port + 1)) + +echo "Using internal port $destport" + +if [ -z "$CODE_SERVER_WS" ]; then + CODE_SERVER_WS="/workspace" +fi + +jhsingle-native-proxy --port $port --destport $destport code-server {--}auth none {--}bind-addr 0.0.0.0:$destport {--}user-data-dir /workspace $CODE_SERVER_WS \ No newline at end of file diff --git a/advanced-tooling/files/init.sh b/advanced-tooling/files/init.sh index 6973b4c..4775bde 100644 --- a/advanced-tooling/files/init.sh +++ b/advanced-tooling/files/init.sh @@ -19,27 +19,9 @@ echo '{"workbench.colorTheme": "Visual Studio Dark"}' > /workspace/User/settings python -m venv /workspace/.venv source /workspace/.venv/bin/activate -/workspace/.venv/bin/python -m pip install --no-cache-dir tomlq calrissian -# /workspace/.venv/bin/python -m ipykernel install --user --name mastering_env --display-name "Python (Mastering Application Package)" - -echo "**** install kubectl ****" -curl -s -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" -chmod +x kubectl -mkdir -p /workspace/.venv/bin -mv ./kubectl /workspace/.venv/bin/kubectl - - -curl -s -L https://github.com/pypa/hatch/releases/latest/download/hatch-x86_64-unknown-linux-gnu.tar.gz | tar -xzvf - -C /workspace/.venv/bin/ -chmod +x /workspace/.venv/bin/hatch -curl -s -L https://github.com/go-task/task/releases/download/v3.41.0/task_linux_amd64.tar.gz | tar -xzvf - -C /workspace/.venv/bin/ -chmod +x /workspace/.venv/bin/task - -curl -s -L https://storage.googleapis.com/skaffold/releases/latest/skaffold-linux-amd64 > /workspace/.venv/bin/skaffold -chmod +x /workspace/.venv/bin/skaffold - -curl -s -LO https://github.com/mikefarah/yq/releases/download/v4.45.1/yq_linux_amd64.tar.gz -tar -xvf yq_linux_amd64.tar.gz -mv yq_linux_amd64 /workspace/.venv/bin/yq +/workspace/.venv/bin/pip install --upgrade uv +/workspace/.venv/bin/python -m uv pip install --no-cache-dir tomlq calrissian +/workspace/.venv/bin/python -m ipykernel install --user --name advanced_tooling --display-name "Python (Application Package Patterns)" export AWS_DEFAULT_REGION="us-east-1" diff --git a/advanced-tooling/skaffold.yaml b/advanced-tooling/skaffold.yaml index 0b3fe58..5b23f4b 100644 --- a/advanced-tooling/skaffold.yaml +++ b/advanced-tooling/skaffold.yaml @@ -2,7 +2,18 @@ apiVersion: skaffold/v4beta9 kind: Config metadata: name: eoap-advanced-tooling - + +build: + tagPolicy: + envTemplate: + template: 2h + + artifacts: + - image: ttl.sh/eoap-advanced-tooling-coder + context: . + docker: + dockerfile: Dockerfile.coder + deploy: helm: releases: @@ -11,9 +22,9 @@ deploy: namespace: eoap-advanced-tooling createNamespace: true setValues: - coder.coderImage: eoepca/pde-code-server:1.0.0 + coder.coderImage: ttl.sh/eoap-advanced-tooling-coder:2h coder.workspace: advanced-tooling - coderstorageClassName: standard + coder.storageClassName: standard #openebs-kernel-nfs-scw coder.workspaceStorage: 10Gi coderResources.limits.cpu: '2' coderResources.limits.memory: '6442450944' @@ -21,7 +32,9 @@ deploy: coderResources.requests.memory: '4294967296' coder.securityContext.privileged: false calrissian.enabled: true + calrissian.storageClassName: standard #openebs-kernel-nfs-scw skaffold.enabled: true + kaniko.enabled: true setFiles: { initScript: ./files/init.sh, diff --git a/application-package-patterns/Dockerfile.coder b/application-package-patterns/Dockerfile.coder new file mode 100644 index 0000000..0462763 --- /dev/null +++ b/application-package-patterns/Dockerfile.coder @@ -0,0 +1,126 @@ +FROM docker.io/library/python:3.10.12@sha256:bac3a0e0d16125977e351c861e2f4b12ecafeaa6f72431dc970d0b9155103232 + +RUN \ + apt-get update && \ + apt-get install -y \ + build-essential \ + curl \ + gcc \ + vim \ + tree \ + file + +RUN \ + echo "**** install node repo ****" && \ + curl -s https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - && \ + echo 'deb https://deb.nodesource.com/node_14.x jammy main' \ + > /etc/apt/sources.list.d/nodesource.list && \ + echo "**** install build dependencies ****" && \ + apt-get update && \ + apt-get install -y \ + nodejs && \ + apt remove yq jq -y + +RUN \ + echo "**** install runtime dependencies ****" && \ + apt-get install -y \ + git \ + libatomic1 \ + nano \ + net-tools \ + sudo \ + podman \ + wget \ + python3 \ + python3-pip + +RUN \ + echo "**** install code-server ****" && \ + if [ -z ${CODE_RELEASE+x} ]; then \ + CODE_RELEASE=$(curl -sX GET https://api.github.com/repos/coder/code-server/releases/latest \ + | awk '/tag_name/{print $4;exit}' FS='[""]' | sed 's|^v||'); \ + fi && \ + mkdir -p /app/code-server && \ + curl -o \ + /tmp/code-server.tar.gz -L \ + "https://github.com/coder/code-server/releases/download/v${CODE_RELEASE}/code-server-${CODE_RELEASE}-linux-amd64.tar.gz" && \ + tar xf /tmp/code-server.tar.gz -C \ + /app/code-server --strip-components=1 && \ + echo "**** patch 4.0.2 ****" && \ + if [ "${CODE_RELEASE}" = "4.0.2" ] && [ "$(uname -m)" != "x86_64" ]; then \ + cd /app/code-server && \ + npm i --production @node-rs/argon2; \ + fi && \ + echo "**** clean up ****" && \ + apt-get purge --auto-remove -y \ + build-essential && \ + apt-get clean && \ + rm -rf \ + /config/* \ + /tmp/* \ + /var/lib/apt/lists/* \ + /var/tmp/* \ + /etc/apt/sources.list.d/nodesource.list + +ENV USER=jovyan \ + UID=1001 \ + GID=100 \ + HOME=/workspace \ + PATH=/opt/conda/bin:/app/code-server/bin/:$PATH:/app/code-server/ + +RUN \ + echo "**** install aws cli ****" && \ + pip install awscli && \ + pip install awscli-plugin-endpoint + +RUN \ + echo "**** install jupyter-hub native proxy ****" && \ + pip install jhsingle-native-proxy>=0.0.9 && \ + echo "**** install bash kernel ****" && \ + pip install bash_kernel && \ + python -m bash_kernel.install + +RUN \ + echo "**** adds user jovyan ****" && \ + useradd -m -s /bin/bash -N -u $UID $USER + +COPY entrypoint.sh /opt/entrypoint.sh + +RUN chmod +x /opt/entrypoint.sh + +ENTRYPOINT ["/opt/entrypoint.sh"] + +EXPOSE 8888 + +RUN echo "**** install kubectl ****" && \ + curl -s -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" && \ + chmod +x kubectl && \ + mkdir -p /workspace/.venv/bin && \ + mv ./kubectl /usr/local/bin/kubectl + + +RUN curl -s -L https://github.com/pypa/hatch/releases/latest/download/hatch-x86_64-unknown-linux-gnu.tar.gz | tar -xzvf - -C /usr/local/bin/ && \ + chmod +x /usr/local/bin/hatch + +RUN curl -s -L https://github.com/go-task/task/releases/download/v3.41.0/task_linux_amd64.tar.gz | tar -xzvf - -C /usr/local/bin/ && \ + chmod +x /usr/local/bin/task + +RUN curl -s -L https://storage.googleapis.com/skaffold/releases/latest/skaffold-linux-amd64 > /usr/local/bin/skaffold && \ + chmod +x /usr/local/bin/skaffold + +RUN pip3 install tomlq && tomlq --version + +RUN rm -f /usr/bin/yq && \ + rm -f /usr/local/bin/yq && \ + curl -s -LO https://github.com/mikefarah/yq/releases/download/v4.45.1/yq_linux_amd64.tar.gz && \ + tar -xvf yq_linux_amd64.tar.gz && \ + mv yq_linux_amd64 /usr/local/bin/yq && \ + cp -v /usr/local/bin/yq /usr/bin/yq && \ + chmod +x /usr/bin/yq /usr/local/bin/yq + +RUN curl -s -L https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux-amd64 > /usr/bin/jq && \ + chmod +x /usr/bin/jq + +RUN pip install --upgrade uv + +USER jovyan \ No newline at end of file diff --git a/application-package-patterns/codemeta.json b/application-package-patterns/codemeta.json new file mode 100644 index 0000000..dfba51c --- /dev/null +++ b/application-package-patterns/codemeta.json @@ -0,0 +1,3 @@ +{ + "version": "0.1.0" +} \ No newline at end of file diff --git a/application-package-patterns/entrypoint.sh b/application-package-patterns/entrypoint.sh new file mode 100644 index 0000000..f5b1057 --- /dev/null +++ b/application-package-patterns/entrypoint.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +collect_port=0 +port="8888" +delim='=' + +for var in "$@" +do + echo "$var" + + if [ "$collect_port" == "1" ]; then + echo "Collecting external port $var" + port=$var + collect_port=0 + fi + + splitarg=${var%%$delim*} + + if [ "$splitarg" == "--port" ]; then + if [ ${#splitarg} == ${#var} ]; then + collect_port=1 + else + port=${var#*$delim} + echo "Setting external port $port" + fi + fi +done + +destport=$((port + 1)) + +echo "Using internal port $destport" + +if [ -z "$CODE_SERVER_WS" ]; then + CODE_SERVER_WS="/workspace" +fi + +jhsingle-native-proxy --port $port --destport $destport code-server {--}auth none {--}bind-addr 0.0.0.0:$destport {--}user-data-dir /workspace $CODE_SERVER_WS \ No newline at end of file diff --git a/application-package-patterns/files/bash-login b/application-package-patterns/files/bash-login new file mode 100644 index 0000000..5becb33 --- /dev/null +++ b/application-package-patterns/files/bash-login @@ -0,0 +1 @@ +source /workspace/.bashrc \ No newline at end of file diff --git a/application-package-patterns/files/bash-rc b/application-package-patterns/files/bash-rc new file mode 100644 index 0000000..a13eb1d --- /dev/null +++ b/application-package-patterns/files/bash-rc @@ -0,0 +1,8 @@ +alias ll="ls -l" + +alias aws="aws --endpoint-url=http://eoap-application-package-patterns-localstack:4566" + +export PATH=/workspace/.venv/bin:$PATH +export TASK_X_REMOTE_TASKFILES=1 + +source /workspace/.venv/bin/activate \ No newline at end of file diff --git a/application-package-patterns/files/init.sh b/application-package-patterns/files/init.sh new file mode 100644 index 0000000..941ff94 --- /dev/null +++ b/application-package-patterns/files/init.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +cd /workspace + +git clone 'https://github.com/eoap/application-package-patterns.git' + +code-server --install-extension ms-python.python +code-server --install-extension redhat.vscode-yaml +code-server --install-extension sbg-rabix.benten-cwl +code-server --install-extension ms-toolsai.jupyter + +ln -s /workspace/.local/share/code-server/extensions /workspace/extensions + +mkdir -p /workspace/User/ + +echo '{"workbench.colorTheme": "Visual Studio Dark"}' > /workspace/User/settings.json + +python -m venv /workspace/.venv +source /workspace/.venv/bin/activate +/workspace/.venv/bin/pip install --upgrade uv +/workspace/.venv/bin/python -m uv pip install --no-cache-dir stactools calrissian tomlq + +/workspace/.venv/bin/python -m ipykernel install --user --name cwl_how_to_env --display-name "Python (Application Package patterns)" + +cd /workspace +git clone https://github.com/EOEPCA/eoap-cwlwrap.git +cd eoap-cwlwrap +/workspace/.venv/bin/pip install --no-cache-dir -e . + + +export AWS_DEFAULT_REGION="us-east-1" +export AWS_ACCESS_KEY_ID="test" +export AWS_SECRET_ACCESS_KEY="test" + +aws s3 mb s3://results --endpoint-url=http://eoap-application-package-patterns-localstack:4566 + +rm -f /workspace/.venv/bin/yq + +# for calrissian (eoap-cwlwrap uses a recent version of cwltool) +/workspace/.venv/bin/python -m uv pip install cwltool==3.1.20240708091337 \ No newline at end of file diff --git a/application-package-patterns/skaffold.yaml b/application-package-patterns/skaffold.yaml new file mode 100644 index 0000000..1c2b4fd --- /dev/null +++ b/application-package-patterns/skaffold.yaml @@ -0,0 +1,58 @@ +apiVersion: skaffold/v4beta9 +kind: Config +metadata: + name: eoap-application-package-patterns + +build: + tagPolicy: + envTemplate: + template: 2h + + artifacts: + - image: ttl.sh/eoap-application-package-patterns-coder + context: . + docker: + dockerfile: Dockerfile.coder + +deploy: + helm: + releases: + - name: eoap-application-package-patterns + chartPath: ../charts/coder + namespace: eoap-application-package-patterns + createNamespace: true + setValueTemplates: + coder.coderImage: ttl.sh/eoap-application-package-patterns-coder:2h + setValues: + coder.workspace: application-package-patterns + coder.storageClassName: standard #openebs-kernel-nfs-scw + coder.workspaceStorage: 10Gi + coderResources.limits.cpu: '2' + coderResources.limits.memory: '6442450944' + coderResources.requests.cpu: '1' + coderResources.requests.memory: '4294967296' + calrissian.enabled: true + calrissian.storageClassName: standard #openebs-kernel-nfs-scw + calrissian.imagePullSecrets: [] + skaffold.enabled: True + kaniko.enabled: true + + setFiles: { + initScript: ./files/init.sh, + bashrcScript: ./files/bash-rc, + bashloginScript: ./files/bash-login + } + + - name: eoap-application-package-patterns-localstack + remoteChart: localstack/localstack + namespace: eoap-application-package-patterns + createNamespace: true + setValues: + service.type: ClusterIP + +portForward: + - resourceType: service + resourceName: code-server-service + namespace: eoap-application-package-patterns + port: 8080 + localPort: 8000 \ No newline at end of file diff --git a/charts/coder/templates/coder.yaml b/charts/coder/templates/coder.yaml index 163ff06..55ca6c1 100644 --- a/charts/coder/templates/coder.yaml +++ b/charts/coder/templates/coder.yaml @@ -63,9 +63,13 @@ spec: - mountPath: /calrissian name: calrissian-volume - name: node-selectors - mountPath: /etc/calrissian/pod-node-selector.yaml - subPath: pod-node-selector.yaml + mountPath: /etc/calrissian/pod-node-selector.yaml + subPath: pod-node-selector.yaml {{ end }} + {{- if .Values.daskGateway.enabled }} + - name: dask-gateway-config + mountPath: /etc/dask + {{- end }} env: - name: XDG_CONFIG_HOME value: /workspace/.local @@ -86,7 +90,15 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name - {{ end }} + {{- end }} + {{- if .Values.daskGateway.enabled }} + - name: DASK_GATEWAY_ADDRESS + value: "{{ .Values.daskGateway.daskGatewayUrl }}" + - name: DASK_IMAGE + value: "{{ .Values.daskGateway.image }}" + {{- end }} + {{ if .Values.kaniko.enabled }} + {{- end }} resources: limits: cpu: {{ .Values.coderResources.limits.cpu }} @@ -119,6 +131,11 @@ spec: name: node-selectors defaultMode: 420 {{ end }} + {{- if .Values.daskGateway.enabled }} + - name: dask-gateway-config + configMap: + name: dask-gateway-config + {{- end }} --- apiVersion: v1 kind: PersistentVolumeClaim @@ -143,7 +160,7 @@ spec: - name: web port: 8080 targetPort: 8080 - type: ClusterIP + type: {{.Values.coder.serviceType}} --- apiVersion: v1 kind: ConfigMap @@ -151,7 +168,7 @@ metadata: name: init data: init: | - {{ required "A valid .Values.initScript entry required!" ( tpl ( .Values.initScript | default ( .Files.Get "files/init.sh")) . | nindent 10 ) }} + {{ required "A valid .Values.initScript entry required!" ( tpl ( .Values.initScript | default ( .Files.Get "files/init.sh")) . | nindent 10 ) }} --- apiVersion: v1 kind: ConfigMap @@ -159,7 +176,7 @@ metadata: name: bash-rc data: bash-rc: | - {{ required "A valid .Values.bashrcScript entry required!" ( tpl ( .Values.bashrcScript | default ( .Files.Get "files/bash-rc")) . | nindent 10 ) }} + {{ required "A valid .Values.bashrcScript entry required!" ( tpl ( .Values.bashrcScript | default ( .Files.Get "files/bash-rc")) . | nindent 10 ) }} --- apiVersion: v1 kind: ConfigMap @@ -167,7 +184,7 @@ metadata: name: bash-login data: bash-login: | - {{ required "A valid .Values.bashloginScript entry required!" ( tpl ( .Values.bashloginScript | default ( .Files.Get "files/bash-login")) . | nindent 10 ) }} + {{ required "A valid .Values.bashloginScript entry required!" ( tpl ( .Values.bashloginScript | default ( .Files.Get "files/bash-login")) . | nindent 10 ) }} --- {{ if .Values.calrissian.enabled }} apiVersion: v1 @@ -177,4 +194,4 @@ metadata: data: pod-node-selector.yaml: | {} -{{ end }} \ No newline at end of file +{{ end }} diff --git a/charts/coder/templates/daskgateway-config.yaml b/charts/coder/templates/daskgateway-config.yaml new file mode 100644 index 0000000..c56e6e9 --- /dev/null +++ b/charts/coder/templates/daskgateway-config.yaml @@ -0,0 +1,13 @@ +{{- if .Values.daskGateway.enabled }} +kind: ConfigMap +apiVersion: v1 +metadata: + name: dask-gateway-config +data: + gateway.yaml: | + gateway: + address: "{{ .Values.daskGateway.daskGatewayUrl }}" +{{- end }} + + + \ No newline at end of file diff --git a/charts/coder/templates/kaniko-rbac.yaml b/charts/coder/templates/kaniko-rbac.yaml new file mode 100644 index 0000000..38ee269 --- /dev/null +++ b/charts/coder/templates/kaniko-rbac.yaml @@ -0,0 +1,51 @@ +{{- if .Values.kaniko.enabled -}} +apiVersion: v1 +kind: Secret +metadata: + name: kaniko-secret +type: kubernetes.io/dockerconfigjson +data: + .dockerconfigjson: ew0KICAgICJhdXRocyI6IHsNCiAgICAgICAgImZha2UucmVnaXN0cnkuaW8iOiB7DQogICAgICAgICAgICAidXNlcm5hbWUiOiAiZmFrZXVzZXIiLA0KICAgICAgICAgICAgInBhc3N3b3JkIjogImZha2VwYXNzd29yZCIsDQogICAgICAgICAgICAiZW1haWwiOiAiZmFrZUBleGFtcGxlLmNvbSIsDQogICAgICAgICAgICAiYXV0aCI6ICIiDQogICAgICAgIH0NCiAgICB9DQp9 +--- +apiVersion: v1 +kind: Secret +metadata: + name: ttl-secret +type: kubernetes.io/dockerconfigjson +data: + .dockerconfigjson: ew0KICAgICJhdXRocyI6IHsNCiAgICAgICAgImZha2UucmVnaXN0cnkuaW8iOiB7DQogICAgICAgICAgICAidXNlcm5hbWUiOiAiZmFrZXVzZXIiLA0KICAgICAgICAgICAgInBhc3N3b3JkIjogImZha2VwYXNzd29yZCIsDQogICAgICAgICAgICAiZW1haWwiOiAiZmFrZUBleGFtcGxlLmNvbSIsDQogICAgICAgICAgICAiYXV0aCI6ICIiDQogICAgICAgIH0NCiAgICB9DQp9 +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kaniko-sa +imagePullSecrets: + - name: kaniko-secret +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: kaniko-role +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["create", "get", "list", "watch", "delete"] + - apiGroups: [""] + resources: ["pods/exec"] + verbs: ["create"] + - apiGroups: [""] + resources: ["pods/log"] + verbs: ["get","list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: kaniko-rolebinding +subjects: + - kind: ServiceAccount + name: kaniko-sa +roleRef: + kind: Role + name: kaniko-role + apiGroup: rbac.authorization.k8s.io +{{- end -}} \ No newline at end of file diff --git a/charts/coder/templates/role-binding.yaml b/charts/coder/templates/role-binding.yaml index eba5b8d..df02c12 100644 --- a/charts/coder/templates/role-binding.yaml +++ b/charts/coder/templates/role-binding.yaml @@ -48,4 +48,18 @@ roleRef: subjects: - kind: ServiceAccount name: calrissian-sa -{{- end }} \ No newline at end of file +{{- end }} +{{- if .Values.daskGateway.enabled }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: dask-configmap-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: dask-configmap-role +subjects: +- kind: ServiceAccount + name: calrissian-sa +{{- end }} diff --git a/charts/coder/templates/roles.yaml b/charts/coder/templates/roles.yaml index 2b93aa0..2039c21 100644 --- a/charts/coder/templates/roles.yaml +++ b/charts/coder/templates/roles.yaml @@ -39,4 +39,19 @@ rules: - '' resources: - pods/exec -{{- end }} \ No newline at end of file +{{- end }} +{{- if .Values.daskGateway.enabled }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: dask-configmap-role +rules: +- verbs: + - create + - delete + apiGroups: + - '' + resources: + - configmaps +{{- end }} diff --git a/charts/coder/values.yaml b/charts/coder/values.yaml index 820f35e..d6094f5 100644 --- a/charts/coder/values.yaml +++ b/charts/coder/values.yaml @@ -6,10 +6,11 @@ coder: workspaceStorage: 20Gi securityContext: privileged: true + serviceType: ClusterIP calrissian: enabled: True - storage: 10Gi + storage: 20Gi storageClassName: standard imagePullSecrets: - kaniko-secret @@ -23,4 +24,12 @@ coderResources: memory: '4294967296' skaffold: - enabled: False \ No newline at end of file + enabled: False + +daskGateway: + enabled: False + daskGatewayUrl: "" + image: "" + +kaniko: + enabled: False diff --git a/container/coder/Dockerfile.coder b/container/coder/Dockerfile.coder new file mode 100644 index 0000000..be43270 --- /dev/null +++ b/container/coder/Dockerfile.coder @@ -0,0 +1,125 @@ +FROM docker.io/library/python:3.12.11-bookworm@sha256:bea386df48d7ee07eed0a1f3e6f9d5c0292c228b8d8ed2ea738b7a57b29c4470 + +RUN \ + apt-get update && \ + apt-get install -y \ + build-essential \ + curl \ + gcc \ + vim \ + tree \ + file + +RUN \ + echo "**** install node repo ****" && \ + curl -s https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - && \ + echo 'deb https://deb.nodesource.com/node_14.x bookworm main' \ + > /etc/apt/sources.list.d/nodesource.list && \ + echo "**** install build dependencies ****" && \ + apt-get update && \ + apt-get install -y \ + nodejs && \ + apt remove yq jq -y + +RUN \ + echo "**** install runtime dependencies ****" && \ + apt-get install -y \ + git \ + libatomic1 \ + nano \ + net-tools \ + sudo \ + podman \ + wget \ + graphviz + +RUN \ + echo "**** install code-server ****" && \ + if [ -z ${CODE_RELEASE+x} ]; then \ + CODE_RELEASE=$(curl -sX GET https://api.github.com/repos/coder/code-server/releases/latest \ + | awk '/tag_name/{print $4;exit}' FS='[""]' | sed 's|^v||'); \ + fi && \ + mkdir -p /app/code-server && \ + curl -o \ + /tmp/code-server.tar.gz -L \ + "https://github.com/coder/code-server/releases/download/v${CODE_RELEASE}/code-server-${CODE_RELEASE}-linux-amd64.tar.gz" && \ + tar xf /tmp/code-server.tar.gz -C \ + /app/code-server --strip-components=1 && \ + echo "**** patch 4.0.2 ****" && \ + if [ "${CODE_RELEASE}" = "4.0.2" ] && [ "$(uname -m)" != "x86_64" ]; then \ + cd /app/code-server && \ + npm i --production @node-rs/argon2; \ + fi && \ + echo "**** clean up ****" && \ + apt-get purge --auto-remove -y \ + build-essential && \ + apt-get clean && \ + rm -rf \ + /config/* \ + /tmp/* \ + /var/lib/apt/lists/* \ + /var/tmp/* \ + /etc/apt/sources.list.d/nodesource.list + +ENV USER=jovyan \ + UID=1001 \ + GID=100 \ + HOME=/workspace \ + PATH=/opt/conda/bin:/app/code-server/bin/:$PATH:/app/code-server/ + +RUN \ + echo "**** install aws cli ****" && \ + pip install awscli && \ + pip install awscli-plugin-endpoint + +RUN \ + echo "**** install jupyter-hub native proxy ****" && \ + pip install jhsingle-native-proxy>=0.0.9 && \ + echo "**** install bash kernel ****" && \ + pip install bash_kernel && \ + python -m bash_kernel.install + +RUN \ + echo "**** adds user jovyan ****" && \ + useradd -m -s /bin/bash -N -u $UID $USER + +COPY entrypoint.sh /opt/entrypoint.sh + +RUN chmod +x /opt/entrypoint.sh + +ENTRYPOINT ["/opt/entrypoint.sh"] + +EXPOSE 8888 + +RUN echo "**** install kubectl ****" && \ + curl -s -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" && \ + chmod +x kubectl && \ + mkdir -p /workspace/.venv/bin && \ + mv ./kubectl /usr/local/bin/kubectl + + +RUN curl -s -L https://github.com/pypa/hatch/releases/latest/download/hatch-x86_64-unknown-linux-gnu.tar.gz | tar -xzvf - -C /usr/local/bin/ && \ + chmod +x /usr/local/bin/hatch + +RUN curl -s -L https://github.com/go-task/task/releases/download/v3.41.0/task_linux_amd64.tar.gz | tar -xzvf - -C /usr/local/bin/ && \ + chmod +x /usr/local/bin/task + +RUN curl -s -L https://storage.googleapis.com/skaffold/releases/latest/skaffold-linux-amd64 > /usr/local/bin/skaffold && \ + chmod +x /usr/local/bin/skaffold + +RUN pip3 install tomlq && tomlq --version + +RUN rm -f /usr/bin/yq && \ + rm -f /usr/local/bin/yq && \ + curl -s -LO https://github.com/mikefarah/yq/releases/download/v4.45.1/yq_linux_amd64.tar.gz && \ + tar -xvf yq_linux_amd64.tar.gz && \ + mv yq_linux_amd64 /usr/local/bin/yq && \ + cp -v /usr/local/bin/yq /usr/bin/yq && \ + chmod +x /usr/bin/yq /usr/local/bin/yq + +RUN curl -s -L https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux-amd64 > /usr/bin/jq && \ + chmod +x /usr/bin/jq + +RUN pip install --upgrade uv + +USER jovyan \ No newline at end of file diff --git a/container/coder/codemeta.json b/container/coder/codemeta.json new file mode 100644 index 0000000..b107aae --- /dev/null +++ b/container/coder/codemeta.json @@ -0,0 +1,6 @@ +{ + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "type": "SoftwareSourceCode", + "name": "container-coder", + "version": "0.1.0" +} \ No newline at end of file diff --git a/container/coder/entrypoint.sh b/container/coder/entrypoint.sh new file mode 100644 index 0000000..f5b1057 --- /dev/null +++ b/container/coder/entrypoint.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +collect_port=0 +port="8888" +delim='=' + +for var in "$@" +do + echo "$var" + + if [ "$collect_port" == "1" ]; then + echo "Collecting external port $var" + port=$var + collect_port=0 + fi + + splitarg=${var%%$delim*} + + if [ "$splitarg" == "--port" ]; then + if [ ${#splitarg} == ${#var} ]; then + collect_port=1 + else + port=${var#*$delim} + echo "Setting external port $port" + fi + fi +done + +destport=$((port + 1)) + +echo "Using internal port $destport" + +if [ -z "$CODE_SERVER_WS" ]; then + CODE_SERVER_WS="/workspace" +fi + +jhsingle-native-proxy --port $port --destport $destport code-server {--}auth none {--}bind-addr 0.0.0.0:$destport {--}user-data-dir /workspace $CODE_SERVER_WS \ No newline at end of file diff --git a/dask-gateway/Dockerfile.client b/dask-gateway/Dockerfile.client new file mode 100644 index 0000000..91c0ef3 --- /dev/null +++ b/dask-gateway/Dockerfile.client @@ -0,0 +1,34 @@ +FROM docker.io/library/python:3.12.11-bookworm@sha256:bea386df48d7ee07eed0a1f3e6f9d5c0292c228b8d8ed2ea738b7a57b29c4470 + + +# Install necessary packages +RUN apt-get update && apt-get install -y --no-install-recommends \ + git \ + curl \ + wget \ + vim \ + tree \ + sudo && \ + rm -rf /var/lib/apt/lists + +# Create a user named 'dask-user' with sudo privileges +RUN useradd -m dask-user && \ + echo "dask-user:password" | chpasswd && \ + adduser dask-user sudo && \ + echo "dask-user ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers + + +ADD requirements.txt /tmp/requirements.txt + +RUN pip install --no-cache-dir -r /tmp/requirements.txt && \ + rm -rf /tmp/requirements.txt + +# Set the user to 'dask-user' +USER dask-user + +# Add alias for ll="ls -l" to the bash profile of dask-user +RUN echo 'alias ll="ls -l"' >> /home/dask-user/.bashrc + +WORKDIR /home/dask-user + +ENTRYPOINT [] diff --git a/dask-gateway/Dockerfile.coder b/dask-gateway/Dockerfile.coder new file mode 100644 index 0000000..791277f --- /dev/null +++ b/dask-gateway/Dockerfile.coder @@ -0,0 +1,150 @@ +FROM docker.io/library/python:3.12.11-bookworm@sha256:bea386df48d7ee07eed0a1f3e6f9d5c0292c228b8d8ed2ea738b7a57b29c4470 + +RUN \ + apt-get update && \ + apt-get install -y \ + build-essential \ + curl \ + gcc \ + vim \ + tree \ + file + +RUN \ + echo "**** install node repo ****" && \ + curl -s https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - && \ + echo 'deb https://deb.nodesource.com/node_14.x bookworm main' \ + > /etc/apt/sources.list.d/nodesource.list && \ + echo "**** install build dependencies ****" && \ + apt-get update && \ + apt-get install -y \ + nodejs && \ + apt remove yq jq -y + +RUN \ + echo "**** install runtime dependencies ****" && \ + apt-get install -y \ + git \ + libatomic1 \ + nano \ + net-tools \ + sudo \ + podman \ + wget + +RUN \ + echo "**** install code-server ****" && \ + if [ -z ${CODE_RELEASE+x} ]; then \ + CODE_RELEASE=$(curl -sX GET https://api.github.com/repos/coder/code-server/releases/latest \ + | awk '/tag_name/{print $4;exit}' FS='[""]' | sed 's|^v||'); \ + fi && \ + mkdir -p /app/code-server && \ + curl -o \ + /tmp/code-server.tar.gz -L \ + "https://github.com/coder/code-server/releases/download/v${CODE_RELEASE}/code-server-${CODE_RELEASE}-linux-amd64.tar.gz" && \ + tar xf /tmp/code-server.tar.gz -C \ + /app/code-server --strip-components=1 && \ + echo "**** patch 4.0.2 ****" && \ + if [ "${CODE_RELEASE}" = "4.0.2" ] && [ "$(uname -m)" != "x86_64" ]; then \ + cd /app/code-server && \ + npm i --production @node-rs/argon2; \ + fi && \ + echo "**** clean up ****" && \ + apt-get purge --auto-remove -y \ + build-essential \ + nodejs && \ + apt-get clean && \ + rm -rf \ + /config/* \ + /tmp/* \ + /var/lib/apt/lists/* \ + /var/tmp/* \ + /etc/apt/sources.list.d/nodesource.list + +ENV USER=jovyan \ + UID=1001 \ + GID=100 \ + HOME=/workspace \ + PATH=/opt/conda/bin:/app/code-server/bin/:$PATH:/app/code-server/ + + +# RUN \ +# echo "**** install conda ****" && \ +# wget https://repo.anaconda.com/miniconda/Miniconda3-py39_4.12.0-Linux-x86_64.sh -O miniconda.sh -q && \ +# sh miniconda.sh -b -p /opt/conda && \ +# conda install -n base -c conda-forge mamba && \ +# conda config --system --append channels conda-forge && \ +# conda config --system --append channels terradue && \ +# conda config --system --append channels eoepca && \ +# conda config --system --append channels r && \ +# conda config --system --set auto_update_conda false && \ +# conda config --system --set show_channel_urls true && \ +# conda config --system --set channel_priority "flexible" + + +# RUN \ +# mamba install -n base cwltool cwl-wrapper==0.12.2 nodejs && \ +# mamba clean -a + +RUN \ + echo "**** install aws cli ****" && \ + pip install awscli && \ + pip install awscli-plugin-endpoint + +RUN \ + echo "**** install jupyter-hub native proxy ****" && \ + pip install jhsingle-native-proxy>=0.0.9 && \ + echo "**** install bash kernel ****" && \ + pip install bash_kernel && \ + python -m bash_kernel.install + +RUN \ + echo "**** adds user jovyan ****" && \ + useradd -m -s /bin/bash -N -u $UID $USER + +COPY entrypoint.sh /opt/entrypoint.sh + +RUN chmod +x /opt/entrypoint.sh + +# RUN chown -R jovyan:100 /opt/conda + +# RUN \ +# echo "**** required by cwltool docker pull even if running with --podman ****" && \ +# ln -s /usr/bin/podman /usr/bin/docker + +ENTRYPOINT ["/opt/entrypoint.sh"] + +EXPOSE 8888 + +RUN echo "**** install kubectl ****" && \ + curl -s -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" && \ + chmod +x kubectl && \ + mkdir -p /workspace/.venv/bin && \ + mv ./kubectl /usr/local/bin/kubectl + + +RUN curl -s -L https://github.com/pypa/hatch/releases/latest/download/hatch-x86_64-unknown-linux-gnu.tar.gz | tar -xzvf - -C /usr/local/bin/ && \ + chmod +x /usr/local/bin/hatch + +RUN curl -s -L https://github.com/go-task/task/releases/download/v3.41.0/task_linux_amd64.tar.gz | tar -xzvf - -C /usr/local/bin/ && \ + chmod +x /usr/local/bin/task + +RUN curl -s -L https://storage.googleapis.com/skaffold/releases/latest/skaffold-linux-amd64 > /usr/local/bin/skaffold && \ + chmod +x /usr/local/bin/skaffold + +RUN pip3 install tomlq && tomlq --version + +RUN rm -f /usr/bin/yq && \ + rm -f /usr/local/bin/yq && \ + curl -s -LO https://github.com/mikefarah/yq/releases/download/v4.45.1/yq_linux_amd64.tar.gz && \ + tar -xvf yq_linux_amd64.tar.gz && \ + mv yq_linux_amd64 /usr/local/bin/yq && \ + cp -v /usr/local/bin/yq /usr/bin/yq && \ + chmod +x /usr/bin/yq /usr/local/bin/yq + +RUN curl -s -L https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux-amd64 > /usr/bin/jq && \ + chmod +x /usr/bin/jq + +RUN pip install --upgrade uv + +USER jovyan \ No newline at end of file diff --git a/dask-gateway/Dockerfile.worker b/dask-gateway/Dockerfile.worker new file mode 100644 index 0000000..7ae8f0a --- /dev/null +++ b/dask-gateway/Dockerfile.worker @@ -0,0 +1,8 @@ +FROM docker.io/library/python:3.12.11-bookworm@sha256:bea386df48d7ee07eed0a1f3e6f9d5c0292c228b8d8ed2ea738b7a57b29c4470 + +ADD requirements.txt /tmp/requirements.txt + +RUN pip install --no-cache-dir -r /tmp/requirements.txt && \ + rm -rf /tmp/requirements.txt + +ENTRYPOINT [] diff --git a/dask-gateway/README.md b/dask-gateway/README.md new file mode 100644 index 0000000..92a48e6 --- /dev/null +++ b/dask-gateway/README.md @@ -0,0 +1,23 @@ +# Dask Gateway + +This folder contains a skaffold configuration to deploy Dask Gateway and Code Server. + +This deployment is used to run the e-learning module [Dask application package](https://github.com/eoap/dask-app-package) + +## Usage + +Use `skaffold` to deploy on minikube: + +``` +skaffold dev +``` + +You can also use a remote cluster with: + +``` +skaffold dev --default-repo +``` + +## Why the deployment is so long? + +THe code server pod has an init container that installs several tools, code extensions and python packages and this takes time. \ No newline at end of file diff --git a/dask-gateway/codemeta.json b/dask-gateway/codemeta.json new file mode 100644 index 0000000..6455fc9 --- /dev/null +++ b/dask-gateway/codemeta.json @@ -0,0 +1,3 @@ +{ + "version": "0.2.0" +} \ No newline at end of file diff --git a/dask-gateway/entrypoint.sh b/dask-gateway/entrypoint.sh new file mode 100644 index 0000000..f5b1057 --- /dev/null +++ b/dask-gateway/entrypoint.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +collect_port=0 +port="8888" +delim='=' + +for var in "$@" +do + echo "$var" + + if [ "$collect_port" == "1" ]; then + echo "Collecting external port $var" + port=$var + collect_port=0 + fi + + splitarg=${var%%$delim*} + + if [ "$splitarg" == "--port" ]; then + if [ ${#splitarg} == ${#var} ]; then + collect_port=1 + else + port=${var#*$delim} + echo "Setting external port $port" + fi + fi +done + +destport=$((port + 1)) + +echo "Using internal port $destport" + +if [ -z "$CODE_SERVER_WS" ]; then + CODE_SERVER_WS="/workspace" +fi + +jhsingle-native-proxy --port $port --destport $destport code-server {--}auth none {--}bind-addr 0.0.0.0:$destport {--}user-data-dir /workspace $CODE_SERVER_WS \ No newline at end of file diff --git a/dask-gateway/files/bash-login b/dask-gateway/files/bash-login new file mode 100644 index 0000000..5becb33 --- /dev/null +++ b/dask-gateway/files/bash-login @@ -0,0 +1 @@ +source /workspace/.bashrc \ No newline at end of file diff --git a/dask-gateway/files/bash-rc b/dask-gateway/files/bash-rc new file mode 100644 index 0000000..4c25a44 --- /dev/null +++ b/dask-gateway/files/bash-rc @@ -0,0 +1,8 @@ +alias ll="ls -l" + +alias aws="aws --endpoint-url=http://eoap-dask-gateway-localstack:4566" + +export PATH=/workspace/.local/bin:/workspace/.venv/bin:$PATH +export TASK_X_REMOTE_TASKFILES=1 + +source /workspace/.venv/bin/activate \ No newline at end of file diff --git a/dask-gateway/files/init.sh b/dask-gateway/files/init.sh new file mode 100644 index 0000000..bfd9bed --- /dev/null +++ b/dask-gateway/files/init.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +set -x + +cd /workspace + +git clone 'https://github.com/eoap/dask-app-package.git' + +code-server --install-extension ms-python.python +#code-server --install-extension redhat.vscode-yaml +#code-server --install-extension sbg-rabix.benten-cwl +code-server --install-extension ms-toolsai.jupyter +code-server --install-extension tamasfe.even-better-toml + +ln -s /workspace/.local/share/code-server/extensions /workspace/extensions + +mkdir -p /workspace/User/ + +echo '{"workbench.colorTheme": "Visual Studio Dark"}' > /workspace/User/settings.json + +pip install --no-warn-script-location git+https://github.com/Terradue/calrissian@dask-gateway # calrissian + +# eopf-sentinel-2 environment +python -m venv /workspace/.venv-eopf +source /workspace/.venv-eopf/bin/activate +/workspace/.venv-eopf/bin/pip install --upgrade uv +/workspace/.venv-eopf/bin/python -m uv pip install --no-cache-dir numpy==1.26.3 dask==2025.4.0 distributed==2025.4.0 dask-gateway==2025.4.0 pystac bokeh rioxarray==0.18.1 loguru==0.7.3 odc-stac[botocore]==0.3.10 stackstac==0.5.1 pystac-client==0.8.6 planetary-computer==1.0.0 xarray-spatial==0.4.0 "xarray[complete]==2025.4.0" ipykernel rio-color==2.0.1 +res=$? +if [ $res -ne 0 ]; then + echo "Failed to install packages" + exit $res +fi + +cd /workspace/dask-app-package/eopf-sentinel-2 +/workspace/.venv-eopf/bin/python -m uv pip install --no-cache-dir -e . + +/workspace/.venv-eopf/bin/python -m ipykernel install --user --name dask-gateway --display-name "Python (EOPF Dask Application Package)" + +# cloudless-mosaic environment +python -m venv /workspace/.venv-cloudless-mosaic +source /workspace/.venv-cloudless-mosaic/bin/activate + +/workspace/.venv-cloudless-mosaic/bin/pip install --upgrade uv +/workspace/.venv-cloudless-mosaic/bin/python -m uv pip install --no-cache-dir numpy==1.26.3 dask==2025.4.0 distributed==2025.4.0 dask-gateway==2025.4.0 pystac bokeh rioxarray==0.18.1 loguru==0.7.3 odc-stac[botocore]==0.3.10 stackstac==0.5.1 pystac-client==0.8.6 planetary-computer==1.0.0 xarray-spatial==0.4.0 "xarray[complete]==2025.4.0" ipykernel rio-color==2.0.1 +res=$? +if [ $res -ne 0 ]; then + echo "Failed to install packages" + exit $res +fi + +cd /workspace/dask-app-package/cloudless-mosaic +/workspace/.venv-cloudless-mosaic/bin/python -m uv pip install --no-cache-dir -e . + +/workspace/.venv-cloudless-mosaic/bin/python -m ipykernel install --user --name dask-gateway --display-name "Python (Cloudless Mosaic Dask Application Package)" + + +export AWS_DEFAULT_REGION="us-east-1" +export AWS_ACCESS_KEY_ID="test" +export AWS_SECRET_ACCESS_KEY="test" + +aws s3 mb s3://results --endpoint-url=http://eoap-dask-gateway-localstack:4566 + + + diff --git a/dask-gateway/requirements.txt b/dask-gateway/requirements.txt new file mode 100644 index 0000000..431358a --- /dev/null +++ b/dask-gateway/requirements.txt @@ -0,0 +1,15 @@ +numpy==1.26.3 +dask==2025.4.0 +distributed==2025.4.0 +dask-gateway==2025.4.0 +pystac==1.13.0 +bokeh==3.6.2 +rioxarray==0.18.1 +loguru==0.7.3 +odc-stac[botocore]==0.3.10 +stackstac==0.5.1 +pystac-client==0.8.6 +planetary-computer==1.0.0 +xarray-spatial==0.4.0 +rio-color==2.0.1 +xarray[complete]==2025.4.0 \ No newline at end of file diff --git a/dask-gateway/skaffold.yaml b/dask-gateway/skaffold.yaml new file mode 100644 index 0000000..d8a0394 --- /dev/null +++ b/dask-gateway/skaffold.yaml @@ -0,0 +1,76 @@ +apiVersion: skaffold/v4beta9 +kind: Config +metadata: + name: eoap-dask-gateway + +deploy: + statusCheckDeadlineSeconds: 2400 + helm: + + releases: + - name: dask-gateway + remoteChart: https://helm.dask.org/dask-gateway-2024.1.0.tgz + namespace: eoap-dask-gateway + createNamespace: true + valuesFiles: + - values.yaml + setValueTemplates: + gateway.backend.image.name: ghcr.io/eoap/dev-platform-eoap/dask-gateway-worker:0.1.0 + gateway.backend.image.tag: 2h + gateway.backend.imagePullSecrets: + - name: kaniko-secret + traefik.service.type: "ClusterIP" + + - name: eoap-dask-gateway + chartPath: ../charts/coder + namespace: eoap-dask-gateway + createNamespace: true + setValueTemplates: + coder.coderImage: ghcr.io/eoap/dev-platform-eoap/dask-gateway-coder:0.1.0 + daskGateway.image: ghcr.io/eoap/dev-platform-eoap/dask-gateway-worker:0.1.0 + setValues: + coder.workspace: dask-app-package + coder.storageClassName: standard + coder.workspaceStorage: 10Gi + coderResources.limits.cpu: '2' + coderResources.limits.memory: '6442450944' + coderResources.requests.cpu: '1' + coderResources.requests.memory: '4294967296' + calrissian.enabled: true + calrissian.storageClassName: standard + skaffold.enabled: true + daskGateway.enabled: true + daskGateway.daskGatewayUrl: "http://traefik-dask-gateway.eoap-dask-gateway.svc.cluster.local:80" + kaniko.enabled: true + setFiles: { + initScript: ./files/init.sh, + bashrcScript: ./files/bash-rc, + bashloginScript: ./files/bash-login + } + - name: eoap-dask-gateway-localstack + remoteChart: localstack/localstack + namespace: eoap-dask-gateway + createNamespace: true + setValues: + service.type: ClusterIP + + hooks: + before: [] + after: + - host: + command: ["sh", "-c", "./wait-for-it.sh"] + os: [darwin, linux] + +portForward: + - resourceType: service + resourceName: code-server-service + namespace: eoap-dask-gateway + address: localhost + port: 8080 + localPort: 8000 + - resourceType: service + resourceName: traefik-dask-gateway + namespace: eoap-dask-gateway + address: localhost + port: 80 + localPort: 8001 \ No newline at end of file diff --git a/dask-gateway/values.yaml b/dask-gateway/values.yaml new file mode 100644 index 0000000..e1d580a --- /dev/null +++ b/dask-gateway/values.yaml @@ -0,0 +1,11 @@ +gateway: + extraConfig: + dask_gateway_config.py: | + c = get_config() + from dask_gateway_server.options import Options, String, Integer, Float + c.Backend.cluster_options = Options( + Float("worker_cores_limit", default=1, label="Worker Cores Limit"), + Float("worker_cores", default=1, label="Worker Cores"), + String("worker_memory", default="1 G", label="Worker Memory"), + String("image", default="daskgateway/dask-worker:latest", label="Worker Image") + ) \ No newline at end of file diff --git a/dask-gateway/wait-for-it.sh b/dask-gateway/wait-for-it.sh new file mode 100755 index 0000000..8f5d851 --- /dev/null +++ b/dask-gateway/wait-for-it.sh @@ -0,0 +1,23 @@ +#!/bin/bash + + +# Label to identify the deployment +LABEL="app.kubernetes.io/name=dask-gateway" +NAMESPACE="eoap-dask-gateway" + +# Check if the deployment is running +while true; do + # Get the status of the deployment with the specified label + DEPLOYMENT_STATUS=$(kubectl get deployment traefik-dask-gateway -n $NAMESPACE -o "jsonpath={.status.availableReplicas}") + echo "Deployment replicas: $DEPLOYMENT_STATUS" + # Check if the status is 'True' + if [ "$DEPLOYMENT_STATUS" = "1" ]; then + echo "Deployment with label $LABEL is running" + break + else + echo "Waiting for deployment to be running..." + fi + + # Wait for a short period before checking again + sleep 5 +done \ No newline at end of file diff --git a/event-driven-with-argo/README.md b/event-driven-with-argo/README.md index d2cef86..c51493d 100644 --- a/event-driven-with-argo/README.md +++ b/event-driven-with-argo/README.md @@ -1,57 +1,38 @@ - -``` -helm repo add argo https://argoproj.github.io/argo-helm -helm repo add localstack https://helm.localstack.cloud -``` +# Event driven with argo: +## Installation -```python +- The user must start a Minikube cluster using the command below: -from os import environ -from redis import Redis -from time import sleep + ``` + minikube start + ``` -stream_key = environ.get("STREAM", "STREAM") -producer = environ.get("PRODUCER", "user-1") +- Please add the following repositories to your Helm chart to ensure access to the required packages: + ```sh + helm repo add argo https://argoproj.github.io/argo-helm + helm repo add localstack https://helm.localstack.cloud + ``` -def connect_to_redis(): - hostname = environ.get("REDIS_HOSTNAME", "redis-service") - port = environ.get("REDIS_PORT", 6379) + Adding these repositories allows you to seamlessly install and manage **Argo** (a workflow orchestration tool) and **LocalStack** (a local AWS cloud emulator) using Helm. This ensures that your Kubernetes environment is properly configured for deploying workflows and simulating AWS services efficiently. +- Run the command below: + ``` + skaffold dev + ``` - return Redis(hostname, port, retry_on_timeout=True) +- Wait for the deployment to stabilize (1-3 minutes) and then open your browser on the link printed, usually http://127.0.0.1:8000. +- You will see the code server is running. Follow the documentations under `doc` folder. + > Note: It is highly recommended to follow [this tutorial](https://eoap.github.io/event-driven-with-argo/) to know the rationale behind this activity. -def send_event(redis_connection, aoi, reference): - count = 0 - try: - # TODO cloud events - # un-map the "data" wrt app package parameters - data = { - "producer": producer, - "href": reference, - } - resp = redis_connection.xadd(stream_key, data) - print(resp) - count += 1 - except ConnectionError as e: - print(f"ERROR REDIS CONNECTION: {e}") +### **Troubleshooting** -connection = connect_to_redis() +If the **Code Server** is unreachable at `http://127.0.0.1:8000`, there may be an existing process occupying the port. You can terminate the process using the command below, allowing Kubernetes to automatically re-establish port forwarding for traffic on port `8000`: -references = [ - "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2A_10TFK_20210708_0_L2A", - "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210713_0_L2A", - "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2A_10TFK_20210718_0_L2A", - "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2A_10TFK_20220524_0_L2A", - "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2A_10TFK_20220514_0_L2A", - "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2A_10TFK_20220504_0_L2A" -] - -for reference in references: - send_event(connection, aoi, reference) - sleep(1) +```sh +sudo fuser -k 8000/tcp ``` \ No newline at end of file diff --git a/event-driven-with-argo/files/init.sh b/event-driven-with-argo/files/init.sh index e371ec1..fe17126 100644 --- a/event-driven-with-argo/files/init.sh +++ b/event-driven-with-argo/files/init.sh @@ -15,7 +15,7 @@ echo '{"workbench.colorTheme": "Visual Studio Dark"}' > /workspace/User/settings python -m venv /workspace/.venv source /workspace/.venv/bin/activate -/workspace/.venv/bin/python -m pip install --no-cache-dir ipykernel requests pyyaml boto3==1.35.23 loguru redis +/workspace/.venv/bin/python -m pip install --no-cache-dir ipykernel requests pyyaml boto3==1.35.23 loguru redis pystac_client /workspace/.venv/bin/python -m ipykernel install --user --name zoo_env --display-name "Python (Event Driven with Argo)" export AWS_DEFAULT_REGION="us-east-1" diff --git a/how-to/Dockerfile b/how-to/Dockerfile.coder similarity index 100% rename from how-to/Dockerfile rename to how-to/Dockerfile.coder diff --git a/how-to/codemeta.json b/how-to/codemeta.json new file mode 100644 index 0000000..dfba51c --- /dev/null +++ b/how-to/codemeta.json @@ -0,0 +1,3 @@ +{ + "version": "0.1.0" +} \ No newline at end of file diff --git a/how-to/files/bash-rc b/how-to/files/bash-rc index b4d7f46..00102c4 100644 --- a/how-to/files/bash-rc +++ b/how-to/files/bash-rc @@ -2,4 +2,7 @@ alias ll="ls -l" alias aws="aws --endpoint-url=http://eoap-how-to-localstack:4566" +export PATH=/workspace/.venv/bin:$PATH +export TASK_X_REMOTE_TASKFILES=1 + source /workspace/.venv/bin/activate \ No newline at end of file diff --git a/how-to/files/init.sh b/how-to/files/init.sh index a7ce028..6103190 100644 --- a/how-to/files/init.sh +++ b/how-to/files/init.sh @@ -17,6 +17,16 @@ echo '{"workbench.colorTheme": "Visual Studio Dark"}' > /workspace/User/settings python -m venv /workspace/.venv source /workspace/.venv/bin/activate -/workspace/.venv/bin/python -m pip install --no-cache-dir stactools rasterio requests stac-asset click-logging tabulate tqdm pystac-client ipykernel loguru scikit-image rio_stac +/workspace/.venv/bin/python -m pip install --no-cache-dir stactools rasterio requests stac-asset click-logging tabulate tqdm pystac-client ipykernel loguru scikit-image rio_stac calrissian tomlq -/workspace/.venv/bin/python -m ipykernel install --user --name cwl_how_to_env --display-name "Python (CWL How-To's)" \ No newline at end of file +/workspace/.venv/bin/python -m ipykernel install --user --name cwl_how_to_env --display-name "Python (CWL How-To's)" + +curl -s -L https://github.com/go-task/task/releases/download/v3.41.0/task_linux_amd64.tar.gz | tar -xzvf - -C /workspace/.venv/bin/ +chmod +x /workspace/.venv/bin/task + +curl -s -L https://storage.googleapis.com/skaffold/releases/latest/skaffold-linux-amd64 > /workspace/.venv/bin/skaffold +chmod +x /workspace/.venv/bin/skaffold + +curl -s -LO https://github.com/mikefarah/yq/releases/download/v4.45.1/yq_linux_amd64.tar.gz +tar -xvf yq_linux_amd64.tar.gz +mv yq_linux_amd64 /workspace/.venv/bin/yq \ No newline at end of file diff --git a/how-to/skaffold.yaml b/how-to/skaffold.yaml index fa81fc6..b9aafba 100644 --- a/how-to/skaffold.yaml +++ b/how-to/skaffold.yaml @@ -4,9 +4,16 @@ metadata: name: eoap-how-to build: + tagPolicy: + envTemplate: + template: 2h + artifacts: - - image: appimage - + - image: ttl.sh/eoap-how-to-coder + context: . + docker: + dockerfile: Dockerfile.coder + deploy: helm: releases: @@ -15,15 +22,19 @@ deploy: namespace: eoap-how-to createNamespace: true setValueTemplates: - coder.coderImage: "{{.IMAGE_REPO_appimage}}/{{.IMAGE_TAG_appimage}}@{{.IMAGE_DIGEST_appimage}}" + coder.coderImage: ttl.sh/eoap-how-to-coder:2h setValues: coder.workspace: how-to - coderstorageClassName: standard + coder.storageClassName: standard #openebs-kernel-nfs-scw coder.workspaceStorage: 10Gi coderResources.limits.cpu: '2' coderResources.limits.memory: '6442450944' coderResources.requests.cpu: '1' coderResources.requests.memory: '4294967296' + calrissian.enabled: true + calrissian.storageClassName: standard #openebs-kernel-nfs-scw + calrissian.imagePullSecrets: [] + skaffold.enabled: True setFiles: { initScript: ./files/init.sh, bashrcScript: ./files/bash-rc, diff --git a/machine-learning-process/README.md b/machine-learning-process/README.md new file mode 100644 index 0000000..bbf546b --- /dev/null +++ b/machine-learning-process/README.md @@ -0,0 +1,28 @@ +# Installation + +1- install mlflow using helm: +``` +helm repo add community-charts https://community-charts.github.io/helm-charts +helm repo add localstack https://helm.localstack.cloud +helm repo update + +``` + +2- Deploy everything using command below: +``` +skaffold dev +``` +3- Open the code-server and mlflow on your browser: +``` +code-server: http://localhost:8000 +mlflow: http://localhost:5000 +``` + +### **Troubleshooting** + +If the **Code Server** and **Mlflow** are unreachable at `http://127.0.0.1:8000` and `http://127.0.0.1:5000` respectively, there may be an existing process occupying those port. You can terminate the process using the command below, allowing Kubernetes to automatically re-establish port forwarding for traffic on port `8000` and `5000`: + +```sh +sudo fuser -k 8000/tcp +sudo fuser -k 5000/tcp +``` \ No newline at end of file diff --git a/machine-learning-process/charts/cluster-config/Chart.yaml b/machine-learning-process/charts/cluster-config/Chart.yaml new file mode 100644 index 0000000..c2ff6f5 --- /dev/null +++ b/machine-learning-process/charts/cluster-config/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: cluster-config +description: simple cluster-config chart +type: application +version: 0.1.0 +appVersion: "1.0.0" diff --git a/machine-learning-process/charts/cluster-config/templates/role.yaml b/machine-learning-process/charts/cluster-config/templates/role.yaml new file mode 100644 index 0000000..cb64d4a --- /dev/null +++ b/machine-learning-process/charts/cluster-config/templates/role.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: kaniko-role +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["create", "get", "list", "watch", "delete"] + - apiGroups: [""] + resources: ["pods/exec"] + verbs: ["create"] + - apiGroups: [""] + resources: ["pods/log"] + verbs: ["get","list"] \ No newline at end of file diff --git a/machine-learning-process/charts/cluster-config/templates/rolebinding.yaml b/machine-learning-process/charts/cluster-config/templates/rolebinding.yaml new file mode 100644 index 0000000..703c0dc --- /dev/null +++ b/machine-learning-process/charts/cluster-config/templates/rolebinding.yaml @@ -0,0 +1,11 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: kaniko-rolebinding +subjects: + - kind: ServiceAccount + name: kaniko-sa +roleRef: + kind: Role + name: kaniko-role + apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/machine-learning-process/charts/cluster-config/templates/sercret.yaml b/machine-learning-process/charts/cluster-config/templates/sercret.yaml new file mode 100644 index 0000000..8adb2fd --- /dev/null +++ b/machine-learning-process/charts/cluster-config/templates/sercret.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +metadata: + name: kaniko-secret +data: + .dockerconfigjson: {{.Values.secret.dockerconfigjson}} +type: kubernetes.io/dockerconfigjson \ No newline at end of file diff --git a/machine-learning-process/charts/cluster-config/templates/serviceaccount.yaml b/machine-learning-process/charts/cluster-config/templates/serviceaccount.yaml new file mode 100644 index 0000000..fb4ef8f --- /dev/null +++ b/machine-learning-process/charts/cluster-config/templates/serviceaccount.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kaniko-sa +imagePullSecrets: + - name: kaniko-secret \ No newline at end of file diff --git a/machine-learning-process/charts/cluster-config/values.yaml b/machine-learning-process/charts/cluster-config/values.yaml new file mode 100644 index 0000000..006849e --- /dev/null +++ b/machine-learning-process/charts/cluster-config/values.yaml @@ -0,0 +1,2 @@ +secret: + dockerconfigjson: "" \ No newline at end of file diff --git a/machine-learning-process/files/bash-login b/machine-learning-process/files/bash-login new file mode 100644 index 0000000..5becb33 --- /dev/null +++ b/machine-learning-process/files/bash-login @@ -0,0 +1 @@ +source /workspace/.bashrc \ No newline at end of file diff --git a/machine-learning-process/files/bash-rc b/machine-learning-process/files/bash-rc new file mode 100644 index 0000000..0d43c96 --- /dev/null +++ b/machine-learning-process/files/bash-rc @@ -0,0 +1,10 @@ +alias ll="ls -l" +DEBIAN_FRONTEND=noninteractive apt-get install -y tzdata +dpkg-reconfigure --frontend noninteractive tzdata +alias aws="aws --endpoint-url=http://eoap-event-driven-localstack:4566" +export PATH=/workspace/.venv/bin:$PATH +export TASK_X_REMOTE_TASKFILES=1 +export MLFLOW_TRACKING_URI="http://my-mlflow:5000" +export ARROW_TZDATA_DIR=/usr/share/zoneinfo +export PATH=$PATH:/workspace/.venv/bin:/workspace/.venv/bin/yq +source /workspace/.venv/bin/activate \ No newline at end of file diff --git a/machine-learning-process/files/init.sh b/machine-learning-process/files/init.sh new file mode 100644 index 0000000..c92175a --- /dev/null +++ b/machine-learning-process/files/init.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +set -x +cd /workspace + + +# Setup workspace +mkdir machine-learning-process +git clone https://github.com/eoap/machine-learning-process.git +cd machine-learning-process + +# Install VS Code Extensions +code-server --install-extension ms-python.python +code-server --install-extension redhat.vscode-yaml +code-server --install-extension sbg-rabix.benten-cwl +code-server --install-extension ms-toolsai.jupyter +ln -s /workspace/.local/share/code-server/extensions /workspace/extensions + +# Setup user settings +mkdir -p /workspace/User/ +echo '{"workbench.colorTheme": "Visual Studio Dark"}' > /workspace/User/settings.json + +# Setup Python virtual environment +python -m venv /workspace/.venv +source /workspace/.venv/bin/activate +/workspace/.venv/bin/python -m pip install --no-cache-dir tomlq calrissian yq backports.zoneinfo + + +echo "**** install kubectl ****" +curl -s -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" +chmod +x kubectl +mkdir -p /workspace/.venv/bin +mv ./kubectl /workspace/.venv/bin/kubectl + + +curl -s -L https://github.com/pypa/hatch/releases/latest/download/hatch-x86_64-unknown-linux-gnu.tar.gz | tar -xzvf - -C /workspace/.venv/bin/ +chmod +x /workspace/.venv/bin/hatch +curl -s -L https://github.com/go-task/task/releases/download/v3.41.0/task_linux_amd64.tar.gz | tar -xzvf - -C /workspace/.venv/bin/ +chmod +x /workspace/.venv/bin/task + +curl -s -L https://storage.googleapis.com/skaffold/releases/latest/skaffold-linux-amd64 > /workspace/.venv/bin/skaffold +chmod +x /workspace/.venv/bin/skaffold + + + + +chmod +x /workspace/.venv/bin/yq +# AWS environment variables +export AWS_ACCESS_KEY_ID="test" +export AWS_SECRET_ACCESS_KEY="test" +export MLFLOW_TRACKING_URI="http://my-mlflow:5000" +export ARROW_TZDATA_DIR=/usr/share/zoneinfo +export TASK_X_REMOTE_TASKFILES="1" +export PATH=$PATH:/workspace/.venv/bin diff --git a/machine-learning-process/skaffold.yaml b/machine-learning-process/skaffold.yaml new file mode 100644 index 0000000..a61f634 --- /dev/null +++ b/machine-learning-process/skaffold.yaml @@ -0,0 +1,88 @@ +apiVersion: skaffold/v4beta9 +kind: Config +metadata: + name: tile-based-training +deploy: + helm: + releases: + + - name: tile-based-training-localstack + remoteChart: localstack/localstack + namespace: tile-based-training + createNamespace: true + setValues: + service.type: NodePort + + # Code-Server with Calrissian + - name: code-server + chartPath: ../charts/coder + namespace: tile-based-training + createNamespace: true + setValues: + coder.serviceType: NodePort + coder.storageClassName: standard + coder.coderImage: eoepca/pde-code-server:devlatest + coder.workspace: machine-learning-process + coder.workspaceStorage: 40Gi + coderResources.limits.cpu: '3' + coderResources.limits.memory: '10737418240' + coderResources.requests.cpu: '1' + coderResources.requests.memory: '4294967296' + calrissian.enabled: true + #calrissian.storage: 30Gi + coder.securityContext.privileged: true + kaniko.enabled: false + skaffold.enabled: true + setFiles: { + initScript: ./files/init.sh, + bashrcScript: ./files/bash-rc, + bashloginScript: ./files/bash-login, + } + + # MLflow + - name: my-mlflow + remoteChart: community-charts/mlflow + version: 0.15.0 + namespace: tile-based-training + createNamespace: true + setValues: + service.type: ClusterIP + service.port: 5000 + ingress.enabled: true + ingress.className: standard + resources.limits.memory: 5Gi + resources.limits.cpu: 500m + resources.requests.cpu: 200m + resources.requests.memory: 3Gi + backendStore: + databaseMigration: true # Enable database migration to ensure schema is up-to-date + postgres: + enabled: false # Disable PostgreSQL + mysql: + enabled: false # Disable MySQL + artifactRoot: + proxiedArtifactStorage: true # Disable proxied artifact storage access + azureBlob: + enabled: false # Disable Azure Blob Storage + s3: + enabled: false # Disable AWS S3 + gcs: + enabled: false # Disable Google Cloud Storage + + + +portForward: + - resourceType: service + resourceName: my-mlflow + address: localhost + namespace: tile-based-training + port: 5000 + localPort: 5000 + + + - resourceType: service + resourceName: code-server-service # code server service name in ../chart/coder/templates/coder.yaml + namespace: tile-based-training + address: localhost + port: 8080 + localPort: 8000 diff --git a/mastering-app-package/skaffold.yaml b/mastering-app-package/skaffold.yaml index 7b24805..ceb82f6 100644 --- a/mastering-app-package/skaffold.yaml +++ b/mastering-app-package/skaffold.yaml @@ -13,13 +13,14 @@ deploy: setValues: coder.coderImage: eoepca/pde-code-server:1.0.0 coder.workspace: mastering-app-package - coderstorageClassName: standard + coder.storageClassName: standard coder.workspaceStorage: 10Gi coderResources.limits.cpu: '2' coderResources.limits.memory: '6442450944' coderResources.requests.cpu: '1' coderResources.requests.memory: '4294967296' calrissian.enabled: true + calrissian.imagePullSecrets: [] setFiles: { initScript: ./files/init.sh, bashrcScript: ./files/bash-rc, diff --git a/ogc-api-processes-with-zoo/skaffold.yaml b/ogc-api-processes-with-zoo/skaffold.yaml index 266e2b6..c657b4b 100644 --- a/ogc-api-processes-with-zoo/skaffold.yaml +++ b/ogc-api-processes-with-zoo/skaffold.yaml @@ -6,120 +6,95 @@ metadata: build: platforms: ["linux/amd64"] -profiles: - - name: macos - - deploy: - helm: - releases: - - name: zoo-project-dru - remoteChart: zoo-project/zoo-project-dru - namespace: eoap-zoo-project - createNamespace: true - version: 0.3.36 - valuesFiles: - - values.yaml - setValues: - iam.enabled: false - cookiecutter.templateUrl: https://github.com/eoap/zoo-service-template.git - cookiecutter.templateBranch: feature-secrets-nodeselector-imagepullsecrets - filter_in.enabled: true - filter_out.enabled: true - zoofpm.image.pullPolicy: Never - zookernel.image.pullPolicy: Never - workflow.storageClass: hostpath - workflow.argo.storageClass: hostpath - persistence.storageClass: hostpath - persistence.procServicesStorageClass: hostpath - persistence.tmpStorageClass: hostpath - persistence.procServicesAccessMode: ReadWriteMany - workflow.additionalInputs: - s3_bucket: results - region_name: us-east-1 - aws_secret_access_key: test - aws_access_key_id: test - endpoint_url: http://eoap-zoo-project-localstack.eoap-zoo-project.svc.cluster.local:4566 - - - name: eoap-zoo-project-coder - chartPath: ../charts/coder - namespace: eoap-zoo-project - createNamespace: true - setValues: - coder.coderImage: eoepca/pde-code-server:1.0.0 - coder.workspace: ogc-api-processes-with-zoo - #coder.storageClassName: standard - calrissian.storageClassName: hostpath - coder.storageClassName: hostpath - coder.workspaceStorage: 10Gi - coderResources.limits.cpu: '2' - coderResources.limits.memory: '6442450944' - coderResources.requests.cpu: '1' - coderResources.requests.memory: '4294967296' - calrissian.enabled: true +deploy: + helm: + releases: + - name: zoo-project-dru + remoteChart: zoo-project/zoo-project-dru + namespace: eoap-zoo-project + createNamespace: true + #version: 0.4.14 + version: 0.5.0 + valuesFiles: + - values.yaml + setValues: + iam.enabled: false + cookiecutter.templateUrl: https://github.com/eoap/zoo-service-template.git + cookiecutter.templateBranch: feature-secrets-nodeselector-imagepullsecrets + filter_in.enabled: true + filter_out.enabled: true + persistence.procServicesAccessMode: ReadWriteMany + websocketd.enabled: true + websocketd.port: 8888 + websocketd.image.repository: zooproject/websocketd + websocketd.image.tag: 67449315857b54bbc970f02c7aa4fd10a94721f0 + websocketd.image.pullPolicy: IfNotPresent + redis.enabled: true + workflow.additionalInputs: + s3_bucket: results + region_name: us-east-1 + aws_secret_access_key: test + aws_access_key_id: test + endpoint_url: http://eoap-zoo-project-localstack.eoap-zoo-project.svc.cluster.local:4566 - setFiles: { - initScript: ./files/init.sh, - bashrcScript: ./files/bash-rc, - bashloginScript: ./files/bash-login - } - - name: eoap-zoo-project-localstack - remoteChart: localstack/localstack - namespace: eoap-zoo-project - createNamespace: true - setValues: - service.type: ClusterIP + - name: eoap-zoo-project-coder + chartPath: ../charts/coder + namespace: eoap-zoo-project + createNamespace: true + setValues: + coder.coderImage: eoepca/pde-code-server:1.0.0 + coder.workspace: ogc-api-processes-with-zoo + coder.workspaceStorage: 10Gi + coderResources.limits.cpu: '2' + coderResources.limits.memory: '6442450944' + coderResources.requests.cpu: '1' + coderResources.requests.memory: '4294967296' + calrissian.enabled: true - - name: standard + setFiles: { + initScript: ./files/init.sh, + bashrcScript: ./files/bash-rc, + bashloginScript: ./files/bash-login + } + - name: eoap-zoo-project-localstack + remoteChart: localstack/localstack + namespace: eoap-zoo-project + createNamespace: true + setValues: + service.type: ClusterIP - deploy: - helm: - releases: - - name: zoo-project-dru - remoteChart: zoo-project/zoo-project-dru - namespace: eoap-zoo-project - createNamespace: true - version: 0.3.36 - valuesFiles: - - values.yaml - setValues: - iam.enabled: false - cookiecutter.templateUrl: https://github.com/eoap/zoo-service-template.git - cookiecutter.templateBranch: feature-secrets-nodeselector-imagepullsecrets - filter_in.enabled: true - filter_out.enabled: true - persistence.procServicesAccessMode: ReadWriteMany - workflow.additionalInputs: - s3_bucket: results - region_name: us-east-1 - aws_secret_access_key: test - aws_access_key_id: test - endpoint_url: http://eoap-zoo-project-localstack.eoap-zoo-project.svc.cluster.local:4566 +profiles: - - name: eoap-zoo-project-coder - chartPath: ../charts/coder - namespace: eoap-zoo-project - createNamespace: true - setValues: - coder.coderImage: eoepca/pde-code-server:1.0.0 - coder.workspace: ogc-api-processes-with-zoo - coder.workspaceStorage: 10Gi - coderResources.limits.cpu: '2' - coderResources.limits.memory: '6442450944' - coderResources.requests.cpu: '1' - coderResources.requests.memory: '4294967296' - calrissian.enabled: true + - name: macos - setFiles: { - initScript: ./files/init.sh, - bashrcScript: ./files/bash-rc, - bashloginScript: ./files/bash-login - } - - name: eoap-zoo-project-localstack - remoteChart: localstack/localstack - namespace: eoap-zoo-project - createNamespace: true - setValues: - service.type: ClusterIP + patches: + - op: add + path: /deploy/helm/releases/0/setValues/zoofpm.image.pullPolicy + value: Never + - op: add + path: /deploy/helm/releases/0/setValues/zookernel.image.pullPolicy + value: Never + - op: add + path: /deploy/helm/releases/0/setValues/workflow.storageClass + value: hostpath + - op: add + path: /deploy/helm/releases/0/setValues/workflow.argo.storageClass + value: hostpath + - op: add + path: /deploy/helm/releases/0/setValues/persistence.storageClass + value: hostpath + - op: add + path: /deploy/helm/releases/0/setValues/persistence.procServicesStorageClass + value: hostpath + - op: add + path: /deploy/helm/releases/0/setValues/persistence.tmpStorageClass + value: hostpath + - op: add + path: /deploy/helm/releases/1/setValues/coder.storageClassName + value: hostpath + - op: add + path: /deploy/helm/releases/1/setValues/calrissian.storageClassName + value: hostpath portForward: - resourceType: service @@ -127,7 +102,12 @@ portForward: namespace: eoap-zoo-project address: localhost port: 8080 - localPort: 8000 + localPort: 8000 + - resourceType: service + resourceName: zoo-project-dru-websocketd + namespace: eoap-zoo-project + port: 8888 + localPort: 8888 - resourceType: service resourceName: zoo-project-dru-service namespace: eoap-zoo-project diff --git a/zarr-cloud-native-format/files/init.sh b/zarr-cloud-native-format/files/init.sh index d87d282..3da6f5d 100644 --- a/zarr-cloud-native-format/files/init.sh +++ b/zarr-cloud-native-format/files/init.sh @@ -17,6 +17,6 @@ echo '{"workbench.colorTheme": "Visual Studio Dark"}' > /workspace/User/settings python -m venv /workspace/.venv source /workspace/.venv/bin/activate -/workspace/.venv/bin/python -m pip install --no-cache-dir odc-stac ipykernel cwltool zarr matplotlib +/workspace/.venv/bin/python -m pip install --no-cache-dir "odc-stac" ipykernel cwltool zarr matplotlib loguru click "graphviz" "pillow" "cwl-loader>=0.5.0" "eoap-cwlwrap" "plantuml" "cwl2puml>=0.8.0" stactools[validate] rioxarray /workspace/.venv/bin/python -m ipykernel install --user --name cwl_eoap_env --display-name "Python (EOAP)" \ No newline at end of file diff --git a/zarr-cloud-native-format/skaffold.yaml b/zarr-cloud-native-format/skaffold.yaml index c02141e..fc8aa7d 100644 --- a/zarr-cloud-native-format/skaffold.yaml +++ b/zarr-cloud-native-format/skaffold.yaml @@ -11,7 +11,7 @@ deploy: namespace: zarr-cloud-native-format createNamespace: true setValues: - coder.coderImage: eoepca/pde-code-server:1.0.0 + coder.coderImage: ghcr.io/eoap/dev-platform-eoap/eoap-coder:0.1.0 coder.workspace: zarr-cloud-native-format coderstorageClassName: standard coder.workspaceStorage: 10Gi