From 47613a5eeb121a219d41a76b4454f6fb39927f80 Mon Sep 17 00:00:00 2001 From: Abby Drury Date: Thu, 20 Apr 2023 08:50:56 -0400 Subject: [PATCH 1/3] feat(environment): Work towards images that are tagged --- .gitignore | 1 + environment/Dockerfile | 25 ++++++- environment/README.md | 27 +++++-- environment/compose.yml | 1 + environment/docker-environment-common.sh | 94 ++++++++++++++++++++++++ 5 files changed, 137 insertions(+), 11 deletions(-) create mode 100755 environment/docker-environment-common.sh diff --git a/.gitignore b/.gitignore index c92ab98..3bed88b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ # ignore dotfiles and directories (except .gitignore) .* !.gitignore +environment/image-tags diff --git a/environment/Dockerfile b/environment/Dockerfile index 562669f..ca9e762 100644 --- a/environment/Dockerfile +++ b/environment/Dockerfile @@ -3,18 +3,35 @@ # Start from a core stack version FROM jupyter/datascience-notebook:9e63909e0317 -# Volumes aren't in place yet +# Volumes aren't in place yet, get the stuff into the container WORKDIR /home/jovyan/ COPY --chown=${NB_UID}:${NB_GID} . . -# Set up jupyterlab/ for UI to use +# Tag the image +ARG BUILD_DATE +ARG VCS_REF +# +LABEL org.label-schema.build-date=$BUILD_DATE \ + org.label-schema.name="mtholyoke/docker-for-data-science" \ + org.label-schema.description="Standardized and reproducible data science research environment" \ + org.label-schema.url="https://github.com/mtholyoke/docker-data-science-environment/wiki" \ + org.label-schema.vcs-url="https://github.com/mtholyoke/docker-data-science-environment" \ + org.label-schema.vcs-ref=$VCS_REF \ + org.label-schema.vendor="Mount Holyoke College" \ + org.label-schema.version="1.1.0" \ + org.label-schema.schema-version="1.0.0-rc.1" +# org.label-schema.usage="" \ + + +# Set up jupyterlab/ for UI to use, putting data/ and analysis/ at the root of the UI +WORKDIR /home/jovyan/ RUN mkdir jupyterlab WORKDIR /home/jovyan/jupyterlab RUN ln -s ../analysis analysis RUN ln -s ../data data +WORKDIR /home/jovyan/environment # Install Python packages from requirements.txt file -WORKDIR /home/jovyan/environment RUN pip install --quiet --no-cache-dir --requirement ./requirements.txt && \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" @@ -25,5 +42,5 @@ RUN ./r-packages.R # Install JupyterLab extensions RUN ./install-jupyter-extensions.sh - +# Put everything back so the UI doesn't get confused WORKDIR /home/jovyan/ diff --git a/environment/README.md b/environment/README.md index 3d4183d..6f246f4 100644 --- a/environment/README.md +++ b/environment/README.md @@ -16,8 +16,8 @@ Otherwise, you may proceed to [Starting a stopped container](#Starting-a-stopped At the command line: ```bash -docker compose -f environment/compose.yml build -docker compose -f environment/compose.yml up -d +./environment/docker-environment-common.sh build +./environment/docker-environment-common.sh up ``` When you bring up JupyterLab, you may see this warning. That's okay, you can ignore it. @@ -30,22 +30,33 @@ WARN[0000] Found orphan containers ([]) for this project. I To start the JupyterLab Docker container, at the command line: ```bash -docker compose -f environment/compose.yml start +./environment/docker-environment-common.sh start ``` ### Accessing JupyterLab -When it's done spinning up, the container will be accessible via any webbrowser on the host machine at `http://localhost:PORT_NUMBER/` (eg. http://localhost:10000). +When it's done spinning up, the container will be accessible via any web browser on the host machine at `http://localhost:PORT_NUMBER/` (eg. http://localhost:10000). ## Stopping a container To stop the JupyterLab Docker container, at the command line: ```bash -docker compose -f environment/compose.yml stop +./environment/docker-environment-common.sh stop ``` +## View container logs + +To view the JupyterLab Docker container logs, at the command line: +```bash +./environment/docker-environment-common.sh logs +``` + +Exit using `^C` (control + c) as many times as required. + + + # Structure ## Docker elements @@ -80,6 +91,10 @@ Additional JupyterLab extensions to be installed when the container is built. ## Supporting files +### `docker-environment-common.sh` + +This script helps abstract away the details of Docker, while maintaining a versioning/tagging system on the Docker images and containers. It is there to help make interacting with the Docker environment easier. + ### `install-jupyter-extensions.sh` This script installs the JupyterLab extensions, and is run by the `Dockerfile`. @@ -88,5 +103,3 @@ This script installs the JupyterLab extensions, and is run by the `Dockerfile`. ### `jupyter_server_config.py` This is where you can make changes to the Jupyter server configuration. By default, it sets the JupyterLab UI to launch in the `jupyterlab` folder of the main project and removes server authentication since this project is only meant to be run on a researcher's computer and not in a shared or production environment. - - diff --git a/environment/compose.yml b/environment/compose.yml index df59080..2acc2c1 100644 --- a/environment/compose.yml +++ b/environment/compose.yml @@ -1,6 +1,7 @@ version: '3.9' services: datascience-notebook: + image: datascience-notebook:${TAG} # Specify where the project parent directory is, and where the Dockerfile # is within that structure. build: diff --git a/environment/docker-environment-common.sh b/environment/docker-environment-common.sh new file mode 100755 index 0000000..b251b46 --- /dev/null +++ b/environment/docker-environment-common.sh @@ -0,0 +1,94 @@ +#! /bin/bash + +# Controls to help generalize Docker commands. + +# Usage information +function help() { + local bold=$(tput bold) + local normal=$(tput sgr0) + echo -e "${bold}Usage:${normal} $0 {start|stop|build|up|down|logs}" + echo + echo -e "Manage interactions with Docker / Compose" +} + +# Retrieve the appropriate image/container tag and generate the `docker compose` command. +function retrieve_tag() { + TAG_FILE="environment/image-tags" + if [ ! -f "$TAG_FILE" ]; then + echo "The tagfile $TAG_FILE does not exist, make sure to build the image!" + exit 3 + fi + TAG=$(head -n1 $TAG_FILE) + + if [ -z "${TAG}" ]; then + echo "The tag in $TAG_FILE is empty, make sure to build the image!" + exit 4 + fi + export TAG +} + + +# Make sure Docker is running +if ! docker info > /dev/null 2>&1; then + echo "This script uses Docker, and it isn't running - please start Docker and try again!" + exit 1 +fi + +# Detect what action to take. Everything is handled pretty identically, aside from `build`. +case "$1" in + logs) + retrieve_tag + CMD="docker compose -f environment/compose.yml logs -f" + ;; + + start) + retrieve_tag + CMD="docker compose -f environment/compose.yml start" + ;; + + stop) + retrieve_tag + CMD="docker compose -f environment/compose.yml stop" + ;; + up) + retrieve_tag + CMD="docker compose -f environment/compose.yml up -d" + ;; + + down) + retrieve_tag + CMD="docker compose -f environment/compose.yml down" + ;; + + build) + # Configure the image label information + ## Look at with + ### `docker image ls` to find the image + ### – given `image: datascience-notebook:$TAG` in compose.yml, the image name is `datascience-notebook` + ### `docker inspect datascience-notebook:2023-04-18 | grep "org.label-schema"` to inspect the image + ### - use the image name and the tag from `docker image ls` + ### You can also cross-reference the "IMAGE ID" from `docker image ls` with the "IMAGE" from `docker container ls` + + VCS_COMMIT_HASH=$(git log --pretty=format:'%h' -n 1) + NO_CACHE="--no-cache=true" + BUILD_DATE="--build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" + VCS_REF="--build-arg VCS_REF=$VCS_COMMIT_HASH" + # PROGRESS="--progress=plain" + + # Configure image tag information from the Git commit hash, used in compose.yml and in docker-environment-common.sh + export TAG=$VCS_COMMIT_HASH + echo $TAG > environment/image-tags + + # Build image + CMD="docker compose -f environment/compose.yml build $NO_CACHE $BUILD_DATE $VCS_REF $PROGRESS datascience-notebook" + ;; + + *) + echo -e "Invalid argument: $1" + help + exit 2 +esac + +# Do the thing +echo $CMD +$CMD From 7f1f8a5422600ed67a88b0ef275f034cf9eb296e Mon Sep 17 00:00:00 2001 From: Abby Drury Date: Thu, 20 Apr 2023 11:34:57 -0400 Subject: [PATCH 2/3] feat(environment): Update to use opencontainers.org label schema for Docker objects, add version and tag commands to the script for interacting with Compose --- environment/Dockerfile | 33 +++++++++------ environment/docker-environment-common.sh | 54 +++++++++++++----------- 2 files changed, 49 insertions(+), 38 deletions(-) diff --git a/environment/Dockerfile b/environment/Dockerfile index ca9e762..5bf8d10 100644 --- a/environment/Dockerfile +++ b/environment/Dockerfile @@ -7,21 +7,26 @@ FROM jupyter/datascience-notebook:9e63909e0317 WORKDIR /home/jovyan/ COPY --chown=${NB_UID}:${NB_GID} . . -# Tag the image +# Allow these build-args to be passed into the Dockerfile's context for tagging the image from environment/docker-environment-common.sh +## NOTE: You can update REPOSITORY_URL in environment/docker-environment-common.sh +ARG REPOSITORY_URL ARG BUILD_DATE -ARG VCS_REF -# -LABEL org.label-schema.build-date=$BUILD_DATE \ - org.label-schema.name="mtholyoke/docker-for-data-science" \ - org.label-schema.description="Standardized and reproducible data science research environment" \ - org.label-schema.url="https://github.com/mtholyoke/docker-data-science-environment/wiki" \ - org.label-schema.vcs-url="https://github.com/mtholyoke/docker-data-science-environment" \ - org.label-schema.vcs-ref=$VCS_REF \ - org.label-schema.vendor="Mount Holyoke College" \ - org.label-schema.version="1.1.0" \ - org.label-schema.schema-version="1.0.0-rc.1" -# org.label-schema.usage="" \ - +ARG REVISION +ARG LATEST_GIT_TAG +# Tag the image. Schema based on https://github.com/opencontainers/image-spec/blob/main/annotations.md. +## NOTE: org.opencontainers.image.version will always be the latest Git release tag +LABEL \ + org.opencontainers.image.created=$BUILD_DATE \ + org.opencontainers.image.authors="adrury@mtholyoke.edu" \ + org.opencontainers.image.url="$REPOSITORY_URL/blob/main/environment/README.md" \ + org.opencontainers.image.documentation="$REPOSITORY_URL/wiki" \ + org.opencontainers.image.source=$REPOSITORY_URL \ + org.opencontainers.image.revision=$REVISION \ + org.opencontainers.image.vendor="Trustees of Mount Holyoke College" \ + org.opencontainers.image.license="MIT" \ + org.opencontainers.image.title="Docker Data Science Environment" \ + org.opencontainers.image.description="Standardized and reproducible data science research environment" \ + org.opencontainers.image.version=$LATEST_GIT_TAG # Set up jupyterlab/ for UI to use, putting data/ and analysis/ at the root of the UI WORKDIR /home/jovyan/ diff --git a/environment/docker-environment-common.sh b/environment/docker-environment-common.sh index b251b46..56ca5e4 100755 --- a/environment/docker-environment-common.sh +++ b/environment/docker-environment-common.sh @@ -6,9 +6,10 @@ function help() { local bold=$(tput bold) local normal=$(tput sgr0) - echo -e "${bold}Usage:${normal} $0 {start|stop|build|up|down|logs}" + echo -e "${bold}Usage:${normal} $0 {start|stop|build|up|down|logs|images|version|labels|tags}" echo echo -e "Manage interactions with Docker / Compose" + echo -e "Some commands, like version and tags, will only work if the container is already running." } # Retrieve the appropriate image/container tag and generate the `docker compose` command. @@ -38,49 +39,58 @@ fi case "$1" in logs) retrieve_tag - CMD="docker compose -f environment/compose.yml logs -f" + docker compose -f environment/compose.yml logs -f ;; start) retrieve_tag - CMD="docker compose -f environment/compose.yml start" + docker compose -f environment/compose.yml start ;; stop) retrieve_tag - CMD="docker compose -f environment/compose.yml stop" + docker compose -f environment/compose.yml stop ;; up) retrieve_tag - CMD="docker compose -f environment/compose.yml up -d" + docker compose -f environment/compose.yml up -d ;; down) retrieve_tag - CMD="docker compose -f environment/compose.yml down" + docker compose -f environment/compose.yml down ;; + + images|version) + retrieve_tag + docker compose -f environment/compose.yml images + ;; + + labels|tags) + retrieve_tag + CONTAINER_ID=`docker compose -f environment/compose.yml ps -q` + if [ -z "$CONTAINER_ID" ]; then + echo -e "No running containers. Try \`$0 build\` and \`$0 up\`" + exit 5 + fi + docker inspect --format='{{ range $k,$v:=.Config.Labels }}{{ if eq (printf "%.24s" $k) "org.opencontainers.image" }}{{ $k }}: {{ $v }}{{ printf "\n" }}{{end}}{{end}}' $CONTAINER_ID + ;; + build) - # Configure the image label information - ## Look at with - ### `docker image ls` to find the image - ### – given `image: datascience-notebook:$TAG` in compose.yml, the image name is `datascience-notebook` - ### `docker inspect datascience-notebook:2023-04-18 | grep "org.label-schema"` to inspect the image - ### - use the image name and the tag from `docker image ls` - ### You can also cross-reference the "IMAGE ID" from `docker image ls` with the "IMAGE" from `docker container ls` - - VCS_COMMIT_HASH=$(git log --pretty=format:'%h' -n 1) + REPOSITORY_URL="https://github.com/mtholyoke/docker-data-science-environment" ## NOTE: Update this. + REVISION=$(git log --pretty=format:'%as_%h' -n 1) + LATEST_GIT_TAG="$(git describe --abbrev=0 --tags)" + BUILD_DATE="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" + # PROGRESS="--progress=plain" # Print debug information during Docker build phase NO_CACHE="--no-cache=true" - BUILD_DATE="--build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" - VCS_REF="--build-arg VCS_REF=$VCS_COMMIT_HASH" - # PROGRESS="--progress=plain" # Configure image tag information from the Git commit hash, used in compose.yml and in docker-environment-common.sh - export TAG=$VCS_COMMIT_HASH + export TAG=$REVISION_$LATEST_GIT_TAG echo $TAG > environment/image-tags # Build image - CMD="docker compose -f environment/compose.yml build $NO_CACHE $BUILD_DATE $VCS_REF $PROGRESS datascience-notebook" + docker compose -f environment/compose.yml build $NO_CACHE --build-arg BUILD_DATE=$BUILD_DATE --build-arg REVISION=$REVISION --build-arg REPOSITORY_URL=$REPOSITORY_URL --build-arg LATEST_GIT_TAG=$LATEST_GIT_TAG $PROGRESS datascience-notebook ;; *) @@ -88,7 +98,3 @@ case "$1" in help exit 2 esac - -# Do the thing -echo $CMD -$CMD From 7a7514674d5bcafaf1522d7e91a1c78dd4a495c7 Mon Sep 17 00:00:00 2001 From: Abby Drury Date: Thu, 20 Apr 2023 11:36:57 -0400 Subject: [PATCH 3/3] docs(environment): Add documentation for new version and tag commands to the script for interacting with Compose --- environment/README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/environment/README.md b/environment/README.md index 6f246f4..6379ae4 100644 --- a/environment/README.md +++ b/environment/README.md @@ -56,6 +56,20 @@ To view the JupyterLab Docker container logs, at the command line: Exit using `^C` (control + c) as many times as required. +## View container version + +To view the JupyterLab Docker container version, at the command line: +```bash +./environment/docker-environment-common.sh version +``` + + +## View container tags / labels + +To view the JupyterLab Docker container tags, at the command line: +```bash +./environment/docker-environment-common.sh tags +``` # Structure