Refactor docker setup

.github/workflows/build.yml

@@ -0,0 +1,39 @@
+name: Build and publish Docker image
+
+on:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+jobs:
+  build:
+    name: Build image
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Login to container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata for Docker
+        id: metadata
+        uses: docker/metadata-action@v4
+        with:
+          images: ghcr.io/${{ github.repository_owner }}/oioioi
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build and publish image
+        uses: docker/build-push-action@v6
+        with:
+          platforms: linux/amd64
+          push: true
+          tags: ${{ steps.metadata.outputs.tags }}
+          labels: ${{ steps.metadata.outputs.labels }}

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.10
+FROM python:3.10 AS base
 
 ENV PYTHONUNBUFFERED 1
 
@@ -33,7 +33,7 @@ RUN apt-get update && \
 # This is placed here to avoid redownloading package on uid change
 ARG oioioi_uid=1234
 
-#Bash as shell, setup folders, create oioioi user
+# Bash as shell, setup folders, create oioioi user
 RUN rm /bin/sh && ln -s /bin/bash /bin/sh && \
     mkdir -pv /sio2/oioioi && \
     mkdir -pv /sio2/sandboxes && \
@@ -66,23 +66,37 @@ RUN pip3 install -r requirements_static.txt --user
 
 COPY --chown=oioioi:oioioi . /sio2/oioioi
 
-
-ENV OIOIOI_DB_ENGINE 'django.db.backends.postgresql'
-ENV RABBITMQ_HOST 'broker'
-ENV RABBITMQ_PORT '5672'
-ENV RABBITMQ_USER 'oioioi'
-ENV RABBITMQ_PASSWORD 'oioioi'
-ENV FILETRACKER_LISTEN_ADDR '0.0.0.0'
-ENV FILETRACKER_LISTEN_PORT '9999'
-ENV FILETRACKER_URL 'http://web:9999'
-
 RUN oioioi-create-config /sio2/deployment
 
 WORKDIR /sio2/deployment
 
 RUN mkdir -p /sio2/deployment/logs/{supervisor,runserver}
 
-# Download sandboxes
+# The stage below is independent of base and can be built in parallel to optimize build time.
+FROM python:3.10 AS development-sandboxes
+
+ENV DOWNLOAD_DIR=/sio2/sandboxes
+ENV MANIFEST_URL=https://downloads.sio2project.mimuw.edu.pl/sandboxes/Manifest
+
+# Download the file and invalidate the cache if the Manifest checksum changes.
+ADD $MANIFEST_URL /sio2/Manifest
+
+RUN apt-get update && \
+    apt-get install -y curl wget bash && \
+    apt-get clean
+
+COPY download_sandboxes.sh /download_sandboxes.sh
+RUN chmod +x /download_sandboxes.sh
+
+# Run script to download sandbox data from the given Manifest.
+RUN ./download_sandboxes.sh -q -y -d $DOWNLOAD_DIR -m $MANIFEST_URL
+
+FROM base AS development
+
+COPY --from=development-sandboxes /sio2/sandboxes /sio2/sandboxes
+RUN chmod +x /sio2/oioioi/download_sandboxes.sh
+
 RUN ./manage.py supervisor > /dev/null --daemonize --nolaunch=uwsgi && \
-    ./manage.py download_sandboxes -q -y -c /sio2/sandboxes && \
+    /sio2/oioioi/wait-for-it.sh -t 60 "127.0.0.1:9999" && \
+    ./manage.py upload_sandboxes_to_filetracker -d /sio2/sandboxes && \
     ./manage.py supervisor stop all

README.rst

@@ -27,6 +27,12 @@ as described `in Docker docs`_.
 
 .. _in Docker docs: https://docs.docker.com/compose/reference/up/
 
+Docker image
+============
+.. _official Docker image: https://github.com/sio2project/oioioi/pkgs/container/oioioi
+
+An `official Docker image`_ for oioioi is available on the GitHub Container Registry.
+
 Docker (for development)
 ~~~~~~~~~~~~~~~~~~~~~~~
 

docker-compose-dev.yml

@@ -14,10 +14,22 @@ services:
     build:
       context: .
       dockerfile: Dockerfile
+      target: development
       args:
         - "oioioi_uid=${OIOIOI_UID}"
       extra_hosts:
         - "web:127.0.0.1"
+    environment:
+      OIOIOI_DB_ENGINE: 'django.db.backends.postgresql'
+      RABBITMQ_HOST: 'broker'
+      RABBITMQ_PORT: '5672'
+      RABBITMQ_USER: 'oioioi'
+      RABBITMQ_PASSWORD: 'oioioi'
+      FILETRACKER_LISTEN_ADDR: '0.0.0.0'
+      FILETRACKER_LISTEN_PORT: '9999'
+      FILETRACKER_URL: 'http://web:9999'
+      DATABASE_HOST: 'db'
+      DATABASE_PORT: '5432'
     ports:
     # web server
       - "8000:8000"

docker-compose.yml

@@ -11,6 +11,17 @@ services:
   web:
     image: sio2project/oioioi:$OIOIOI_VERSION
     command: ["/sio2/oioioi/oioioi_init.sh"]
+    environment:
+      OIOIOI_DB_ENGINE: 'django.db.backends.postgresql'
+      RABBITMQ_HOST: 'broker'
+      RABBITMQ_PORT: '5672'
+      RABBITMQ_USER: 'oioioi'
+      RABBITMQ_PASSWORD: 'oioioi'
+      FILETRACKER_LISTEN_ADDR: '0.0.0.0'
+      FILETRACKER_LISTEN_PORT: '9999'
+      FILETRACKER_URL: 'http://web:9999'
+      DATABASE_HOST: 'db'
+      DATABASE_PORT: '5432'
     ports:
       - "8000:8000"
     stop_grace_period: 3m

download_sandboxes.sh

@@ -0,0 +1,166 @@
+#!/bin/bash
+
+DEFAULT_MANIFEST_URL="https://downloads.sio2project.mimuw.edu.pl/sandboxes/Manifest"
+DEFAULT_DOWNLOAD_DIR="sandboxes-download"
+DEFAULT_WGET="wget"
+QUIET=false
+AGREE_LICENSE=false
+
+echoerr() { echo "$@" 1>&2; }
+
+usage() {
+    echo "Usage: $0 [options] [sandbox1 sandbox2 ...]"
+    echo ""
+    echo "Options:"
+    echo "  -m, --manifest URL       Specifies URL with the Manifest file listing available sandboxes (default: $DEFAULT_MANIFEST_URL)"
+    echo "  -d, --download-dir DIR   Specify the download directory (default: $DEFAULT_DOWNLOAD_DIR)"
+    echo "  -c, --cache-dir DIR   Load cached sandboxes from a local directory (default: None)"
+    echo "  --wget PATH              Specify the wget binary to use (default: $DEFAULT_WGET)"
+    echo "  -y, --yes                Enabling this options means that you agree to the license terms and conditions, so no license prompt will be displayed"
+    echo "  -q, --quiet              Disables wget interactive progress bars"
+    echo "  -h, --help               Display this help message"
+    exit 1
+}
+
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        -m|--manifest)
+            MANIFEST_URL="$2"
+            shift 2
+            ;;
+        -d|--download-dir)
+            DOWNLOAD_DIR="$2"
+            shift 2
+            ;;
+        -c|--cache-dir)
+            CACHE_DIR="$2"
+            shift 2
+            ;;
+        --wget)
+            WGET_CMD="$2"
+            shift 2
+            ;;
+        -y|--yes)
+            AGREE_LICENSE=true
+            shift
+            ;;
+        -q|--quiet)
+            QUIET=true
+            shift
+            ;;
+        -h|--help)
+            usage
+            ;;
+        --)
+            shift
+            break
+            ;;
+        -*)
+            echoerr "Unknown argument: $1"
+            usage
+            ;;
+        *)
+            break
+            ;;
+    esac
+done
+
+MANIFEST_URL="${MANIFEST_URL:-$DEFAULT_MANIFEST_URL}"
+DOWNLOAD_DIR="${DOWNLOAD_DIR:-$DEFAULT_DOWNLOAD_DIR}"
+WGET_CMD="${WGET_CMD:-$DEFAULT_WGET}"
+
+SANDBOXES=("$@")
+
+
+if ! MANIFEST_CONTENT=$(curl -fsSL "$MANIFEST_URL"); then
+    echoerr "Error: Unable to download manifest from $MANIFEST_URL"
+    exit 1
+fi
+
+IFS=$'\n' read -d '' -r -a MANIFEST <<< "$MANIFEST_CONTENT"
+
+
+BASE_URL=$(dirname "$MANIFEST_URL")/
+LICENSE_URL="${BASE_URL}LICENSE"
+
+LICENSE_CONTENT=$(curl -fsSL "$LICENSE_URL")
+LICENSE_STATUS=$?
+
+if [[ $LICENSE_STATUS -eq 0 ]]; then
+    if ! $AGREE_LICENSE; then
+        echoerr ""
+        echoerr "The sandboxes are accompanied with a license:"
+        echoerr "$LICENSE_CONTENT"
+        while true; do
+            read -rp "Do you accept the license? (yes/no): " yn
+            case "$yn" in
+                yes ) break;;
+                no ) echoerr "License not accepted. Exiting..."; exit 1;;
+                * ) echoerr 'Please enter either "yes" or "no".';;
+            esac
+        done
+    fi
+elif [[ $LICENSE_STATUS -ne 22 ]]; then
+    echoerr "Error: Unable to download LICENSE from $LICENSE_URL"
+    exit 1
+fi
+
+if [[ ${#SANDBOXES[@]} -eq 0 ]]; then
+    SANDBOXES=("${MANIFEST[@]}")
+fi
+
+
+URLS=()
+for SANDBOX in "${SANDBOXES[@]}"; do
+    found=false
+    for item in "${MANIFEST[@]}"; do
+        if [[ "$item" == "$SANDBOX" ]]; then
+            found=true
+            break
+        fi
+    done
+
+    if [[ $found == false ]]; then
+        echoerr "Error: Sandbox '$SANDBOX' not available (not in Manifest)"
+        exit 1
+    fi
+
+    echo "$SANDBOX";
+
+    BASENAME="${SANDBOX}.tar.gz"
+
+     if [[ -n "$CACHE_DIR" && -f "$CACHE_DIR/$BASENAME" ]]; then
+        continue
+    fi
+
+    URL="${BASE_URL}${BASENAME}"
+    URLS+=("$URL")
+done
+
+if [[ ! -d "$DOWNLOAD_DIR" ]]; then
+    if ! mkdir -p "$DOWNLOAD_DIR"; then
+        echoerr "Error: Unable to create download directory '$DOWNLOAD_DIR'"
+        exit 1
+    fi
+fi
+
+if ! command -v "$WGET_CMD" &> /dev/null; then
+    echoerr "Error: '$WGET_CMD' is not installed or not in PATH."
+    exit 1
+fi
+
+WGET_OPTIONS=("--no-check-certificate")
+if $QUIET; then
+    WGET_OPTIONS+=("-nv")
+fi
+
+for URL in "${URLS[@]}"; do
+    BASENAME=$(basename "$URL")
+    OUTPUT_PATH="$DOWNLOAD_DIR/$BASENAME"
+    if ! "$WGET_CMD" "${WGET_OPTIONS[@]}" -O "$OUTPUT_PATH" "$URL"; then
+        echoerr "Error: Failed to download $BASENAME"
+        exit 1
+    fi
+done
+
+exit 0