diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000..4cfbec567 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,39 @@ +name: Build and publish Docker image + +on: + push: + branches: + - main + workflow_dispatch: + +jobs: + build: + name: Build image + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Login to container registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata for Docker + id: metadata + uses: docker/metadata-action@v4 + with: + images: ghcr.io/${{ github.repository_owner }}/oioioi + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build and publish image + uses: docker/build-push-action@v6 + with: + platforms: linux/amd64 + push: true + tags: ${{ steps.metadata.outputs.tags }} + labels: ${{ steps.metadata.outputs.labels }} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 347029593..957f430de 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10 +FROM python:3.10 AS base ENV PYTHONUNBUFFERED 1 @@ -33,7 +33,7 @@ RUN apt-get update && \ # This is placed here to avoid redownloading package on uid change ARG oioioi_uid=1234 -#Bash as shell, setup folders, create oioioi user +# Bash as shell, setup folders, create oioioi user RUN rm /bin/sh && ln -s /bin/bash /bin/sh && \ mkdir -pv /sio2/oioioi && \ mkdir -pv /sio2/sandboxes && \ @@ -66,23 +66,37 @@ RUN pip3 install -r requirements_static.txt --user COPY --chown=oioioi:oioioi . /sio2/oioioi - -ENV OIOIOI_DB_ENGINE 'django.db.backends.postgresql' -ENV RABBITMQ_HOST 'broker' -ENV RABBITMQ_PORT '5672' -ENV RABBITMQ_USER 'oioioi' -ENV RABBITMQ_PASSWORD 'oioioi' -ENV FILETRACKER_LISTEN_ADDR '0.0.0.0' -ENV FILETRACKER_LISTEN_PORT '9999' -ENV FILETRACKER_URL 'http://web:9999' - RUN oioioi-create-config /sio2/deployment WORKDIR /sio2/deployment RUN mkdir -p /sio2/deployment/logs/{supervisor,runserver} -# Download sandboxes +# The stage below is independent of base and can be built in parallel to optimize build time. +FROM python:3.10 AS development-sandboxes + +ENV DOWNLOAD_DIR=/sio2/sandboxes +ENV MANIFEST_URL=https://downloads.sio2project.mimuw.edu.pl/sandboxes/Manifest + +# Download the file and invalidate the cache if the Manifest checksum changes. +ADD $MANIFEST_URL /sio2/Manifest + +RUN apt-get update && \ + apt-get install -y curl wget bash && \ + apt-get clean + +COPY download_sandboxes.sh /download_sandboxes.sh +RUN chmod +x /download_sandboxes.sh + +# Run script to download sandbox data from the given Manifest. +RUN ./download_sandboxes.sh -q -y -d $DOWNLOAD_DIR -m $MANIFEST_URL + +FROM base AS development + +COPY --from=development-sandboxes /sio2/sandboxes /sio2/sandboxes +RUN chmod +x /sio2/oioioi/download_sandboxes.sh + RUN ./manage.py supervisor > /dev/null --daemonize --nolaunch=uwsgi && \ - ./manage.py download_sandboxes -q -y -c /sio2/sandboxes && \ + /sio2/oioioi/wait-for-it.sh -t 60 "127.0.0.1:9999" && \ + ./manage.py upload_sandboxes_to_filetracker -d /sio2/sandboxes && \ ./manage.py supervisor stop all diff --git a/README.rst b/README.rst index 2b31c3c2e..1183480ef 100644 --- a/README.rst +++ b/README.rst @@ -27,6 +27,12 @@ as described `in Docker docs`_. .. _in Docker docs: https://docs.docker.com/compose/reference/up/ +Docker image +============ +.. _official Docker image: https://github.com/sio2project/oioioi/pkgs/container/oioioi + +An `official Docker image`_ for oioioi is available on the GitHub Container Registry. + Docker (for development) ~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 37a93b3c8..9c2b8f4fa 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -14,10 +14,22 @@ services: build: context: . dockerfile: Dockerfile + target: development args: - "oioioi_uid=${OIOIOI_UID}" extra_hosts: - "web:127.0.0.1" + environment: + OIOIOI_DB_ENGINE: 'django.db.backends.postgresql' + RABBITMQ_HOST: 'broker' + RABBITMQ_PORT: '5672' + RABBITMQ_USER: 'oioioi' + RABBITMQ_PASSWORD: 'oioioi' + FILETRACKER_LISTEN_ADDR: '0.0.0.0' + FILETRACKER_LISTEN_PORT: '9999' + FILETRACKER_URL: 'http://web:9999' + DATABASE_HOST: 'db' + DATABASE_PORT: '5432' ports: # web server - "8000:8000" diff --git a/docker-compose.yml b/docker-compose.yml index 4eb9cc100..197d23505 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,6 +11,17 @@ services: web: image: sio2project/oioioi:$OIOIOI_VERSION command: ["/sio2/oioioi/oioioi_init.sh"] + environment: + OIOIOI_DB_ENGINE: 'django.db.backends.postgresql' + RABBITMQ_HOST: 'broker' + RABBITMQ_PORT: '5672' + RABBITMQ_USER: 'oioioi' + RABBITMQ_PASSWORD: 'oioioi' + FILETRACKER_LISTEN_ADDR: '0.0.0.0' + FILETRACKER_LISTEN_PORT: '9999' + FILETRACKER_URL: 'http://web:9999' + DATABASE_HOST: 'db' + DATABASE_PORT: '5432' ports: - "8000:8000" stop_grace_period: 3m diff --git a/download_sandboxes.sh b/download_sandboxes.sh new file mode 100644 index 000000000..ce4ee4dc9 --- /dev/null +++ b/download_sandboxes.sh @@ -0,0 +1,166 @@ +#!/bin/bash + +DEFAULT_MANIFEST_URL="https://downloads.sio2project.mimuw.edu.pl/sandboxes/Manifest" +DEFAULT_DOWNLOAD_DIR="sandboxes-download" +DEFAULT_WGET="wget" +QUIET=false +AGREE_LICENSE=false + +echoerr() { echo "$@" 1>&2; } + +usage() { + echo "Usage: $0 [options] [sandbox1 sandbox2 ...]" + echo "" + echo "Options:" + echo " -m, --manifest URL Specifies URL with the Manifest file listing available sandboxes (default: $DEFAULT_MANIFEST_URL)" + echo " -d, --download-dir DIR Specify the download directory (default: $DEFAULT_DOWNLOAD_DIR)" + echo " -c, --cache-dir DIR Load cached sandboxes from a local directory (default: None)" + echo " --wget PATH Specify the wget binary to use (default: $DEFAULT_WGET)" + echo " -y, --yes Enabling this options means that you agree to the license terms and conditions, so no license prompt will be displayed" + echo " -q, --quiet Disables wget interactive progress bars" + echo " -h, --help Display this help message" + exit 1 +} + +while [[ $# -gt 0 ]]; do + case "$1" in + -m|--manifest) + MANIFEST_URL="$2" + shift 2 + ;; + -d|--download-dir) + DOWNLOAD_DIR="$2" + shift 2 + ;; + -c|--cache-dir) + CACHE_DIR="$2" + shift 2 + ;; + --wget) + WGET_CMD="$2" + shift 2 + ;; + -y|--yes) + AGREE_LICENSE=true + shift + ;; + -q|--quiet) + QUIET=true + shift + ;; + -h|--help) + usage + ;; + --) + shift + break + ;; + -*) + echoerr "Unknown argument: $1" + usage + ;; + *) + break + ;; + esac +done + +MANIFEST_URL="${MANIFEST_URL:-$DEFAULT_MANIFEST_URL}" +DOWNLOAD_DIR="${DOWNLOAD_DIR:-$DEFAULT_DOWNLOAD_DIR}" +WGET_CMD="${WGET_CMD:-$DEFAULT_WGET}" + +SANDBOXES=("$@") + + +if ! MANIFEST_CONTENT=$(curl -fsSL "$MANIFEST_URL"); then + echoerr "Error: Unable to download manifest from $MANIFEST_URL" + exit 1 +fi + +IFS=$'\n' read -d '' -r -a MANIFEST <<< "$MANIFEST_CONTENT" + + +BASE_URL=$(dirname "$MANIFEST_URL")/ +LICENSE_URL="${BASE_URL}LICENSE" + +LICENSE_CONTENT=$(curl -fsSL "$LICENSE_URL") +LICENSE_STATUS=$? + +if [[ $LICENSE_STATUS -eq 0 ]]; then + if ! $AGREE_LICENSE; then + echoerr "" + echoerr "The sandboxes are accompanied with a license:" + echoerr "$LICENSE_CONTENT" + while true; do + read -rp "Do you accept the license? (yes/no): " yn + case "$yn" in + yes ) break;; + no ) echoerr "License not accepted. Exiting..."; exit 1;; + * ) echoerr 'Please enter either "yes" or "no".';; + esac + done + fi +elif [[ $LICENSE_STATUS -ne 22 ]]; then + echoerr "Error: Unable to download LICENSE from $LICENSE_URL" + exit 1 +fi + +if [[ ${#SANDBOXES[@]} -eq 0 ]]; then + SANDBOXES=("${MANIFEST[@]}") +fi + + +URLS=() +for SANDBOX in "${SANDBOXES[@]}"; do + found=false + for item in "${MANIFEST[@]}"; do + if [[ "$item" == "$SANDBOX" ]]; then + found=true + break + fi + done + + if [[ $found == false ]]; then + echoerr "Error: Sandbox '$SANDBOX' not available (not in Manifest)" + exit 1 + fi + + echo "$SANDBOX"; + + BASENAME="${SANDBOX}.tar.gz" + + if [[ -n "$CACHE_DIR" && -f "$CACHE_DIR/$BASENAME" ]]; then + continue + fi + + URL="${BASE_URL}${BASENAME}" + URLS+=("$URL") +done + +if [[ ! -d "$DOWNLOAD_DIR" ]]; then + if ! mkdir -p "$DOWNLOAD_DIR"; then + echoerr "Error: Unable to create download directory '$DOWNLOAD_DIR'" + exit 1 + fi +fi + +if ! command -v "$WGET_CMD" &> /dev/null; then + echoerr "Error: '$WGET_CMD' is not installed or not in PATH." + exit 1 +fi + +WGET_OPTIONS=("--no-check-certificate") +if $QUIET; then + WGET_OPTIONS+=("-nv") +fi + +for URL in "${URLS[@]}"; do + BASENAME=$(basename "$URL") + OUTPUT_PATH="$DOWNLOAD_DIR/$BASENAME" + if ! "$WGET_CMD" "${WGET_OPTIONS[@]}" -O "$OUTPUT_PATH" "$URL"; then + echoerr "Error: Failed to download $BASENAME" + exit 1 + fi +done + +exit 0 diff --git a/oioioi/sioworkers/management/commands/download_sandboxes.py b/oioioi/sioworkers/management/commands/download_sandboxes.py deleted file mode 100644 index d061d1f8b..000000000 --- a/oioioi/sioworkers/management/commands/download_sandboxes.py +++ /dev/null @@ -1,182 +0,0 @@ -from __future__ import print_function - -import os -import os.path - -import urllib.error -import urllib.parse -import urllib.request -from django.conf import settings -from django.core.management.base import BaseCommand, CommandError - -from oioioi.base.utils.execute import ExecuteError, execute -from oioioi.filetracker.client import get_client - -DEFAULT_SANDBOXES_MANIFEST = getattr( - settings, - 'SANDBOXES_MANIFEST', - 'https://downloads.sio2project.mimuw.edu.pl/sandboxes/Manifest', -) - - -class Command(BaseCommand): - def add_arguments(self, parser): - parser.add_argument( - '-m', - '--manifest', - metavar='URL', - dest='manifest_url', - default=DEFAULT_SANDBOXES_MANIFEST, - help="Specifies URL with the Manifest file listing available sandboxes", - ) - parser.add_argument( - '-c', - '--cache-dir', - metavar='DIR', - dest='cache_dir', - default=None, - help="Load cached sandboxes from a local directory", - ) - parser.add_argument( - '-d', - '--download-dir', - metavar='DIR', - dest='download_dir', - default="sandboxes-download", - help="Temporary directory where the downloaded files will be stored", - ) - parser.add_argument( - '--wget', - metavar='PATH', - dest='wget', - default="wget", - help="Specifies the wget binary to use", - ) - parser.add_argument( - '-y', - '--yes', - dest='license_agreement', - default=False, - action='store_true', - help="Enabling this options means that you agree to the license " - "terms and conditions, so no license prompt will be " - "displayed", - ) - parser.add_argument( - '-q', - '--quiet', - dest='quiet', - default=False, - action='store_true', - help="Disables wget interactive progress bars", - ) - parser.add_argument( - 'sandboxes', type=str, nargs='*', help='List of sandboxes to be downloaded' - ) - - help = "Downloads sandboxes and stores them in the Filetracker." - - requires_model_validation = False - - def display_license(self, license): - print("\nThe sandboxes are accompanied with a license:\n", file=self.stdout) - self.stdout.write(license) - msg = "\nDo you accept the license? (yes/no):" - confirm = input(msg) - while 1: - if confirm not in ('yes', 'no'): - confirm = input('Please enter either "yes" or "no": ') - continue - if confirm == 'no': - raise CommandError("License not accepted") - break - - def handle(self, *args, **options): - print("--- Downloading Manifest ...", file=self.stdout) - try: - manifest_url = options['manifest_url'] - manifest = ( - urllib.request.urlopen(manifest_url).read().decode('utf-8') - ) - manifest = manifest.strip().splitlines() - except Exception as e: - raise CommandError("Error downloading manifest: %s" % (e,)) - - print("--- Looking for license ...", file=self.stdout) - try: - license_url = urllib.parse.urljoin(manifest_url, 'LICENSE') - license = ( - urllib.request.urlopen(license_url).read().decode('utf-8') - ) - if not options['license_agreement']: - self.display_license(license) - except urllib.error.HTTPError as e: - if e.code != 404: - raise - - args = options['sandboxes'] - if not args: - args = manifest - - print("--- Preparing ...", file=self.stdout) - urls = [] - cached_args = [] - for arg in args: - basename = arg + '.tar.gz' - if options['cache_dir']: - path = os.path.join(options['cache_dir'], basename) - if os.path.isfile(path): - cached_args.append(arg) - continue - if arg not in manifest: - raise CommandError( - "Sandbox '%s' not available (not in Manifest)" % (arg,) - ) - urls.append(urllib.parse.urljoin(manifest_url, basename)) - - filetracker = get_client() - - download_dir = options['download_dir'] - if not os.path.exists(download_dir): - os.makedirs(download_dir) - - try: - execute([options['wget'], '--version']) - except ExecuteError: - raise CommandError( - "Wget not working. Please specify a working " - "Wget binary using --wget option." - ) - - if len(urls) > 0: - print("--- Downloading sandboxes ...", file=self.stdout) - - quiet_flag = ['-nv'] if options['quiet'] else [] - execute( - [options['wget'], '-N', '--no-check-certificate', '-i', '-'] + quiet_flag, - stdin='\n'.join(urls).encode('utf-8'), - capture_output=False, - cwd=download_dir, - ) - - print("--- Saving sandboxes to the Filetracker ...", file=self.stdout) - for arg in args: - basename = arg + '.tar.gz' - if arg in cached_args: - local_file = os.path.join(options['cache_dir'], basename) - else: - local_file = os.path.join(download_dir, basename) - print(" ", basename, file=self.stdout) - filetracker.put_file('/sandboxes/' + basename, local_file) - if arg not in cached_args: - os.unlink(local_file) - - try: - os.rmdir(download_dir) - except OSError: - print( - "--- Done, but couldn't remove the downloads directory.", - file=self.stdout, - ) - else: - print("--- Done.", file=self.stdout) diff --git a/oioioi/sioworkers/management/commands/upload_sandboxes_to_filetracker.py b/oioioi/sioworkers/management/commands/upload_sandboxes_to_filetracker.py new file mode 100644 index 000000000..d0573e359 --- /dev/null +++ b/oioioi/sioworkers/management/commands/upload_sandboxes_to_filetracker.py @@ -0,0 +1,37 @@ +from __future__ import print_function + +import os +import os.path + +from django.core.management.base import BaseCommand + +from oioioi.filetracker.client import get_client + + +class Command(BaseCommand): + def add_arguments(self, parser): + parser.add_argument( + '-d', + '--sandboxes-dir', + metavar='DIR', + dest='sandboxes_dir', + default=None, + help="Load sandboxes from a local directory", + ) + + help = "Upload sandboxes to the Filetracker." + + def handle(self, *args, **options): + filetracker = get_client() + + print("--- Saving sandboxes to the Filetracker ...", file=self.stdout) + + sandboxes_dir = os.fsencode(options['sandboxes_dir']) + for file in os.listdir(sandboxes_dir): + filename = os.fsdecode(file) + if not filename.endswith(".tar.gz"): + continue + + filetracker.put_file('/sandboxes/' + filename, os.path.join(options['sandboxes_dir'], filename)) + + print("--- Done.", file=self.stdout) diff --git a/worker_init.sh b/worker_init.sh index 32911c01a..cb10a0da0 100755 --- a/worker_init.sh +++ b/worker_init.sh @@ -4,13 +4,12 @@ set -x sudo apt install -y proot -/sio2/oioioi/wait-for-it.sh -t 60 "db:5432" +/sio2/oioioi/wait-for-it.sh -t 60 "${DATABASE_HOST}:${DATABASE_PORT}" /sio2/oioioi/wait-for-it.sh -t 0 "web:8000" mkdir -pv /sio2/deployment/logs/database echo "LOG: Launching worker at `hostname`" -export FILETRACKER_URL="http://web:9999" exec python3 $(which twistd) --nodaemon --pidfile=/home/oioioi/worker.pid \ -l /sio2/deployment/logs/worker`hostname`.log worker \ --can-run-cpu-exec \