From 799f7c6632f44419d2f0fe434eada124e3b07f79 Mon Sep 17 00:00:00 2001 From: Iteron-dev Date: Tue, 28 Jan 2025 15:11:50 +0100 Subject: [PATCH 1/4] Refactor&Build: Update Dockerfile and implement sandbox caching --- Dockerfile | 37 ++-- docker-compose-dev.yml | 12 ++ docker-compose.yml | 11 ++ download_sandboxes.sh | 166 ++++++++++++++++++ .../management/commands/download_sandboxes.py | 101 ++++------- worker_init.sh | 2 +- 6 files changed, 250 insertions(+), 79 deletions(-) create mode 100644 download_sandboxes.sh diff --git a/Dockerfile b/Dockerfile index 347029593..3bc6e9e0f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10 +FROM python:3.10 AS base ENV PYTHONUNBUFFERED 1 @@ -66,23 +66,34 @@ RUN pip3 install -r requirements_static.txt --user COPY --chown=oioioi:oioioi . /sio2/oioioi - -ENV OIOIOI_DB_ENGINE 'django.db.backends.postgresql' -ENV RABBITMQ_HOST 'broker' -ENV RABBITMQ_PORT '5672' -ENV RABBITMQ_USER 'oioioi' -ENV RABBITMQ_PASSWORD 'oioioi' -ENV FILETRACKER_LISTEN_ADDR '0.0.0.0' -ENV FILETRACKER_LISTEN_PORT '9999' -ENV FILETRACKER_URL 'http://web:9999' - RUN oioioi-create-config /sio2/deployment WORKDIR /sio2/deployment RUN mkdir -p /sio2/deployment/logs/{supervisor,runserver} -# Download sandboxes +FROM python:3.10 AS development-sandboxes + +ENV DOWNLOAD_DIR=/sio2/sandboxes +ENV MANIFEST_URL=https://downloads.sio2project.mimuw.edu.pl/sandboxes/Manifest + +RUN apt-get update && \ + apt-get install -y curl wget bash && \ + apt-get clean + +ADD $MANIFEST_URL /sio2/Manifest + +COPY download_sandboxes.sh /download_sandboxes.sh +RUN chmod +x /download_sandboxes.sh + +RUN ./download_sandboxes.sh -q -y -d $DOWNLOAD_DIR -m $MANIFEST_URL + +FROM base AS development + +COPY --from=development-sandboxes /sio2/sandboxes /sio2/sandboxes +RUN chmod +x /sio2/oioioi/download_sandboxes.sh + RUN ./manage.py supervisor > /dev/null --daemonize --nolaunch=uwsgi && \ - ./manage.py download_sandboxes -q -y -c /sio2/sandboxes && \ + /sio2/oioioi/wait-for-it.sh -t 60 "127.0.0.1:9999" && \ + ./manage.py download_sandboxes -q -y -c /sio2/sandboxes -p /sio2/oioioi/download_sandboxes.sh && \ ./manage.py supervisor stop all diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 37a93b3c8..9c2b8f4fa 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -14,10 +14,22 @@ services: build: context: . dockerfile: Dockerfile + target: development args: - "oioioi_uid=${OIOIOI_UID}" extra_hosts: - "web:127.0.0.1" + environment: + OIOIOI_DB_ENGINE: 'django.db.backends.postgresql' + RABBITMQ_HOST: 'broker' + RABBITMQ_PORT: '5672' + RABBITMQ_USER: 'oioioi' + RABBITMQ_PASSWORD: 'oioioi' + FILETRACKER_LISTEN_ADDR: '0.0.0.0' + FILETRACKER_LISTEN_PORT: '9999' + FILETRACKER_URL: 'http://web:9999' + DATABASE_HOST: 'db' + DATABASE_PORT: '5432' ports: # web server - "8000:8000" diff --git a/docker-compose.yml b/docker-compose.yml index 4eb9cc100..197d23505 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,6 +11,17 @@ services: web: image: sio2project/oioioi:$OIOIOI_VERSION command: ["/sio2/oioioi/oioioi_init.sh"] + environment: + OIOIOI_DB_ENGINE: 'django.db.backends.postgresql' + RABBITMQ_HOST: 'broker' + RABBITMQ_PORT: '5672' + RABBITMQ_USER: 'oioioi' + RABBITMQ_PASSWORD: 'oioioi' + FILETRACKER_LISTEN_ADDR: '0.0.0.0' + FILETRACKER_LISTEN_PORT: '9999' + FILETRACKER_URL: 'http://web:9999' + DATABASE_HOST: 'db' + DATABASE_PORT: '5432' ports: - "8000:8000" stop_grace_period: 3m diff --git a/download_sandboxes.sh b/download_sandboxes.sh new file mode 100644 index 000000000..ce4ee4dc9 --- /dev/null +++ b/download_sandboxes.sh @@ -0,0 +1,166 @@ +#!/bin/bash + +DEFAULT_MANIFEST_URL="https://downloads.sio2project.mimuw.edu.pl/sandboxes/Manifest" +DEFAULT_DOWNLOAD_DIR="sandboxes-download" +DEFAULT_WGET="wget" +QUIET=false +AGREE_LICENSE=false + +echoerr() { echo "$@" 1>&2; } + +usage() { + echo "Usage: $0 [options] [sandbox1 sandbox2 ...]" + echo "" + echo "Options:" + echo " -m, --manifest URL Specifies URL with the Manifest file listing available sandboxes (default: $DEFAULT_MANIFEST_URL)" + echo " -d, --download-dir DIR Specify the download directory (default: $DEFAULT_DOWNLOAD_DIR)" + echo " -c, --cache-dir DIR Load cached sandboxes from a local directory (default: None)" + echo " --wget PATH Specify the wget binary to use (default: $DEFAULT_WGET)" + echo " -y, --yes Enabling this options means that you agree to the license terms and conditions, so no license prompt will be displayed" + echo " -q, --quiet Disables wget interactive progress bars" + echo " -h, --help Display this help message" + exit 1 +} + +while [[ $# -gt 0 ]]; do + case "$1" in + -m|--manifest) + MANIFEST_URL="$2" + shift 2 + ;; + -d|--download-dir) + DOWNLOAD_DIR="$2" + shift 2 + ;; + -c|--cache-dir) + CACHE_DIR="$2" + shift 2 + ;; + --wget) + WGET_CMD="$2" + shift 2 + ;; + -y|--yes) + AGREE_LICENSE=true + shift + ;; + -q|--quiet) + QUIET=true + shift + ;; + -h|--help) + usage + ;; + --) + shift + break + ;; + -*) + echoerr "Unknown argument: $1" + usage + ;; + *) + break + ;; + esac +done + +MANIFEST_URL="${MANIFEST_URL:-$DEFAULT_MANIFEST_URL}" +DOWNLOAD_DIR="${DOWNLOAD_DIR:-$DEFAULT_DOWNLOAD_DIR}" +WGET_CMD="${WGET_CMD:-$DEFAULT_WGET}" + +SANDBOXES=("$@") + + +if ! MANIFEST_CONTENT=$(curl -fsSL "$MANIFEST_URL"); then + echoerr "Error: Unable to download manifest from $MANIFEST_URL" + exit 1 +fi + +IFS=$'\n' read -d '' -r -a MANIFEST <<< "$MANIFEST_CONTENT" + + +BASE_URL=$(dirname "$MANIFEST_URL")/ +LICENSE_URL="${BASE_URL}LICENSE" + +LICENSE_CONTENT=$(curl -fsSL "$LICENSE_URL") +LICENSE_STATUS=$? + +if [[ $LICENSE_STATUS -eq 0 ]]; then + if ! $AGREE_LICENSE; then + echoerr "" + echoerr "The sandboxes are accompanied with a license:" + echoerr "$LICENSE_CONTENT" + while true; do + read -rp "Do you accept the license? (yes/no): " yn + case "$yn" in + yes ) break;; + no ) echoerr "License not accepted. Exiting..."; exit 1;; + * ) echoerr 'Please enter either "yes" or "no".';; + esac + done + fi +elif [[ $LICENSE_STATUS -ne 22 ]]; then + echoerr "Error: Unable to download LICENSE from $LICENSE_URL" + exit 1 +fi + +if [[ ${#SANDBOXES[@]} -eq 0 ]]; then + SANDBOXES=("${MANIFEST[@]}") +fi + + +URLS=() +for SANDBOX in "${SANDBOXES[@]}"; do + found=false + for item in "${MANIFEST[@]}"; do + if [[ "$item" == "$SANDBOX" ]]; then + found=true + break + fi + done + + if [[ $found == false ]]; then + echoerr "Error: Sandbox '$SANDBOX' not available (not in Manifest)" + exit 1 + fi + + echo "$SANDBOX"; + + BASENAME="${SANDBOX}.tar.gz" + + if [[ -n "$CACHE_DIR" && -f "$CACHE_DIR/$BASENAME" ]]; then + continue + fi + + URL="${BASE_URL}${BASENAME}" + URLS+=("$URL") +done + +if [[ ! -d "$DOWNLOAD_DIR" ]]; then + if ! mkdir -p "$DOWNLOAD_DIR"; then + echoerr "Error: Unable to create download directory '$DOWNLOAD_DIR'" + exit 1 + fi +fi + +if ! command -v "$WGET_CMD" &> /dev/null; then + echoerr "Error: '$WGET_CMD' is not installed or not in PATH." + exit 1 +fi + +WGET_OPTIONS=("--no-check-certificate") +if $QUIET; then + WGET_OPTIONS+=("-nv") +fi + +for URL in "${URLS[@]}"; do + BASENAME=$(basename "$URL") + OUTPUT_PATH="$DOWNLOAD_DIR/$BASENAME" + if ! "$WGET_CMD" "${WGET_OPTIONS[@]}" -O "$OUTPUT_PATH" "$URL"; then + echoerr "Error: Failed to download $BASENAME" + exit 1 + fi +done + +exit 0 diff --git a/oioioi/sioworkers/management/commands/download_sandboxes.py b/oioioi/sioworkers/management/commands/download_sandboxes.py index d061d1f8b..bcff2a710 100644 --- a/oioioi/sioworkers/management/commands/download_sandboxes.py +++ b/oioioi/sioworkers/management/commands/download_sandboxes.py @@ -2,6 +2,7 @@ import os import os.path +from subprocess import check_output import urllib.error import urllib.parse @@ -59,8 +60,8 @@ def add_arguments(self, parser): default=False, action='store_true', help="Enabling this options means that you agree to the license " - "terms and conditions, so no license prompt will be " - "displayed", + "terms and conditions, so no license prompt will be " + "displayed", ) parser.add_argument( '-q', @@ -70,6 +71,14 @@ def add_arguments(self, parser): action='store_true', help="Disables wget interactive progress bars", ) + parser.add_argument( + '-p', + '--script-path', + metavar='FILEPATH', + dest='script_path', + default=None, + help="Path to script that downloads the sandboxes", + ) parser.add_argument( 'sandboxes', type=str, nargs='*', help='List of sandboxes to be downloaded' ) @@ -92,87 +101,49 @@ def display_license(self, license): break def handle(self, *args, **options): - print("--- Downloading Manifest ...", file=self.stdout) - try: - manifest_url = options['manifest_url'] - manifest = ( - urllib.request.urlopen(manifest_url).read().decode('utf-8') - ) - manifest = manifest.strip().splitlines() - except Exception as e: - raise CommandError("Error downloading manifest: %s" % (e,)) - - print("--- Looking for license ...", file=self.stdout) - try: - license_url = urllib.parse.urljoin(manifest_url, 'LICENSE') - license = ( - urllib.request.urlopen(license_url).read().decode('utf-8') - ) - if not options['license_agreement']: - self.display_license(license) - except urllib.error.HTTPError as e: - if e.code != 404: - raise - + if not options.get('script_path'): + raise CommandError("You must specify a script path") + + license_agreement = "" + if options['license_agreement']: + license_agreement = "-y" + cache_dir = "-c " + if options['cache_dir']: + cache_dir += options['cache_dir'] + + args_str = " ".join(args) + manifest_output = check_output( + f"{options['script_path']} -m {options['manifest_url']} -d {options['download_dir']} --wget {options['wget']} -q {license_agreement} {cache_dir} {args_str}", + shell=True, text=True) + if manifest_output == "": + raise CommandError(f"Manifest output cannot be empty") + + manifest = manifest_output.strip().splitlines() args = options['sandboxes'] if not args: args = manifest - print("--- Preparing ...", file=self.stdout) - urls = [] - cached_args = [] - for arg in args: - basename = arg + '.tar.gz' - if options['cache_dir']: - path = os.path.join(options['cache_dir'], basename) - if os.path.isfile(path): - cached_args.append(arg) - continue - if arg not in manifest: - raise CommandError( - "Sandbox '%s' not available (not in Manifest)" % (arg,) - ) - urls.append(urllib.parse.urljoin(manifest_url, basename)) + print("--- Preparing to save sandboxes to the Filetracker ...", file=self.stdout) + cached_args = [ + arg for arg in args + if options['cache_dir'] and os.path.isfile(os.path.join(options['cache_dir'], arg + '.tar.gz')) + ] filetracker = get_client() - download_dir = options['download_dir'] - if not os.path.exists(download_dir): - os.makedirs(download_dir) - - try: - execute([options['wget'], '--version']) - except ExecuteError: - raise CommandError( - "Wget not working. Please specify a working " - "Wget binary using --wget option." - ) - - if len(urls) > 0: - print("--- Downloading sandboxes ...", file=self.stdout) - - quiet_flag = ['-nv'] if options['quiet'] else [] - execute( - [options['wget'], '-N', '--no-check-certificate', '-i', '-'] + quiet_flag, - stdin='\n'.join(urls).encode('utf-8'), - capture_output=False, - cwd=download_dir, - ) - print("--- Saving sandboxes to the Filetracker ...", file=self.stdout) for arg in args: basename = arg + '.tar.gz' if arg in cached_args: local_file = os.path.join(options['cache_dir'], basename) else: - local_file = os.path.join(download_dir, basename) - print(" ", basename, file=self.stdout) + local_file = os.path.join(options['download_dir'], basename) filetracker.put_file('/sandboxes/' + basename, local_file) if arg not in cached_args: os.unlink(local_file) try: - os.rmdir(download_dir) + os.rmdir(options['download_dir']) except OSError: print( "--- Done, but couldn't remove the downloads directory.", diff --git a/worker_init.sh b/worker_init.sh index 32911c01a..0234ac19c 100755 --- a/worker_init.sh +++ b/worker_init.sh @@ -4,7 +4,7 @@ set -x sudo apt install -y proot -/sio2/oioioi/wait-for-it.sh -t 60 "db:5432" +/sio2/oioioi/wait-for-it.sh -t 60 "${DATABASE_HOST}:${DATABASE_PORT}" /sio2/oioioi/wait-for-it.sh -t 0 "web:8000" mkdir -pv /sio2/deployment/logs/database From 4abe671d17b6c8aa7946315308c40a7d445c50ea Mon Sep 17 00:00:00 2001 From: Iteron-dev Date: Sun, 23 Feb 2025 16:33:21 +0100 Subject: [PATCH 2/4] Feat: Add build.yml workflow, mention Docker image in README.rst, and remove redundant export in worker_init.sh --- .github/workflows/build.yml | 37 +++++++++++++++++++++++++++++++++++++ README.rst | 6 ++++++ worker_init.sh | 1 - 3 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000..4f2e7d81c --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,37 @@ +name: Build and publish Docker image + +on: + push: + branches: + - main + workflow_dispatch: + +jobs: + build: + name: Build image + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Login to container registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata for Docker + id: metadata + uses: docker/metadata-action@v4 + with: + images: ghcr.io/${{ github.repository_owner }}/oioioi + + - name: Build and publish image + uses: docker/build-push-action@v3 + with: + platforms: linux/amd64 + push: true + tags: ${{ steps.metadata.outputs.tags }} + labels: ${{ steps.metadata.outputs.labels }} \ No newline at end of file diff --git a/README.rst b/README.rst index 2b31c3c2e..1183480ef 100644 --- a/README.rst +++ b/README.rst @@ -27,6 +27,12 @@ as described `in Docker docs`_. .. _in Docker docs: https://docs.docker.com/compose/reference/up/ +Docker image +============ +.. _official Docker image: https://github.com/sio2project/oioioi/pkgs/container/oioioi + +An `official Docker image`_ for oioioi is available on the GitHub Container Registry. + Docker (for development) ~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/worker_init.sh b/worker_init.sh index 0234ac19c..cb10a0da0 100755 --- a/worker_init.sh +++ b/worker_init.sh @@ -10,7 +10,6 @@ sudo apt install -y proot mkdir -pv /sio2/deployment/logs/database echo "LOG: Launching worker at `hostname`" -export FILETRACKER_URL="http://web:9999" exec python3 $(which twistd) --nodaemon --pidfile=/home/oioioi/worker.pid \ -l /sio2/deployment/logs/worker`hostname`.log worker \ --can-run-cpu-exec \ From 19c9eb13f5df9182becc59caa3f1aa3c1cab356a Mon Sep 17 00:00:00 2001 From: Iteron-dev Date: Sun, 23 Feb 2025 17:31:02 +0100 Subject: [PATCH 3/4] Feat: Add buildx to build.yml --- .github/workflows/build.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4f2e7d81c..4cfbec567 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,7 +16,7 @@ jobs: steps: - name: Login to container registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} @@ -27,9 +27,11 @@ jobs: uses: docker/metadata-action@v4 with: images: ghcr.io/${{ github.repository_owner }}/oioioi + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 - name: Build and publish image - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v6 with: platforms: linux/amd64 push: true From cd58cb16824b0682480c4b0965c83b91c6209380 Mon Sep 17 00:00:00 2001 From: Iteron-dev Date: Wed, 5 Mar 2025 09:23:29 +0100 Subject: [PATCH 4/4] =?UTF-8?q?Refactor:=20Add=20comments=20to=20Dockerfil?= =?UTF-8?q?e,=20rename=20download=5Fsandboxes.py=20to=20upload=5Fsandboxes?= =?UTF-8?q?=5Fto=5Ffiletracker.py,=20and=20update=20the=20command=E2=80=99?= =?UTF-8?q?s=20functionality?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 11 +- .../management/commands/download_sandboxes.py | 153 ------------------ .../upload_sandboxes_to_filetracker.py | 37 +++++ 3 files changed, 44 insertions(+), 157 deletions(-) delete mode 100644 oioioi/sioworkers/management/commands/download_sandboxes.py create mode 100644 oioioi/sioworkers/management/commands/upload_sandboxes_to_filetracker.py diff --git a/Dockerfile b/Dockerfile index 3bc6e9e0f..957f430de 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,7 +33,7 @@ RUN apt-get update && \ # This is placed here to avoid redownloading package on uid change ARG oioioi_uid=1234 -#Bash as shell, setup folders, create oioioi user +# Bash as shell, setup folders, create oioioi user RUN rm /bin/sh && ln -s /bin/bash /bin/sh && \ mkdir -pv /sio2/oioioi && \ mkdir -pv /sio2/sandboxes && \ @@ -72,20 +72,23 @@ WORKDIR /sio2/deployment RUN mkdir -p /sio2/deployment/logs/{supervisor,runserver} +# The stage below is independent of base and can be built in parallel to optimize build time. FROM python:3.10 AS development-sandboxes ENV DOWNLOAD_DIR=/sio2/sandboxes ENV MANIFEST_URL=https://downloads.sio2project.mimuw.edu.pl/sandboxes/Manifest +# Download the file and invalidate the cache if the Manifest checksum changes. +ADD $MANIFEST_URL /sio2/Manifest + RUN apt-get update && \ apt-get install -y curl wget bash && \ apt-get clean -ADD $MANIFEST_URL /sio2/Manifest - COPY download_sandboxes.sh /download_sandboxes.sh RUN chmod +x /download_sandboxes.sh +# Run script to download sandbox data from the given Manifest. RUN ./download_sandboxes.sh -q -y -d $DOWNLOAD_DIR -m $MANIFEST_URL FROM base AS development @@ -95,5 +98,5 @@ RUN chmod +x /sio2/oioioi/download_sandboxes.sh RUN ./manage.py supervisor > /dev/null --daemonize --nolaunch=uwsgi && \ /sio2/oioioi/wait-for-it.sh -t 60 "127.0.0.1:9999" && \ - ./manage.py download_sandboxes -q -y -c /sio2/sandboxes -p /sio2/oioioi/download_sandboxes.sh && \ + ./manage.py upload_sandboxes_to_filetracker -d /sio2/sandboxes && \ ./manage.py supervisor stop all diff --git a/oioioi/sioworkers/management/commands/download_sandboxes.py b/oioioi/sioworkers/management/commands/download_sandboxes.py deleted file mode 100644 index bcff2a710..000000000 --- a/oioioi/sioworkers/management/commands/download_sandboxes.py +++ /dev/null @@ -1,153 +0,0 @@ -from __future__ import print_function - -import os -import os.path -from subprocess import check_output - -import urllib.error -import urllib.parse -import urllib.request -from django.conf import settings -from django.core.management.base import BaseCommand, CommandError - -from oioioi.base.utils.execute import ExecuteError, execute -from oioioi.filetracker.client import get_client - -DEFAULT_SANDBOXES_MANIFEST = getattr( - settings, - 'SANDBOXES_MANIFEST', - 'https://downloads.sio2project.mimuw.edu.pl/sandboxes/Manifest', -) - - -class Command(BaseCommand): - def add_arguments(self, parser): - parser.add_argument( - '-m', - '--manifest', - metavar='URL', - dest='manifest_url', - default=DEFAULT_SANDBOXES_MANIFEST, - help="Specifies URL with the Manifest file listing available sandboxes", - ) - parser.add_argument( - '-c', - '--cache-dir', - metavar='DIR', - dest='cache_dir', - default=None, - help="Load cached sandboxes from a local directory", - ) - parser.add_argument( - '-d', - '--download-dir', - metavar='DIR', - dest='download_dir', - default="sandboxes-download", - help="Temporary directory where the downloaded files will be stored", - ) - parser.add_argument( - '--wget', - metavar='PATH', - dest='wget', - default="wget", - help="Specifies the wget binary to use", - ) - parser.add_argument( - '-y', - '--yes', - dest='license_agreement', - default=False, - action='store_true', - help="Enabling this options means that you agree to the license " - "terms and conditions, so no license prompt will be " - "displayed", - ) - parser.add_argument( - '-q', - '--quiet', - dest='quiet', - default=False, - action='store_true', - help="Disables wget interactive progress bars", - ) - parser.add_argument( - '-p', - '--script-path', - metavar='FILEPATH', - dest='script_path', - default=None, - help="Path to script that downloads the sandboxes", - ) - parser.add_argument( - 'sandboxes', type=str, nargs='*', help='List of sandboxes to be downloaded' - ) - - help = "Downloads sandboxes and stores them in the Filetracker." - - requires_model_validation = False - - def display_license(self, license): - print("\nThe sandboxes are accompanied with a license:\n", file=self.stdout) - self.stdout.write(license) - msg = "\nDo you accept the license? (yes/no):" - confirm = input(msg) - while 1: - if confirm not in ('yes', 'no'): - confirm = input('Please enter either "yes" or "no": ') - continue - if confirm == 'no': - raise CommandError("License not accepted") - break - - def handle(self, *args, **options): - if not options.get('script_path'): - raise CommandError("You must specify a script path") - - license_agreement = "" - if options['license_agreement']: - license_agreement = "-y" - cache_dir = "-c " - if options['cache_dir']: - cache_dir += options['cache_dir'] - - args_str = " ".join(args) - manifest_output = check_output( - f"{options['script_path']} -m {options['manifest_url']} -d {options['download_dir']} --wget {options['wget']} -q {license_agreement} {cache_dir} {args_str}", - shell=True, text=True) - if manifest_output == "": - raise CommandError(f"Manifest output cannot be empty") - - manifest = manifest_output.strip().splitlines() - args = options['sandboxes'] - if not args: - args = manifest - - print("--- Preparing to save sandboxes to the Filetracker ...", file=self.stdout) - cached_args = [ - arg for arg in args - if options['cache_dir'] and os.path.isfile(os.path.join(options['cache_dir'], arg + '.tar.gz')) - ] - - filetracker = get_client() - - print("--- Saving sandboxes to the Filetracker ...", file=self.stdout) - for arg in args: - basename = arg + '.tar.gz' - if arg in cached_args: - local_file = os.path.join(options['cache_dir'], basename) - else: - local_file = os.path.join(options['download_dir'], basename) - filetracker.put_file('/sandboxes/' + basename, local_file) - if arg not in cached_args: - os.unlink(local_file) - - try: - os.rmdir(options['download_dir']) - except OSError: - print( - "--- Done, but couldn't remove the downloads directory.", - file=self.stdout, - ) - else: - print("--- Done.", file=self.stdout) diff --git a/oioioi/sioworkers/management/commands/upload_sandboxes_to_filetracker.py b/oioioi/sioworkers/management/commands/upload_sandboxes_to_filetracker.py new file mode 100644 index 000000000..d0573e359 --- /dev/null +++ b/oioioi/sioworkers/management/commands/upload_sandboxes_to_filetracker.py @@ -0,0 +1,37 @@ +from __future__ import print_function + +import os +import os.path + +from django.core.management.base import BaseCommand + +from oioioi.filetracker.client import get_client + + +class Command(BaseCommand): + def add_arguments(self, parser): + parser.add_argument( + '-d', + '--sandboxes-dir', + metavar='DIR', + dest='sandboxes_dir', + default=None, + help="Load sandboxes from a local directory", + ) + + help = "Upload sandboxes to the Filetracker." + + def handle(self, *args, **options): + filetracker = get_client() + + print("--- Saving sandboxes to the Filetracker ...", file=self.stdout) + + sandboxes_dir = os.fsencode(options['sandboxes_dir']) + for file in os.listdir(sandboxes_dir): + filename = os.fsdecode(file) + if not filename.endswith(".tar.gz"): + continue + + filetracker.put_file('/sandboxes/' + filename, os.path.join(options['sandboxes_dir'], filename)) + + print("--- Done.", file=self.stdout)