Start hardening docker grading & optimize ci

.github/workflows/run-tests.yml

@@ -18,10 +18,18 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
+
+      - uses: docker/setup-buildx-action@v1
+        id: buildx
+        with:
+          install: true
+
+      - name: Expose GitHub Runtime
+        uses: crazy-max/ghaction-github-runtime@v1
 
       - name: Build testing Docker image
         run: |
-          make docker-test
+          make docker-ci-test
 
       - name: Run tests
         run: docker run -v /var/run/docker.sock:/var/run/docker.sock -v /tmp:/tmp otter-test bash -c "cd /home/otter-grader; pip install -r requirements-test.txt; coverage run -m test && coverage xml -i && cp ./coverage.xml /tmp/coverage.xml" && cp /tmp/coverage.xml ./coverage.xml

CHANGELOG.md

@@ -8,6 +8,7 @@
 * Updated intercell seeding in R per [#302](https://github.com/ucbds-infra/otter-grader/issues/302)
 * Fixed bug in test case point values for Assign R assignments per [#360](https://github.com/ucbds-infra/otter-grader/issues/360)
 * Enabled `solutions_pdf` and `template_pdf` in Assign R assignments per [#364](https://github.com/ucbds-infra/otter-grader/issues/364)
+* Added options to limit execution time of grading and permit network access
 
 **v3.0.6:** re-release of v3.0.5
 

Dockerfile

@@ -47,7 +47,8 @@ RUN wget -nv -O /tmp/fandol.zip https://mirrors.ctan.org/fonts/fandol.zip && \
     mkdir -p /usr/share/texlive/texmf-dist/fonts/opentype/public/fandol && \
     cp /tmp/fandol/fandol/*.otf /usr/share/texlive/texmf-dist/fonts/opentype/public/fandol && \
     mktexlsr && \
-    fc-cache
+    fc-cache && \
+    rm -rf /tmp/fandol /tmp/fandol.zip
 
 # Set the locale to UTF-8 to ensure that Unicode output is encoded correctly
 ENV LANG C.UTF-8

Makefile

@@ -5,15 +5,21 @@ DOCKER_VERSION := $(shell docker version --format '{{.Server.Version}}' | sed "s
 docs:
 	$(MAKE) -C docs html
 
-docker-test:
+docker-test-prepare: 
 	cp -r Dockerfile test-Dockerfile
 	printf "\nADD . /home/otter-grader\nRUN pip install /home/otter-grader" >> test-Dockerfile
 	printf "\nRUN cd /tmp && curl -sSL -O https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_VERSION}.tgz && tar zxf docker-${DOCKER_VERSION}.tgz && mv ./docker/docker /usr/local/bin && chmod +x /usr/local/bin/docker && rm -rf /tmp/*" >> test-Dockerfile
 	printf "\nCOPY --from=docker/buildx-bin /buildx /usr/libexec/docker/cli-plugins/docker-buildx" >> test-Dockerfile
 	printf "\nENV PYTHONUNBUFFERED 1" >> test-Dockerfile
+
+docker-test: docker-test-prepare
 	docker build . -t otter-test -f test-Dockerfile --cache-from ucbdsinfra/otter-grader:latest
 	rm test-Dockerfile
 
+docker-ci-test:	docker-test-prepare
+	docker buildx build . --load -t otter-test -f test-Dockerfile --cache-from=type=gha --cache-to=type=gha,mode=max
+	rm test-Dockerfile
+
 tutorial:
 	cd docs/tutorial; \
 	zip -r tutorial.zip submissions demo.ipynb requirements.txt -x "*.DS_Store"; \

otter/grade/__init__.py

@@ -12,7 +12,7 @@
 
 def main(*, path="./", output_dir="./", autograder="./autograder.zip", containers=None, 
          scripts=False, no_kill=False, debug=False, zips=False, image="ucbdsinfra/otter-grader", 
-         pdfs=False, verbose=False, prune=False, force=False):
+         pdfs=False, verbose=False, prune=False, force=False, timeout=None, network=True):
     """
     Runs Otter Grade
 
@@ -29,11 +29,13 @@ def main(*, path="./", output_dir="./", autograder="./autograder.zip", container
         no_kill (``bool``): whether to keep containers after grading is finished
         debug (``bool``): whether to print the stdout of each container
         zips (``bool``): whether the submissions are Otter-exported zip files
-        image (``bool``): base image from which to build grading image
+        image (``str``): base image from which to build grading image
         pdfs (``bool``): whether to copy notebook PDFs out of the containers
         verbose (``bool``): whether to log status messages to stdout
         prune (``bool``): whether to prune the grading images; if true, no grading is performed
         force (``bool``): whether to force-prune the images (do not ask for confirmation)
+        timeout (``int``): timeout in seconds for each container
+        network (``bool``): whether to enable networking in the containers
 
     Raises:
         ``AssertionError``: if invalid arguments are provided
@@ -63,7 +65,9 @@ def main(*, path="./", output_dir="./", autograder="./autograder.zip", container
         debug=debug,
         zips=zips,
         image=image,
-        pdfs=pdfs
+        pdfs=pdfs,
+        timeout=timeout,
+        network=network
     )
 
     if verbose:

otter/grade/containers.py

@@ -1,8 +1,5 @@
 """Docker container management for Otter Grade"""
-
-from python_on_whales import docker
 import glob
-import itertools
 import os
 import pandas as pd
 import pickle
@@ -11,6 +8,8 @@
 import tempfile
 
 from concurrent.futures import ThreadPoolExecutor, wait
+from python_on_whales import docker
+from typing import Optional
 
 from .utils import generate_hash, OTTER_DOCKER_IMAGE_TAG
 
@@ -29,8 +28,8 @@ def build_image(zip_path, base_image, tag):
     """
     image = OTTER_DOCKER_IMAGE_TAG + ":" + tag
     dockerfile = pkg_resources.resource_filename(__name__, "Dockerfile")
-    
-    if not docker.image.exists(image): 
+
+    if not docker.image.exists(image):
         print(f"Building new image using {base_image} as base image")
         docker.build(".", build_args={
             "ZIPPATH": zip_path,
@@ -43,7 +42,7 @@ def build_image(zip_path, base_image, tag):
 def launch_grade(zip_path, notebooks_dir, verbose=False, num_containers=None,
                  scripts=False, no_kill=False, output_path="./", debug=False, zips=False,
                  image="ucbdsinfra/otter-grader",
-                 pdfs=False):
+                 pdfs=False, timeout=None, network=True):
     """
     Grades notebooks in parallel Docker containers
 
@@ -66,6 +65,8 @@ def launch_grade(zip_path, notebooks_dir, verbose=False, num_containers=None,
         zips (``bool``, optional): whether the submissions are zip files formatted from ``Notebook.export``
         image (``str``, optional): a base image to use for building Docker images
         pdfs (``bool``, optional): whether to copy PDFs out of the containers
+        timeout (``int``): timeout in seconds for each container
+        network (``bool``): whether to enable networking in the containers
 
     Returns:
         ``list`` of ``pandas.core.frame.DataFrame``: the grades returned by each container spawned during
@@ -97,6 +98,8 @@ def launch_grade(zip_path, notebooks_dir, verbose=False, num_containers=None,
                 output_path=output_path,
                 debug=debug,
                 pdfs=pdfs,
+                timeout=timeout,
+                network=network,
             )
         ]
 
@@ -110,7 +113,8 @@ def launch_grade(zip_path, notebooks_dir, verbose=False, num_containers=None,
 # TODO: these arguments need to be updated. replace notebooks_dir with the path to the notebook that
 # this container will be grading
 def grade_assignments(submission_path, image="ucbdsinfra/otter-grader", verbose=False,
-                      scripts=False, no_kill=False, output_path="./", debug=False, pdfs=False):
+                      scripts=False, no_kill=False, output_path="./", debug=False, pdfs=False,
+                      timeout: Optional[int] = None, network=True):
     """
     Grades multiple submissions in a directory using a single docker container. If no PDF assignment is
     wanted, set all three PDF params (``unfiltered_pdfs``, ``tag_filter``, and ``html_filter``) to ``False``.
@@ -127,6 +131,8 @@ def grade_assignments(submission_path, image="ucbdsinfra/otter-grader", verbose=
         debug (``bool``, False): whether to run grading in debug mode (prints grading STDOUT and STDERR
             from each container to the command line)
         pdfs (``bool``, optional): whether to copy PDFs out of the containers
+        timeout (``int``): timeout in seconds for each container
+        network (``bool``): whether to enable networking in the containers
 
     Returns:
         ``pandas.core.frame.DataFrame``: A dataframe of file to grades information
@@ -150,12 +156,31 @@ def grade_assignments(submission_path, image="ucbdsinfra/otter-grader", verbose=
         ]
         if pdfs:
             volumes.append((pdf_path, f"/autograder/submission/{nb_name}.pdf"))
-        container = docker.container.run(image, command=["/autograder/run_autograder"], volumes=volumes, detach=True)
+
+        args = {}
+
+        if network is not None and not network:
+            args['networks'] = 'none'
+
+        container = docker.container.run(image, command=["/autograder/run_autograder"], volumes=volumes, detach=True, **args)
+
+        if timeout:
+            import threading
+
+            def kill_container():
+                docker.container.kill(container)
+
+            timer = threading.Timer(timeout, kill_container)
+            timer.start()
 
         if verbose:
             print(f"Grading {submission_path} in container {container.id}...")
 
         exit = docker.container.wait(container)
+
+        if timeout:
+            timer.cancel()
+
         if debug:
             print(docker.container.logs(container))