Merge pull request #22 from Autodesk/docker-input-paths

Docker input paths

Author: avirshup

codeship-steps.yml

@@ -15,7 +15,7 @@
        name: check_pyversion
      - command: ./scripts/deploy.sh
        name: pypi_deploy
-  # tests for semantic versions
-  tag: '^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$'
+  # tests for (a subset of) PEP440 versions
+  tag: '^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)((a|rc|b)(0|[1-9]\d*))?$'
 
 

pyccc/docker_utils.py

@@ -22,16 +22,23 @@
 import subprocess
 import io
 import tarfile
+import tempfile
+import gzip
 
 import pyccc
 
 from .exceptions import DockerMachineError
+from . import files
 
 
 def create_provisioned_image(client, image, wdir, inputs, pull=False):
     build_context = create_build_context(image, inputs, wdir)
-    tarobj = make_tar_stream(build_context)
-    imageid = build_dfile_stream(client, tarobj, is_tar=True, pull=pull)
+    with tempfile.NamedTemporaryFile(suffix='.tar.gz', mode='wb') as tfile:
+        gzstream = gzip.GzipFile(fileobj=tfile, mode='wb')
+        make_tar_stream(build_context, gzstream)
+        gzstream.flush()
+        tfile.flush()
+        imageid = build_dfile_stream(client, tfile.name, pull=pull)
     return imageid
 
 
@@ -40,26 +47,38 @@ def create_build_context(image, inputs, wdir):
     Creates a tar archive with a dockerfile and a directory called "inputs"
     The Dockerfile will copy the "inputs" directory to the chosen working directory
     """
-    dockerstring = "FROM %s" % image
+    assert os.path.isabs(wdir)
 
+    dockerlines = ["FROM %s" % image,
+                    "RUN mkdir -p %s" % wdir]
     build_context = {}
-    if inputs:
-        dockerstring += "\nCOPY inputs %s\n" % wdir
-        for name, obj in inputs.items():
-            build_context['inputs/%s' % name] = obj
-    else:
-        dockerstring += "\nRUN mkdir -p %s\n" % wdir
 
+    # This loop creates a Build Context for building the provisioned image
+    # Each input file is assigned a unique name, in preparation for creating a tar archive
+    if inputs:
+        for ifile, (path, obj) in enumerate(inputs.items()):
+            src = os.path.basename(path) + '-%s' % ifile  # mangling to ensure uniqueness
+            if os.path.isabs(path):
+                dest = path
+            else:
+                dest = os.path.join(wdir, path)
+            dockerlines.append('ADD %s %s' % (src, dest))
+            build_context[src] = obj
+
+    dockerstring = '\n'.join(dockerlines)
     build_context['Dockerfile'] = pyccc.BytesContainer(dockerstring.encode('utf-8'))
-
     return build_context
 
 
-def build_dfile_stream(client, dfilestream, is_tar=False, **kwargs):
-    buildcmd = client.build(fileobj=dfilestream,
-                            rm=True,
-                            custom_context=is_tar,
-                            **kwargs)
+def build_dfile_stream(client, dfilepath, **kwargs):
+    # dfilepath is the path to the already .tgz-archived build context
+
+    with open(dfilepath, 'rb') as dfilestream:
+        buildcmd = client.build(fileobj=dfilestream,
+                                rm=True,
+                                custom_context=True,
+                                encoding='gzip',
+                                **kwargs)
 
     # this blocks until the image is done building
     for x in buildcmd:
@@ -104,23 +123,20 @@ def _issue1134_helper(x):
     return s[rootbrace: endbrace+1]
 
 
-def make_tar_stream(sdict):
-    """ Create a file-like tarfile object
+def make_tar_stream(build_context, buffer):
+    """ Write a tar stream of the build context to the provided buffer
 
     Args:
-        sdict (Mapping[str, pyccc.FileReferenceBase]): dict mapping filenames to file references
-
-    Returns:
-        Filelike: TarFile stream
+        build_context (Mapping[str, pyccc.FileReferenceBase]): dict mapping filenames to file references
+        buffer (io.BytesIO): writable binary mode buffer
     """
-    # TODO: this is currently done in memory - bad for big files!
-    tarbuffer = io.BytesIO()
-    tf = tarfile.TarFile(fileobj=tarbuffer, mode='w')
-    for name, fileobj in sdict.items():
-        tar_add_bytes(tf, name, fileobj.read('rb'))
+    tf = tarfile.TarFile(fileobj=buffer, mode='w')
+    for context_path, fileobj in build_context.items():
+        if isinstance(fileobj, (files.LocalDirectoryReference, files.LocalFile)):
+            tf.add(fileobj.localpath, arcname=context_path)
+        else:
+            tar_add_bytes(tf, context_path, fileobj.read('rb'))
     tf.close()
-    tarbuffer.seek(0)
-    return tarbuffer
 
 
 def tar_add_bytes(tf, filename, bytestring):

pyccc/engines/subproc.py

@@ -61,8 +61,11 @@ def submit(self, job):
         self._check_job(job)
         if job.workingdir is None:
             job.workingdir = utils.make_local_temp_dir()
+
+        assert os.path.isabs(job.workingdir)
         if job.inputs:
             for filename, f in job.inputs.items():
+                self._check_input_target_location(filename, job.workingdir)
                 f.put(os.path.join(job.workingdir, filename))
 
         subenv = os.environ.copy()
@@ -77,6 +80,20 @@ def submit(self, job):
         job._started = True
         return job.subproc.pid
 
+    @staticmethod
+    def _check_input_target_location(filename, wdir):
+        """ Raise error if input is being staged to a location not underneath the working dir
+        """
+        p = filename
+        if not os.path.isabs(p):
+            p = os.path.join(wdir, p)
+        targetpath = os.path.realpath(p)
+        wdir = os.path.realpath(wdir)
+        common = os.path.commonprefix([wdir, targetpath])
+        if len(common) < len(wdir):
+            raise ValueError(
+                    "The subprocess engine does not support input files with absolute paths")
+
     def kill(self, job):
         job.subproc.terminate()
 

pyccc/files/__init__.py

@@ -8,3 +8,4 @@
 from .stringcontainer import *
 from .localfiles import *
 from .remotefiles import *
+from .directory import *

pyccc/files/directory.py

@@ -0,0 +1,26 @@
+# Copyright 2016 Autodesk Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class LocalDirectoryReference(object):
+    """ This is a reference to a specific directory on the local filesystem.
+
+    This allows entire directories to be staged into the dockerfile as input
+    """
+    def __init__(self, localpath):
+        self.localpath = localpath
+
+    def put(self, destination):
+        import shutil
+        shutil.copytree(self.localpath, destination)

pyccc/tests/data/a

@@ -0,0 +1 @@
+a

pyccc/tests/data/b

@@ -0,0 +1 @@
+b

pyccc/tests/test_job_types.py

@@ -10,6 +10,7 @@
 
 PYVERSION = '%s.%s' % (sys.version_info.major, sys.version_info.minor)
 PYIMAGE = 'python:%s-slim' % PYVERSION
+THISDIR = os.path.dirname(__file__)
 
 
 ########################
@@ -421,3 +422,37 @@ def test_clean_working_dir(fixture, request):
     job = engine.launch(image='alpine', command='ls')
     job.wait()
     assert job.stdout.strip() == ''
+
+
+class no_context():  # context manager that does nothing that can be used conditionaly
+    def __enter__(self):
+        return None
+    def __exit__(self, exc_type, exc_value, traceback):
+        return False
+
+
+@pytest.mark.parametrize('fixture', fixture_types['engine'])
+def test_abspath_input_files(fixture, request):
+    engine = request.getfuncargvalue(fixture)
+    with pytest.raises(ValueError) if isinstance(engine, pyccc.Subprocess) else no_context():
+        # this is OK with docker but should fail with a subprocess
+        job = engine.launch(image='alpine', command='cat /opt/a',
+                            inputs={'/opt/a': pyccc.LocalFile(os.path.join(THISDIR, 'data', 'a'))})
+    if not isinstance(engine, pyccc.Subprocess):
+        job.wait()
+        assert job.exitcode == 0
+        assert job.stdout.strip() == 'a'
+
+
+@pytest.mark.parametrize('fixture', fixture_types['engine'])
+def test_directory_input(fixture, request):
+    engine = request.getfuncargvalue(fixture)
+
+    # this is OK with docker but should fail with a subprocess
+    job = engine.launch(image='alpine', command='cat data/a data/b',
+                        inputs={'data':
+                                    pyccc.LocalDirectoryReference(os.path.join(THISDIR, 'data'))})
+    job.wait()
+    assert job.exitcode == 0
+    assert job.stdout.strip() == 'a\nb'
+