add support for get and put manifest schema v1

Author: Thomas Grainger

CHANGES.rst

@@ -2,3 +2,5 @@
 ------------------
 
 - First version of docker-registry-client with changelog
+- Support get and push manifest on protocol v2, schema v1.
+  (`Issue #33 <https://github.com/yodle/docker-registry-client/pull/33>`_)

docker_registry_client/_BaseClient.py

@@ -3,6 +3,7 @@
 from requests.exceptions import HTTPError
 import json
 from docker_registry_client.AuthorizationService import AuthorizationService
+from .manifest import sign as sign_manifest
 
 # urllib3 throws some ssl warnings with older versions of python
 #   they're probably ok for the registry client to ignore
@@ -136,10 +137,23 @@ def delete_repository(self, namespace, repository):
                                namespace=namespace, repository=repository)
 
 
+class _Manifest(object):
+    def __init__(self, content, type, digest):
+        self._content = content
+        self._type = type
+        self._digest = digest
+
+
+BASE_CONTENT_TYPE = 'application/vnd.docker.distribution.manifest'
+
+
 class BaseClientV2(CommonBaseClient):
     LIST_TAGS = '/v2/{name}/tags/list'
     MANIFEST = '/v2/{name}/manifests/{reference}'
     BLOB = '/v2/{name}/blobs/{digest}'
+    schema_1_signed = BASE_CONTENT_TYPE + '.v1+prettyjws'
+    schema_1 = BASE_CONTENT_TYPE + '.v1+json'
+    schema_2 = BASE_CONTENT_TYPE + '.v2+json'
 
     def __init__(self, *args, **kwargs):
         auth_service_url = kwargs.pop("auth_service_url", "")
@@ -169,11 +183,33 @@ def get_repository_tags(self, name):
         return self._http_call(self.LIST_TAGS, get, name=name)
 
     def get_manifest_and_digest(self, name, reference):
+        m = self.get_manifest(name, reference)
+        return m._content, m._digest
+
+    def get_manifest(self, name, reference):
         self.auth.desired_scope = 'repository:%s:*' % name
-        response = self._http_response(self.MANIFEST, get,
-                                       name=name, reference=reference)
+        response = self._http_response(
+            self.MANIFEST, get, name=name, reference=reference,
+            schema=self.schema_1_signed,
+        )
         self._cache_manifest_digest(name, reference, response=response)
-        return (response.json(), self._manifest_digests[name, reference])
+        return _Manifest(
+            content=response.json(),
+            type=response.headers.get('Content-Type', 'application/json'),
+            digest=self._manifest_digests[name, reference],
+        )
+
+    def put_manifest(self, name, reference, manifest):
+        self.auth.desired_scope = 'repository:%s:*' % name
+        content = {}
+        content.update(manifest._content)
+        content.update({'name': name, 'tag': reference})
+
+        return self._http_call(
+            self.MANIFEST, put, data=sign_manifest(content),
+            content_type=self.schema_1_signed, schema=self.schema_1_signed,
+            name=name, reference=reference,
+        )
 
     def delete_manifest(self, name, digest):
         self.auth.desired_scope = 'repository:%s:*' % name
@@ -193,16 +229,20 @@ def _cache_manifest_digest(self, name, reference, response=None):
         untrusted_digest = response.headers.get('Docker-Content-Digest')
         self._manifest_digests[(name, reference)] = untrusted_digest
 
-    def _http_response(self, url, method, data=None, **kwargs):
+    def _http_response(self, url, method, data=None, content_type=None,
+                       schema=None, **kwargs):
         """url -> full target url
            method -> method from requests
            data -> request body
            kwargs -> url formatting args
         """
 
+        if schema is None:
+            schema = self.schema_2
+
         header = {
-            'content-type': 'application/json',
-            'Accept': 'application/vnd.docker.distribution.manifest.v2+json',
+            'content-type': content_type or 'application/json',
+            'Accept': schema,
         }
 
         # Token specific part. We add the token in the header if necessary
@@ -219,8 +259,9 @@ def _http_response(self, url, method, data=None, **kwargs):
 
             header['Authorization'] = 'Bearer %s' % self.auth.token
 
-        if data:
+        if data and not content_type:
             data = json.dumps(data)
+
         path = url.format(**kwargs)
         logger.debug("%s %s", method.__name__.upper(), path)
         response = method(self.host + path,

docker_registry_client/manifest.py

@@ -0,0 +1,112 @@
+# code extracted from https://git.io/vM0EB used under license:
+#
+# Copyright (c) 2015 David Halls <https://github.com/davedoesdev/>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+import base64
+import ecdsa
+import jws
+import json
+
+
+def assign(obj, *objs):
+    for o in objs:
+        obj.update(o)
+    return obj
+
+
+def force_bytes(s, encoding='utf8'):
+    return s if isinstance(s, bytes) else s.encode(encoding)
+
+
+def _urlsafe_b64encode_bytes(b):
+    return base64.urlsafe_b64encode(b).rstrip(b'=').decode('utf-8')
+
+
+def _urlsafe_b64encode(s):
+    return _urlsafe_b64encode_bytes(force_bytes(s))
+
+
+jws.utils.to_bytes_2and3 = force_bytes
+jws.algos.to_bytes_2and3 = force_bytes
+
+
+def _num_to_base64(n):
+    b = bytearray()
+    while n:
+        b.insert(0, n & 0xFF)
+        n >>= 8
+    # need to pad to 32 bytes
+    while len(b) < 32:
+        b.insert(0, 0)
+    return _urlsafe_b64encode_bytes(b)
+
+
+def sign(manifest, key=None):
+    m = assign({}, manifest)
+
+    try:
+        del m['signatures']
+    except KeyError:
+        pass
+
+    assert len(m) > 0
+
+    manifest_json = json.dumps(m, sort_keys=True)
+
+    if key is None:
+        key = ecdsa.SigningKey.generate(curve=ecdsa.NIST256p)
+
+    manifest64 = _urlsafe_b64encode(manifest_json)
+    format_length = manifest_json.rfind('}')
+    format_tail = manifest_json[format_length:]
+    protected_json = json.dumps({
+        'formatLength': format_length,
+        'formatTail': _urlsafe_b64encode(format_tail)
+    })
+    protected64 = _urlsafe_b64encode(protected_json)
+    point = key.privkey.public_key.point
+    data = {
+        'key': key,
+        'header': {
+            'alg': 'ES256'
+        }
+    }
+    jws.header.process(data, 'sign')
+    sig = data['signer']("%s.%s" % (protected64, manifest64), key)
+    signatures = [{
+        'header': {
+            'jwk': {
+                'kty': 'EC',
+                'crv': 'P-256',
+                'x': _num_to_base64(point.x()),
+                'y': _num_to_base64(point.y())
+            },
+            'alg': 'ES256'
+        },
+        'signature': _urlsafe_b64encode(sig),
+        'protected': protected64
+    }]
+    return (
+        manifest_json[:format_length] +
+        ', "signatures": ' +
+        json.dumps(signatures) +
+        format_tail
+    )

setup.py

@@ -27,5 +27,7 @@
     packages=find_packages(),
     install_requires=[
         'requests>=2.4.3, <3.0.0',
+        'ecdsa>=0.13.0, <0.14.0',
+        'jws>=0.1.3, <0.2.0',
     ],
 )

tests/conftest.py

@@ -26,7 +26,7 @@ def wait_till_up(url, attempts):
         requests.get(url)
 
 
-@pytest.yield_fixture(scope='session')
+@pytest.yield_fixture()
 def registry(docker_client):
     cli = docker_client
     cli.pull('registry', '2')

tests/integration/fixtures/base/Dockerfile

@@ -0,0 +1,2 @@
+FROM busybox
+ADD . /code

tests/integration/fixtures/base/base.txt

@@ -0,0 +1 @@
+Hi mom

tests/integration/test_base_client.py

@@ -1,6 +1,49 @@
 from docker_registry_client import BaseClient
+import pkg_resources
 
 
 def test_base_client(registry):
     cli = BaseClient('http://localhost:5000', api_version=2)
     assert cli.catalog() == {'repositories': []}
+
+
+def test_base_client_edit_manifest(docker_client, registry):
+    cli = BaseClient('http://localhost:5000', api_version=2)
+    build = docker_client.build(
+        pkg_resources.resource_filename(__name__, 'fixtures/base'),
+        'localhost:5000/x-drc-example:x-drc-test', stream=True,
+    )
+    for line in build:
+        print(line)
+
+    push = docker_client.push(
+        'localhost:5000/x-drc-example', 'x-drc-test', stream=True,
+        insecure_registry=True,
+    )
+
+    for line in push:
+        print(line)
+
+    m = cli.get_manifest('x-drc-example', 'x-drc-test')
+    assert m._content['name'] == 'x-drc-example'
+    assert m._content['tag'] == 'x-drc-test'
+
+    cli.put_manifest('x-drc-example', 'x-drc-test-put', m)
+
+    pull = docker_client.pull(
+        'localhost:5000/x-drc-example', 'x-drc-test-put', stream=True,
+        insecure_registry=True, decode=True,
+    )
+
+    pull = list(pull)
+    tag = 'localhost:5000/x-drc-example:x-drc-test-put'
+
+    expected_statuses = {
+        'Status: Downloaded newer image for ' + tag,
+        'Status: Image is up to date for ' + tag,
+    }
+
+    errors = [evt for evt in pull if 'error' in evt]
+    assert errors == []
+
+    assert {evt.get('status') for evt in pull} & expected_statuses