Merge pull request #21 from CSCfi/feature/chunked-segments

Feature/chunked segments

Author: blankdots

.github/workflows/style.yml

@@ -27,4 +27,6 @@ jobs:
     - name: Test mypy typing with tox
       run: tox -e mypy
     - name: bandit static check
-      run: tox -e bandit
+      run: tox -e bandit
+    - name: black style check
+      run : tox -e black

requirements.txt

@@ -4,3 +4,5 @@ python-swiftclient==3.11.1
 keystoneauth1==4.3.1
 git+https://github.com/cscfi/swift-browser-ui.git
 certifi==2020.12.05 
+gunicorn>=20.0.1
+uvloop==0.15.2

setup.py

@@ -5,48 +5,53 @@
 setuptools.setup(
     name=__name__,
     version=__version__,
-    description='Swift upload / download runner for swift-browser-ui',
+    description="Swift upload / download runner for swift-browser-ui",
     author=__author__,
     author_email="[email protected]",
     project_urls={
-        'Source': 'https://github.com/CSCfi/swift-upload-runner',
+        "Source": "https://github.com/CSCfi/swift-upload-runner",
     },
-    license='MIT',
+    license="MIT",
     install_requires=[
-        'aiohttp',
-        'python-swiftclient',
-        'keystoneauth1',
-        'gunicorn',
-        'certifi',
-        'uvloop',
-        "swift-browser-ui"
-        "@ git+https://github.com/cscfi/swift-browser-ui.git"
+        "aiohttp==3.7.4.post0",
+        "requests==2.25.1",
+        "python-swiftclient==3.11.1",
+        "keystoneauth1==4.3.1",
+        "gunicorn>=20.0.1",
+        "uvloop==0.15.2",
+        "certifi==2020.12.05",
+        "swift-browser-ui" "@ git+https://github.com/cscfi/swift-browser-ui.git",
     ],
     extras_require={
-        'test': ['tox', 'pytest', 'pytest-cov', 'coverage',
-                 'tox', 'flake8', 'flake8-docstrings', 'pytest-aiohttp',
-                 'asynctest'],
+        "test": [
+            "tox==3.23.1",
+            "pytest==6.2.4",
+            "pytest-cov==2.12.1",
+            "coverage==5.5",
+            "flake8==3.9.2",
+            "flake8-docstrings==1.6.0",
+            "asynctest==0.13.0",
+            "black==21.6b0",
+            "pytest-aiohttp==0.3.0",
+        ],
     },
     packages=[__name__],
     include_package_data=True,
-    platforms='any',
+    platforms="any",
     entry_points={
-        'console_scripts': [
-            'swift-upload-runner=swift_upload_runner.server:main',
+        "console_scripts": [
+            "swift-upload-runner=swift_upload_runner.server:main",
         ]
     },
     classifiers=[
-        'Development Status :: 4 - Beta',
-
+        "Development Status :: 4 - Beta",
         # Indicate who your project is intended for
-        'Intended Audience :: Developers',
-        'Intended Audience :: Information Technology',
-        'Topic :: Internet :: WWW/HTTP :: HTTP Servers',
-
+        "Intended Audience :: Developers",
+        "Intended Audience :: Information Technology",
+        "Topic :: Internet :: WWW/HTTP :: HTTP Servers",
         # Pick your license as you wish
-        'License :: OSI Approved :: MIT License',
-
-        'Programming Language :: Python :: 3.7',
-        'Programming Language :: Python :: 3.8',
+        "License :: OSI Approved :: MIT License",
+        "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
     ],
 )

swift_upload_runner/__init__.py

@@ -1,6 +1,6 @@
 """Runner for swift-browser-ui upload and replication operations."""
 
 __name__ = "swift_upload_runner"
-__version__ = "0.1.8"
+__version__ = "0.2.0"
 __author__ = "CSC Developers"
 __license__ = "MIT License"

swift_upload_runner/api.py

@@ -10,9 +10,7 @@
 from .replicate import ObjectReplicationProxy
 
 
-async def handle_get_object(
-        request: aiohttp.web.Request
-) -> aiohttp.web.StreamResponse:
+async def handle_get_object(request: aiohttp.web.Request) -> aiohttp.web.StreamResponse:
     """Handle a request for getting object content."""
     auth = get_auth_instance(request)
 
@@ -21,7 +19,7 @@ async def handle_get_object(
     await download.a_begin_download(
         request.match_info["project"],
         request.match_info["container"],
-        request.match_info["object_name"]
+        request.match_info["object_name"],
     )
 
     resp = aiohttp.web.StreamResponse()
@@ -39,7 +37,7 @@ async def handle_get_object(
 
 
 async def handle_replicate_container(
-        request: aiohttp.web.Request
+    request: aiohttp.web.Request,
 ) -> aiohttp.web.Response:
     """Handle request to replicating a container from a source."""
     auth = get_auth_instance(request)
@@ -51,22 +49,15 @@ async def handle_replicate_container(
     source_container = request.query["from_container"]
 
     replicator = ObjectReplicationProxy(
-        auth,
-        request.app["client"],
-        project,
-        container,
-        source_project,
-        source_container
+        auth, request.app["client"], project, container, source_project, source_container
     )
 
     asyncio.ensure_future(replicator.a_copy_from_container())
 
     return aiohttp.web.Response(status=202)
 
 
-async def handle_replicate_object(
-        request: aiohttp.web.Request
-) -> aiohttp.web.Response:
+async def handle_replicate_object(request: aiohttp.web.Request) -> aiohttp.web.Response:
     """Handle a request to replicating an object from a source."""
     auth = get_auth_instance(request)
 
@@ -78,22 +69,15 @@ async def handle_replicate_object(
     source_object = request.query["from_object"]
 
     replicator = ObjectReplicationProxy(
-        auth,
-        request.app["client"],
-        project,
-        container,
-        source_project,
-        source_container
+        auth, request.app["client"], project, container, source_project, source_container
     )
 
     asyncio.ensure_future(replicator.a_copy_object(source_object))
 
     return aiohttp.web.Response(status=202)
 
 
-async def handle_post_object_chunk(
-        request: aiohttp.web.Request
-) -> aiohttp.web.Response:
+async def handle_post_object_chunk(request: aiohttp.web.Request) -> aiohttp.web.Response:
     """Handle a request for posting an object chunk."""
     if "from_object" in request.query.keys():
         return await handle_replicate_object(request)
@@ -105,22 +89,12 @@ async def handle_post_object_chunk(
 
     query, data = await parse_multipart_in(request)
 
-    upload_session = await get_upload_instance(
-        request,
-        project,
-        container,
-        p_query=query
-    )
+    upload_session = await get_upload_instance(request, project, container, p_query=query)
 
-    return await upload_session.a_add_chunk(
-        query,
-        data
-    )
+    return await upload_session.a_add_chunk(query, data)
 
 
-async def handle_get_object_chunk(
-        request: aiohttp.web.Request
-) -> aiohttp.web.Response:
+async def handle_get_object_chunk(request: aiohttp.web.Request) -> aiohttp.web.Response:
     """Handle a request for checking if a chunk exists."""
     get_auth_instance(request)
 
@@ -134,19 +108,13 @@ async def handle_get_object_chunk(
     except KeyError:
         raise aiohttp.web.HTTPBadRequest(reason="Malformed query string")
 
-    upload_session = await get_upload_instance(
-        request,
-        project,
-        container
-    )
+    upload_session = await get_upload_instance(request, project, container)
 
-    return await upload_session.a_check_segment(
-        chunk_number
-    )
+    return await upload_session.a_check_segment(chunk_number)
 
 
 async def handle_post_object_options(
-        request: aiohttp.web.Request
+    request: aiohttp.web.Request,
 ) -> aiohttp.web.Response:
     """Handle options request for posting the object chunk."""
     resp = aiohttp.web.Response(
@@ -160,7 +128,7 @@ async def handle_post_object_options(
 
 
 async def handle_get_container(
-        request: aiohttp.web.Request
+    request: aiohttp.web.Request,
 ) -> aiohttp.web.StreamResponse:
     """Handle a request for getting container contents as an archive."""
     if "resumableChunkNumber" in request.query.keys():
@@ -183,11 +151,7 @@ async def handle_get_container(
 
     await resp.prepare(request)
 
-    download = ContainerArchiveDownloadProxy(
-        auth,
-        project,
-        container
-    )
+    download = ContainerArchiveDownloadProxy(auth, project, container)
 
     await download.a_begin_container_download()
 
@@ -197,13 +161,13 @@ async def handle_get_container(
     return resp
 
 
-async def handle_health_check(
-        request: aiohttp.web.Request
-) -> aiohttp.web.Response:
+async def handle_health_check(request: aiohttp.web.Request) -> aiohttp.web.Response:
     """Answer a service health check."""
     # Case degraded
 
     # Case nominal
-    return aiohttp.web.json_response({
-        "status": "Ok",
-    })
+    return aiohttp.web.json_response(
+        {
+            "status": "Ok",
+        }
+    )

swift_upload_runner/auth.py

@@ -10,23 +10,15 @@
 
 import aiohttp.web
 
-from swift_browser_ui._convenience import (
-    initiate_os_session
-)
+from swift_browser_ui._convenience import initiate_os_session
 
 AiohttpHandler = typing.Callable[
     [aiohttp.web.Request],
-    typing.Coroutine[
-        typing.Awaitable,
-        typing.Any,
-        aiohttp.web.Response
-    ]
+    typing.Coroutine[typing.Awaitable, typing.Any, aiohttp.web.Response],
 ]
 
 
-async def handle_login(
-        request: aiohttp.web.Request
-) -> aiohttp.web.StreamResponse:
+async def handle_login(request: aiohttp.web.Request) -> aiohttp.web.StreamResponse:
     """Begin a new session for the upload process."""
     session_key = hashlib.sha1(os.urandom(128)).hexdigest()  # nosec
 
@@ -36,67 +28,47 @@ async def handle_login(
         token = login_form["token"]
         request.app[session_key] = {}
         request.app[session_key]["uploads"] = {}
-        request.app[session_key]["auth"] = initiate_os_session(
-            token,
-            project
-        )
+        request.app[session_key]["auth"] = initiate_os_session(token, project)
 
-        resp = aiohttp.web.Response(
-            status=200,
-            body='OK'
-        )
+        resp = aiohttp.web.Response(status=200, body="OK")
         resp.cookies["RUNNER_SESSION_ID"] = session_key
 
         return resp
 
     except KeyError:
-        raise aiohttp.web.HTTPUnauthorized(
-            reason="Login token or project missing"
-        )
+        raise aiohttp.web.HTTPUnauthorized(reason="Login token or project missing")
 
 
-async def read_in_keys(
-        app: aiohttp.web.Application
-) -> None:
+async def read_in_keys(app: aiohttp.web.Application) -> None:
     """Read in keys to the application."""
     keys = os.environ.get("SWIFT_UI_API_AUTH_TOKENS", None)
     app["tokens"] = keys.split(",") if keys is not None else []
     if app["tokens"]:
-        app["tokens"] = [
-            token.encode("utf-8") for token in app["tokens"]
-        ]
+        app["tokens"] = [token.encode("utf-8") for token in app["tokens"]]
 
 
 async def test_signature(
-        tokens: typing.List[bytes],
-        signature: str,
-        message: str,
-        validity: str,
+    tokens: typing.List[bytes],
+    signature: str,
+    message: str,
+    validity: str,
 ) -> bool:
     """Validate signature against the given tokens."""
     # Check signature expiration
     if int(validity) < time.time():
-        raise aiohttp.web.HTTPUnauthorized(
-            reason="Signature expired"
-        )
+        raise aiohttp.web.HTTPUnauthorized(reason="Signature expired")
     byte_message = message.encode("utf-8")
     for token in tokens:
-        digest = hmac.new(
-            token,
-            byte_message,
-            digestmod="sha256"
-        ).hexdigest()
+        digest = hmac.new(token, byte_message, digestmod="sha256").hexdigest()
         if secrets.compare_digest(digest, signature):
             return True
-    raise aiohttp.web.HTTPUnauthorized(
-        reason="Missing valid query signature"
-    )
+    raise aiohttp.web.HTTPUnauthorized(reason="Missing valid query signature")
 
 
 @aiohttp.web.middleware
 async def handle_validate_authentication(
-        request: aiohttp.web.Request,
-        handler: AiohttpHandler,
+    request: aiohttp.web.Request,
+    handler: AiohttpHandler,
 ) -> aiohttp.web.Response:
     """Handle the authentication of a response as a middleware function."""
     try:
@@ -108,11 +80,6 @@ async def handle_validate_authentication(
             reason="Query string missing validity or signature"
         )
 
-    await test_signature(
-        request.app["tokens"],
-        signature,
-        validity + path,
-        validity
-    )
+    await test_signature(request.app["tokens"], signature, validity + path, validity)
 
     return await handler(request)