File storage mvp

Author: Nikita Kolybelkin

.dockerignore

@@ -0,0 +1,8 @@
+*
+!file_storage/
+!scripts/
+!run.py
+!poetry.lock
+!pyproject.toml
+!tests/
+!yoyo.ini

.env

@@ -0,0 +1,4 @@
+DB_NAME=file_storage
+DB_USER=postgres
+DB_PASSWORD=postgres
+DB_SERVER=postgres

.gitignore

@@ -0,0 +1,3 @@
+/venv/
+/.idea
+/.pytest_cache

Dockerfile

@@ -0,0 +1,45 @@
+FROM python:3.7.4-alpine3.10 as builder
+
+ARG DEVELOPMENT_BUILD=true
+ENV PIP_NO_CACHE_DIR=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1 \
+    PYTHONIOENCODING=utf-8 \
+    POETRY_VIRTUALENVS_CREATE="false"
+
+WORKDIR /src
+
+RUN apk add --no-cache gcc make musl-dev libffi-dev libressl-dev git
+
+RUN pip install -U pip==20.2.3 \
+    && pip install poetry==1.0.10
+
+ADD pyproject.toml poetry.lock /src/
+RUN poetry export \
+        --format requirements.txt \
+        --output requirements.txt \
+        --without-hashes \
+        --with-credentials \
+        $(test "$DEVELOPMENT_BUILD" = "true" && echo "--dev") \
+    && apk add --no-cache postgresql-dev \
+    && mkdir /wheels \
+    && pip wheel -r requirements.txt --wheel-dir /wheels \
+    && rm requirements.txt
+
+FROM python:3.7.4-alpine3.10
+
+ENV PIP_NO_CACHE_DIR=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1 \
+    PIP_NO_INDEX=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    LANG=C.UTF-8 \
+    LC_ALL=C.UTF-8
+
+WORKDIR /src
+
+COPY --from=builder /wheels /wheels
+RUN apk add --no-cache libpq libssl1.1 \
+ && pip install /wheels/*
+
+COPY . .
+
+ENTRYPOINT ["/bin/sh"]

Readme.MD

@@ -0,0 +1,8 @@
+# File storage
+
+### Запуск
+`scripts/run.sh`
+
+## Описание
+Тестовый проект, позволяет загружать файлы, в ответ получать уникальные ссылки.
+Обрабатывается сценарий загрузки одинаковых файлов, в данном случае создаём только новый линк. (см. хранимку в миграциях)

docker-compose.yml

@@ -0,0 +1,28 @@
+version: '3.8'
+services:
+  file_storage:
+    depends_on:
+      - postgres
+    image: $COMMIT_HASH
+    networks:
+      - file_storage
+    env_file:
+      - .env
+    ports:
+    - 8080:8080
+    command: ["/src/scripts/start_server.sh"]
+
+  postgres:
+    image: postgres:11
+    networks:
+      - file_storage
+    environment:
+      - POSTGRES_DB=file_storage
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=postgres
+
+    ports:
+      - 5432:5432
+
+networks:
+  file_storage:

file_storage/app.py

@@ -0,0 +1,15 @@
+import typing as tp
+
+from aiohttp import web
+from file_storage.router import setup_api_routes
+from file_storage.setup import on_shutdown, on_startup
+from file_storage.typings import AioHttpApplication
+
+
+def get_application() -> AioHttpApplication:
+    """Get app."""
+    app: AioHttpApplication = tp.cast(AioHttpApplication, web.Application())
+    setup_api_routes(app)
+    app.on_startup.extend(on_startup)
+    app.on_shutdown.extend(on_shutdown)
+    return app

file_storage/conf.py

@@ -0,0 +1,27 @@
+from pydantic import BaseSettings
+
+
+class Settings(BaseSettings):
+    """App settings."""
+
+    DB_NAME: str = "file_storage"
+    DB_USER: str = "postgres"
+    DB_PASSWORD: str = "postgres"
+    DB_SERVER = "localhost"
+    DB_PORT = 5432
+
+    TEST_DB_NAME: str = f"test_{DB_NAME}"
+
+    @property
+    def postgres_uri(self) -> str:
+        """Postgres uri string."""
+        return "postgres://{username}:{password}@{address}:{port}/{db_name}".format(
+            username=self.DB_USER,
+            password=self.DB_PASSWORD,
+            address=self.DB_SERVER,
+            port=self.DB_PORT,
+            db_name=self.DB_NAME,
+        )
+
+
+settings = Settings()

file_storage/migrations/20201207_01_5UMJd.py

@@ -0,0 +1,48 @@
+"""
+
+"""
+
+from yoyo import step
+
+__depends__ = {}
+
+steps = [
+    step(
+        """
+        CREATE TABLE IF NOT EXISTS "uploaded_files" (
+            "id" BIGSERIAL PRIMARY KEY,
+            "data" bytea NOT NULL,
+            "hash" varchar(32) NOT NULL
+        );
+        COMMENT ON TABLE "uploaded_files" IS 'Mapping hash to uploaded file.';
+        CREATE UNIQUE INDEX uploaded_files_hash_idx ON uploaded_files (hash); 
+        CREATE TABLE IF NOT EXISTS "files_meta" (
+            "id" BIGSERIAL PRIMARY KEY,
+            "uuid" uuid NOT NULL UNIQUE,
+            "content_type" TEXT NOT NULL,
+            "hash" varchar(32),
+        FOREIGN KEY (hash) REFERENCES uploaded_files (hash) 
+        );
+        COMMENT ON TABLE "files_meta" IS 'Mapping uuid to name and hash of uploaded file.';
+        CREATE UNIQUE INDEX files_meta_uuid ON files_meta (uuid); 
+        
+        -- Database procedure for update file to database and get hash
+        CREATE OR REPLACE FUNCTION upload_file(data bytea) RETURNS varchar(32) as $$
+        DECLARE computed_hash varchar(32);
+        BEGIN 
+            computed_hash = md5(data);
+            INSERT INTO uploaded_files (data, hash) VALUES (data::bytea, computed_hash)
+            ON CONFLICT (hash) DO NOTHING;
+            RETURN computed_hash;
+        END;
+        $$ LANGUAGE plpgsql;
+        """,
+        """
+        DROP TABLE IF EXISTS "files_meta" CASCADE;
+        DROP INDEX IF EXISTS "files_meta_uuid";
+        DROP TABLE IF EXISTS "uploaded_files" CASCADE;
+        DROP INDEX IF EXISTS "uploaded_files_hash_idx";
+        DROP FUNCTION IF EXISTS "upload_file";
+        """,
+    )
+]

file_storage/router.py

@@ -0,0 +1,8 @@
+from file_storage.typings import AioHttpApplication
+from file_storage.views import FileStorageHandler
+
+
+def setup_api_routes(app: AioHttpApplication) -> None:
+    """Setup landing API routes."""
+    app.router.add_put("/api/v1/file_storage", FileStorageHandler.put)
+    app.router.add_get(r"/api/v1/file_storage/{uuid}", FileStorageHandler.get)

file_storage/setup.py

@@ -0,0 +1,33 @@
+import asyncpg
+from file_storage.conf import settings
+from file_storage.typings import AioHttpApplication
+
+
+async def get_db() -> asyncpg.pool.Pool:
+    """
+    Creates postgres pool.
+
+    We need separate function for testing purposes.
+    """
+    return await asyncpg.create_pool(
+        database=settings.DB_NAME,
+        user=settings.DB_USER,
+        password=settings.DB_PASSWORD,
+        host=settings.DB_SERVER,
+        port=settings.DB_PORT,
+        statement_cache_size=0,
+    )
+
+
+async def setup_app_postgres(app: AioHttpApplication) -> None:
+    """Creates postgres pool for app."""
+    app.db = await get_db()
+
+
+async def shutdown_postgres(app: AioHttpApplication):
+    """Close postgres."""
+    await app.db.close()
+
+
+on_startup = (setup_app_postgres,)
+on_shutdown = (shutdown_postgres,)

file_storage/sql.py

@@ -0,0 +1,26 @@
+def upload_data_query():
+    """
+    Insert new data and return hash.
+
+    On case when hash already exists.
+    """
+    return """SELECT hash FROM upload_file($1) hash;"""
+
+
+def insert_meta_query():
+    """Save meta data for uploaded file."""
+    return """
+    INSERT INTO files_meta (uuid, content_type, hash) VALUES ($1, $2, $3);
+    """
+
+
+def retrieve_data_by_uuid():
+    """Retrieve file by uuid."""
+    return """
+    SELECT 
+        uf.data, 
+        files_meta.content_type 
+    FROM files_meta 
+    JOIN uploaded_files uf ON uf.hash = files_meta.hash 
+    WHERE uuid = $1;
+    """

file_storage/typings.py

@@ -0,0 +1,8 @@
+from aiohttp import web
+from asyncpg.pool import Pool
+
+
+class AioHttpApplication(web.Application):
+    """Type hints for application with additional attributes."""
+
+    db: Pool

file_storage/views.py

@@ -0,0 +1,61 @@
+import logging
+from typing import cast
+from uuid import uuid4
+
+from aiohttp import web
+from asyncpg import Connection
+from file_storage.sql import (
+    insert_meta_query,
+    retrieve_data_by_uuid,
+    upload_data_query,
+)
+from file_storage.typings import AioHttpApplication
+from pydantic import UUID4
+
+
+class FileStorageHandler:
+    logger = logging.getLogger("FileStorageHandler")
+
+    @classmethod
+    async def put(cls, request: web.Request):
+        """Upload action handler."""
+        app: AioHttpApplication = cast(AioHttpApplication, request.app)
+        insert_data_query = upload_data_query()
+        payload = await request.read()
+        conn: Connection
+        async with app.db.acquire() as conn:
+            async with conn.transaction():
+                uploaded_file = await conn.fetchrow(insert_data_query, payload)
+                file_hash = uploaded_file["hash"]
+                file_uuid = str(uuid4())
+                cls.logger.info(
+                    f"File hash: {file_hash}, file_uuid: {file_uuid}"
+                )
+                await conn.execute(
+                    insert_meta_query(),
+                    file_uuid,
+                    request.content_type,
+                    file_hash,
+                )
+                return web.json_response(
+                    {"link": str(request.rel_url / file_uuid)},
+                    status=web.HTTPCreated.status_code,
+                )
+
+    @classmethod
+    async def get(cls, request: web.Request):
+        """Get file from storage by UUID."""
+        try:
+            uuid = UUID4(request.match_info["uuid"])
+        except ValueError:
+            raise web.HTTPBadRequest
+        app: AioHttpApplication = cast(AioHttpApplication, request.app)
+        conn: Connection
+        query = retrieve_data_by_uuid()
+        async with app.db.acquire() as conn:
+            data = await conn.fetchrow(query, uuid)
+            if not data:
+                raise web.HTTPNotFound
+            return web.Response(
+                body=data["data"], content_type=data["content_type"]
+            )