diff --git a/.github/workflows/test-ci.yml b/.github/workflows/test-ci.yml index 2e36786..c5a760e 100644 --- a/.github/workflows/test-ci.yml +++ b/.github/workflows/test-ci.yml @@ -50,6 +50,9 @@ jobs: python-version: ${{ env.PYTHON_VERSION }} cache: 'pip' + - name: Build browse-ocrd Docker image + run: make build-browse-ocrd-docker + - name: Install dependencies using pip run: pip install -e ".[dev]" diff --git a/.gitignore b/.gitignore index 1c78b23..76498d3 100644 --- a/.gitignore +++ b/.gitignore @@ -2,5 +2,5 @@ authorized_keys __pycache__/ .python-version -.pdm.toml +.pdm-python .pdm.lock diff --git a/Makefile b/Makefile index f5651c7..c883c78 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,11 @@ build: pull: docker pull $(TAGNAME) + +build-browse-ocrd-docker: + docker build -t ocrd-browser:latest -f docker-browse-ocrd/Dockerfile docker-browse-ocrd + + define HELP cat <<"EOF" Targets: @@ -68,7 +73,7 @@ test: { echo set -e; \ echo cd /usr/local/ocrd-monitor/; \ echo pip install nox; \ - echo "nox -- -m 'not needs_docker'"; } | \ + echo "nox"; } | \ docker run --rm -i \ $(TAGNAME) bash diff --git a/docker-browse-ocrd/Dockerfile b/docker-browse-ocrd/Dockerfile new file mode 100644 index 0000000..82ad71a --- /dev/null +++ b/docker-browse-ocrd/Dockerfile @@ -0,0 +1,17 @@ +FROM python:3.7 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends libcairo2-dev libgtk-3-bin libgtk-3-dev libglib2.0-dev libgtksourceview-3.0-dev libgirepository1.0-dev gir1.2-webkit2-4.0 pkg-config cmake \ + && pip3 install -U setuptools \ + && pip3 install browse-ocrd + +ENV GDK_BACKEND broadway +ENV BROADWAY_DISPLAY :5 + +EXPOSE 8085 + +COPY init.sh /init.sh + +RUN chmod +x /init.sh + +CMD ["/init.sh"] \ No newline at end of file diff --git a/docker-browse-ocrd/init.sh b/docker-browse-ocrd/init.sh new file mode 100644 index 0000000..2c2e572 --- /dev/null +++ b/docker-browse-ocrd/init.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +set -x +nohup broadwayd :5 & +browse-ocrd /data/mets.xml \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index afbe83a..cd42cf9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,6 +2,9 @@ version: "3.9" services: ocrd-monitor: + depends_on: + - ocrd-database + build: context: . # args: @@ -15,6 +18,7 @@ services: environment: MONITOR_PORT_LOG: ${MONITOR_PORT_LOG} CONTROLLER: "${CONTROLLER_HOST}:${CONTROLLER_PORT_SSH}" + DB_CONNECTION: "mongodb://${DB_ROOT_USER:-root}:${DB_ROOT_PASSWORD:-root_password}@ocrd-database:27017" ports: - ${MONITOR_PORT_WEB}:5000 @@ -37,5 +41,29 @@ services: # DOZZLE_USERNAME= # DOZZLE_PASSWORD= + ocrd-database: + image: "mongo:latest" + container_name: ocrd-database + + environment: + MONGO_INITDB_ROOT_USERNAME: ${DB_ROOT_USER:-root} + MONGO_INITDB_ROOT_PASSWORD: ${DB_ROOT_PASSWORD:-root_password} + + volumes: + - db-volume:/data/db + + + mongo-express: + image: mongo-express:latest + depends_on: + - ocrd-database + ports: + - 8081:8081 + environment: + ME_CONFIG_MONGODB_ADMINUSERNAME: ${DB_ROOT_USER:-root} + ME_CONFIG_MONGODB_ADMINPASSWORD: ${DB_ROOT_PASSWORD:-root_password} + ME_CONFIG_MONGODB_SERVER: ocrd-database + volumes: + db-volume: shared: diff --git a/init.sh b/init.sh index f74408e..e75de6b 100755 --- a/init.sh +++ b/init.sh @@ -20,6 +20,7 @@ fi export OCRD_BROWSER__MODE=native export OCRD_BROWSER__WORKSPACE_DIR=/data export OCRD_BROWSER__PORT_RANGE="[9000,9100]" +export OCRD_BROWSER__DB_CONNECTION_STRING=$DB_CONNECTION export OCRD_LOGVIEW__PORT=$MONITOR_PORT_LOG export OCRD_CONTROLLER__JOB_DIR=/run/lock/ocrd.jobs export OCRD_CONTROLLER__HOST=$CONTROLLER_HOST diff --git a/ocrdbrowser/__init__.py b/ocrdbrowser/__init__.py index 5528a80..409d938 100644 --- a/ocrdbrowser/__init__.py +++ b/ocrdbrowser/__init__.py @@ -5,33 +5,23 @@ OcrdBrowser, OcrdBrowserClient, OcrdBrowserFactory, - filter_owned, - in_other_workspaces, - in_same_workspace, - launch, - stop_all, - stop_owned_in_workspace, ) -from ._docker import DockerOcrdBrowserFactory -from ._port import NoPortsAvailableError -from ._subprocess import SubProcessOcrdBrowserFactory from ._client import HttpBrowserClient +from ._docker import DockerOcrdBrowser, DockerOcrdBrowserFactory +from ._port import NoPortsAvailableError +from ._subprocess import SubProcessOcrdBrowser, SubProcessOcrdBrowserFactory __all__ = [ "Channel", "ChannelClosed", + "DockerOcrdBrowser", "DockerOcrdBrowserFactory", "HttpBrowserClient", "NoPortsAvailableError", "OcrdBrowser", "OcrdBrowserClient", "OcrdBrowserFactory", + "SubProcessOcrdBrowser", "SubProcessOcrdBrowserFactory", - "filter_owned", - "launch", - "in_other_workspaces", - "in_same_workspace", - "stop_all", - "stop_owned_in_workspace", "workspace", ] diff --git a/ocrdbrowser/_browser.py b/ocrdbrowser/_browser.py index aa9d713..f6f9baf 100644 --- a/ocrdbrowser/_browser.py +++ b/ocrdbrowser/_browser.py @@ -6,6 +6,9 @@ class OcrdBrowser(Protocol): + def process_id(self) -> str: + ... + def address(self) -> str: ... @@ -18,9 +21,6 @@ def workspace(self) -> str: def client(self) -> OcrdBrowserClient: ... - async def start(self) -> None: - ... - async def stop(self) -> None: ... @@ -46,67 +46,5 @@ def open_channel(self) -> AsyncContextManager[Channel]: class OcrdBrowserFactory(Protocol): - def __call__(self, owner: str, workspace_path: str) -> OcrdBrowser: + async def __call__(self, owner: str, workspace_path: str) -> OcrdBrowser: ... - - -BrowserProcesses = set[OcrdBrowser] - - -async def launch( - workspace_path: str, - owner: str, - browser_factory: OcrdBrowserFactory, - running_browsers: BrowserProcesses | None = None, -) -> OcrdBrowser: - running_browsers = running_browsers or set() - owned_processes = filter_owned(owner, running_browsers) - in_workspace = in_same_workspace(workspace_path, owned_processes) - - if in_workspace: - return in_workspace.pop() - - return await start_process(browser_factory, workspace_path, owner) - - -def in_same_workspace( - workspace_path: str, browser_processes: BrowserProcesses -) -> BrowserProcesses: - workspace_path = path.abspath(workspace_path) - return { - p for p in browser_processes if path.abspath(p.workspace()) == workspace_path - } - - -def in_other_workspaces( - workspace_path: str, browser_processes: BrowserProcesses -) -> BrowserProcesses: - workspace_path = path.abspath(workspace_path) - return {p for p in browser_processes if p.workspace() != workspace_path} - - -def filter_owned(owner: str, running_processes: BrowserProcesses) -> BrowserProcesses: - return {p for p in running_processes if p.owner() == owner} - - -async def stop_all(owned_processes: BrowserProcesses) -> None: - async with asyncio.TaskGroup() as group: - for p in owned_processes: - group.create_task(p.stop()) - - -async def stop_owned_in_workspace( - owner: str, workspace: str, browsers: set[OcrdBrowser] -) -> set[OcrdBrowser]: - owned = filter_owned(owner, browsers) - in_workspace = in_same_workspace(workspace, owned) - await stop_all(in_workspace) - return in_workspace - - -async def start_process( - process_factory: OcrdBrowserFactory, workspace_path: str, owner: str -) -> OcrdBrowser: - process = process_factory(owner, workspace_path) - await process.start() - return process diff --git a/ocrdbrowser/_client.py b/ocrdbrowser/_client.py index 10ab34e..3e3a36f 100644 --- a/ocrdbrowser/_client.py +++ b/ocrdbrowser/_client.py @@ -1,5 +1,7 @@ from __future__ import annotations +import logging + from types import TracebackType from typing import AsyncContextManager, Type, cast @@ -66,9 +68,14 @@ def __init__(self, address: str) -> None: self.address = address async def get(self, resource: str) -> bytes: - async with httpx.AsyncClient(base_url=self.address) as client: - response = await client.get(resource) - return response.content + try: + async with httpx.AsyncClient(base_url=self.address) as client: + response = await client.get(resource) + return response.content + except Exception as ex: + logging.error(f"Tried to connect to {self.address}") + logging.error(f"Requested resource {resource}") + raise ConnectionError from ex def open_channel(self) -> AsyncContextManager[Channel]: return WebSocketChannel(self.address + "/socket") diff --git a/ocrdbrowser/_docker.py b/ocrdbrowser/_docker.py index 4858eb1..96b2922 100644 --- a/ocrdbrowser/_docker.py +++ b/ocrdbrowser/_docker.py @@ -1,37 +1,43 @@ from __future__ import annotations import asyncio +import functools import logging import os.path as path from typing import Any from ._browser import OcrdBrowser, OcrdBrowserClient -from ._port import Port from ._client import HttpBrowserClient +from ._port import PortBindingError, PortBindingResult, try_bind _docker_run = "docker run --rm -d --name {} -v {}:/data -p {}:8085 ocrd-browser:latest" _docker_stop = "docker stop {}" _docker_kill = "docker kill {}" -async def _run_command(cmd: str, *args: Any) -> asyncio.subprocess.Process: +async def run_command(cmd: str, *args: Any) -> asyncio.subprocess.Process: command = cmd.format(*args) return await asyncio.create_subprocess_shell( command, stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, ) class DockerOcrdBrowser: - def __init__(self, host: str, port: Port, owner: str, workspace: str) -> None: - self._host = host - self._port = port + def __init__( + self, owner: str, workspace: str, address: str, process_id: str + ) -> None: self._owner = owner - self._workspace = path.abspath(workspace) - self.id: str | None = None + self._workspace = workspace + self._address = address + self._process_id: str = process_id + + def process_id(self) -> str: + return self._process_id def address(self) -> str: - return f"{self._host}:{self._port}" + return self._address def workspace(self) -> str: return self._workspace @@ -39,32 +45,17 @@ def workspace(self) -> str: def owner(self) -> str: return self._owner - async def start(self) -> None: - cmd = await _run_command( - _docker_run, self._container_name(), self._workspace, self._port.get() - ) - self.id = str(cmd.stdout).strip() - async def stop(self) -> None: - cmd = await _run_command( - _docker_stop, self._container_name(), self.workspace(), self._port.get() - ) + cmd = await run_command(_docker_stop, self._process_id) if cmd.returncode != 0: logging.info( - f"Stopping container {self.id} returned exit code {cmd.returncode}" + f"Stopping container {self.process_id} returned exit code {cmd.returncode}" ) - self._port.release() - self.id = None - def client(self) -> OcrdBrowserClient: return HttpBrowserClient(self.address()) - def _container_name(self) -> str: - workspace = path.basename(self.workspace()) - return f"ocrd-browser-{self.owner()}-{workspace}" - class DockerOcrdBrowserFactory: def __init__(self, host: str, available_ports: set[int]) -> None: @@ -72,16 +63,63 @@ def __init__(self, host: str, available_ports: set[int]) -> None: self._ports = available_ports self._containers: list[DockerOcrdBrowser] = [] - def __call__(self, owner: str, workspace_path: str) -> OcrdBrowser: - container = DockerOcrdBrowser( - self._host, Port(self._ports), owner, workspace_path - ) + async def __call__(self, owner: str, workspace_path: str) -> OcrdBrowser: + abs_workspace = path.abspath(workspace_path) + port_binding = functools.partial(start_browser, owner, abs_workspace) + container, _ = await try_bind(port_binding, self._host, self._ports) self._containers.append(container) return container async def stop_all(self) -> None: - running_ids = [c.id for c in self._containers if c.id] + running_ids = [c.process_id() for c in self._containers] if running_ids: - await _run_command(_docker_kill, " ".join(running_ids)) + cmd = await run_command(_docker_kill, " ".join(running_ids)) + await cmd.wait() self._containers = [] + + +async def start_browser( + owner: str, workspace: str, host: str, port: int +) -> PortBindingResult[DockerOcrdBrowser]: + cmd = await run_command( + _docker_run, container_name(owner, workspace), workspace, port + ) + + return_code = await wait_for(cmd) + if return_code != 0: + return PortBindingError() + + container = DockerOcrdBrowser( + owner, workspace, f"{host}:{port}", await read_container_id(cmd) + ) + + return container + + +def container_name(owner: str, workspace: str) -> str: + workspace = path.basename(workspace) + return f"ocrd-browser-{owner}-{workspace}" + + +async def wait_for(cmd: asyncio.subprocess.Process) -> int: + return_code = await cmd.wait() + await log_from_stream(cmd.stderr) + + return return_code + + +async def read_container_id(cmd: asyncio.subprocess.Process) -> str: + stdout = cmd.stdout + container_id = "" + if stdout: + container_id = str(await stdout.read()).strip() + + return container_id + + +async def log_from_stream(stream: asyncio.StreamReader | None) -> None: + if not stream: + return + + logging.info(await stream.read()) diff --git a/ocrdbrowser/_port.py b/ocrdbrowser/_port.py index 02938ac..2e5abff 100644 --- a/ocrdbrowser/_port.py +++ b/ocrdbrowser/_port.py @@ -1,32 +1,37 @@ from __future__ import annotations - -from typing import Optional, Set +import logging +from typing import Awaitable, Callable, Generic, Iterable, NamedTuple, TypeVar, Union class NoPortsAvailableError(RuntimeError): pass -class Port: - def __init__(self, available_ports: Set[int]) -> None: - self._available_ports = available_ports - self._port: Optional[int] = self._try_pop() +T = TypeVar("T") + + +class PortBindingError(RuntimeError): + pass + + +PortBindingResult = Union[T, PortBindingError] +PortBinding = Callable[[str, int], Awaitable[PortBindingResult[T]]] + + +class BoundPort(NamedTuple, Generic[T]): + bound_app: T + port: int - def get(self) -> int: - return self._port or self._try_pop() - def release(self) -> None: - if not self._port: - return - self._available_ports.add(self._port) - self._port = None +async def try_bind( + binding: PortBinding[T], host: str, ports: Iterable[int] +) -> BoundPort[T]: + for port in ports: + result = await binding(host, port) + if isinstance(result, PortBindingError): + logging.info(f"Port {port} already in use, continuing to next port") + continue - def _try_pop(self) -> int: - # FIXME: check if port is still free - try: - return self._available_ports.pop() - except KeyError as err: - raise NoPortsAvailableError() from err + return BoundPort(result, port) - def __str__(self) -> str: - return str(self._port) + raise NoPortsAvailableError() diff --git a/ocrdbrowser/_subprocess.py b/ocrdbrowser/_subprocess.py index 832ea09..7989dd6 100644 --- a/ocrdbrowser/_subprocess.py +++ b/ocrdbrowser/_subprocess.py @@ -1,32 +1,47 @@ from __future__ import annotations import asyncio +import functools import logging import os +import signal from shutil import which -from typing import Optional +from typing import NamedTuple, Self, Type, cast from ._browser import OcrdBrowser, OcrdBrowserClient -from ._port import Port from ._client import HttpBrowserClient +from ._port import PortBindingError, PortBindingResult, try_bind BROADWAY_BASE_PORT = 8080 +class BroadwayBrowserId(NamedTuple): + broadway_pid: int + browser_pid: int + + @classmethod + def from_str(cls: Type["BroadwayBrowserId"], id_str: str) -> "BroadwayBrowserId": + ids = map(int, id_str.split("-")) + return BroadwayBrowserId(*ids) + + def __str__(self) -> str: + return f"{self.broadway_pid}-{self.browser_pid}" + + class SubProcessOcrdBrowser: - def __init__(self, localport: Port, owner: str, workspace: str) -> None: - self._localport = localport + def __init__( + self, owner: str, workspace: str, address: str, process_id: str + ) -> None: self._owner = owner self._workspace = workspace - self._process: Optional[asyncio.subprocess.Process] = None + self._address = address + self._process_id = BroadwayBrowserId.from_str(process_id) + + def process_id(self) -> str: + return str(self._process_id) def address(self) -> str: - # as long as we do not have a reverse proxy on BW_PORT, - # we must map the local port range to the exposed range - # (we use 8085 as fixed start of the internal port range, - # and map to the runtime corresponding external port) - localport = self._localport.get() - return "http://localhost:" + str(localport) + return self._address def workspace(self) -> str: return self._workspace @@ -34,51 +49,108 @@ def workspace(self) -> str: def owner(self) -> str: return self._owner - async def start(self) -> None: - browse_ocrd = which("browse-ocrd") - if not browse_ocrd: - raise FileNotFoundError("Could not find browse-ocrd executable") - localport = self._localport.get() - # broadwayd (which uses WebSockets) only allows a single client at a time - # (disconnecting concurrent connections), hence we must start a new daemon - # for each new browser session - # broadwayd starts counting virtual X displays from port 8080 as :0 - displayport = str(localport - BROADWAY_BASE_PORT) - environment = dict(os.environ) - environment["GDK_BACKEND"] = "broadway" - environment["BROADWAY_DISPLAY"] = ":" + displayport - - self._process = await asyncio.create_subprocess_shell( - " ".join( - [ - "broadwayd", - ":" + displayport + " &", - browse_ocrd, - self._workspace + "/mets.xml ;", - "kill $!", - ] - ), - env=environment, - ) - async def stop(self) -> None: - if self._process: - try: - self._process.terminate() - except ProcessLookupError: - logging.info( - f"Attempted to stop already terminated process {self._process.pid}" - ) - finally: - self._localport.release() + self._try_kill(self._process_id.broadway_pid) + self._try_kill(self._process_id.browser_pid) + + @staticmethod + def _try_kill(pid: int) -> None: + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + logging.warning(f"Could not find process with ID {pid}") def client(self) -> OcrdBrowserClient: return HttpBrowserClient(self.address()) +class ProcessLaunchFailedError(RuntimeError): + pass + + class SubProcessOcrdBrowserFactory: def __init__(self, available_ports: set[int]) -> None: self._available_ports = available_ports - def __call__(self, owner: str, workspace_path: str) -> OcrdBrowser: - return SubProcessOcrdBrowser(Port(self._available_ports), owner, workspace_path) + async def __call__(self, owner: str, workspace_path: str) -> OcrdBrowser: + port_binding = functools.partial(start_browser, workspace_path) + pid, port = await try_bind( + port_binding, "http://localhost", self._available_ports + ) + + address = f"http://localhost:{port}" + return SubProcessOcrdBrowser(owner, workspace_path, address, str(pid)) + + +async def start_browser( + workspace: str, host: str, port: int +) -> PortBindingResult[BroadwayBrowserId]: + find_executables_or_raise() + + # broadwayd (which uses WebSockets) only allows a single client at a time + # (disconnecting concurrent connections), hence we must start a new daemon + # for each new browser session + # broadwayd starts counting virtual X displays from port 8080 as :0 + displayport = str(port - BROADWAY_BASE_PORT) + + try: + broadway_process = await launch_broadway(displayport) + + if broadway_process is None: + return PortBindingError() + + environment = prepare_env(displayport) + full_cmd = browser_command(workspace, broadway_process.pid) + browser_process = await asyncio.create_subprocess_shell( + full_cmd, env=environment + ) + + return BroadwayBrowserId(broadway_process.pid, browser_process.pid) + except Exception as err: + logging.error(f"Failed to launch broadway at (real port {port})") + logging.error(repr(err)) + return PortBindingError() + + +def find_executables_or_raise() -> None: + if not which("broadwayd"): + raise FileNotFoundError("Could not find broadwayd executable") + + if not which("browse-ocrd"): + raise FileNotFoundError("Could not find browse-ocrd executable") + + +async def launch_broadway( + displayport: str, +) -> asyncio.subprocess.Process | None: + broadway = cast(str, which("broadwayd")) + broadway_process = await asyncio.create_subprocess_exec( + broadway, f":{displayport}", stderr=asyncio.subprocess.PIPE + ) + + try: + stderr = cast(asyncio.StreamReader, broadway_process.stderr) + err_output = await asyncio.wait_for(stderr.readline(), 5) + if b"Address already in use" in err_output: + return None + except asyncio.TimeoutError: + logging.info( + "The process didn't exit within the given timeout." + + f"Assuming broadway on port {displayport} launched successfully" + ) + + return broadway_process + + +def prepare_env(displayport: str) -> dict[str, str]: + environment = dict(os.environ) + environment["GDK_BACKEND"] = "broadway" + environment["BROADWAY_DISPLAY"] = ":" + displayport + return environment + + +def browser_command(workspace: str, broadway_pid: int) -> str: + mets_path = workspace + "/mets.xml" + kill_broadway = f"; kill {broadway_pid}" + browse_ocrd = cast(str, which("browse-ocrd")) + return " ".join([browse_ocrd, mets_path, kill_broadway]) diff --git a/ocrdmonitor/browserprocess.py b/ocrdmonitor/browserprocess.py new file mode 100644 index 0000000..15df446 --- /dev/null +++ b/ocrdmonitor/browserprocess.py @@ -0,0 +1,31 @@ +from typing import Collection, Protocol +from ocrdbrowser import OcrdBrowser + + +class BrowserRestoringFactory(Protocol): + def __call__( + self, owner: str, workspace: str, address: str, process_id: str + ) -> OcrdBrowser: + ... + + +class BrowserProcessRepository(Protocol): + async def insert(self, browser: OcrdBrowser) -> None: + ... + + async def delete(self, browser: OcrdBrowser) -> None: + ... + + async def find( + self, + *, + owner: str | None = None, + workspace: str | None = None, + ) -> Collection[OcrdBrowser]: + ... + + async def first(self, *, owner: str, workspace: str) -> OcrdBrowser | None: + ... + + async def count(self) -> int: + ... diff --git a/ocrdmonitor/dbmodel.py b/ocrdmonitor/dbmodel.py new file mode 100644 index 0000000..8d2910c --- /dev/null +++ b/ocrdmonitor/dbmodel.py @@ -0,0 +1,155 @@ +import asyncio +import urllib +from typing import Any, Collection, Mapping, Protocol + +import pymongo +from beanie import Document, init_beanie +from beanie.odm.queries.find import FindMany +from motor.motor_asyncio import AsyncIOMotorClient + +from ocrdbrowser import OcrdBrowser +from ocrdmonitor.browserprocess import BrowserRestoringFactory + + +class BrowserProcess(Document): + address: str + owner: str + process_id: str + workspace: str + + class Settings: + indexes = [ + pymongo.IndexModel( + [ + ("owner", pymongo.ASCENDING), + ("workspace", pymongo.ASCENDING), + ] + ) + ] + + +class MongoBrowserProcessRepository: + def __init__(self, restoring_factory: BrowserRestoringFactory) -> None: + self._restoring_factory = restoring_factory + + async def insert(self, browser: OcrdBrowser) -> None: + await BrowserProcess( # type: ignore + address=browser.address(), + owner=browser.owner(), + process_id=browser.process_id(), + workspace=browser.workspace(), + ).insert() + + async def delete(self, browser: OcrdBrowser) -> None: + result = await BrowserProcess.find_one( + BrowserProcess.owner == browser.owner(), + BrowserProcess.workspace == browser.workspace(), + BrowserProcess.address == browser.address(), + BrowserProcess.process_id == browser.process_id(), + ) + + if not result: + return + + await result.delete() + + async def find( + self, + *, + owner: str | None = None, + workspace: str | None = None, + ) -> Collection[OcrdBrowser]: + results: FindMany[BrowserProcess] | None = None + + def find( + results: FindMany[BrowserProcess] | None, + *predicates: Mapping[str, Any] | bool, + ) -> FindMany[BrowserProcess]: + if results is None: + return BrowserProcess.find(*predicates) + + return results.find(*predicates) + + if owner is not None: + results = find(results, BrowserProcess.owner == owner) + + if workspace is not None: + results = find(results, BrowserProcess.workspace == workspace) + + if results is None: + results = BrowserProcess.find_all() + + return [ + self._restoring_factory( + browser.owner, + browser.workspace, + browser.address, + browser.process_id, + ) + for browser in await results.to_list() + ] + + async def first(self, owner: str, workspace: str) -> OcrdBrowser | None: + result = await BrowserProcess.find_one( + BrowserProcess.owner == owner, + BrowserProcess.workspace == workspace, + ) + + if result is None: + return None + + return self._restoring_factory( + result.owner, + result.workspace, + result.address, + result.process_id, + ) + + async def count(self) -> int: + return await BrowserProcess.count() + + async def clean(self) -> None: + await BrowserProcess.delete_all() + + +def rebuild_connection_string(connection_str: str) -> str: + connection_str = connection_str.removeprefix("mongodb://") + credentials, host = connection_str.split("@") + user, password = credentials.split(":") + password = urllib.parse.quote(password) + return f"mongodb://{user}:{password}@{host}" + + +class InitDatabase(Protocol): + async def __call__( + self, connection_str: str, force_initialize: bool = False + ) -> None: + ... + + +def __beanie_initializer() -> InitDatabase: + """ + We use this as a workaround to prevent beanie from being initialized + multiple times when requesting the repository from OcrdBrowserSettings + unless stated explicitly (e.g. for testing purposes) + """ + __initialized = False + + async def init(connection_str: str, force_initialize: bool = False) -> None: + nonlocal __initialized + if __initialized and not force_initialize: + return + + __initialized = True + connection_str = rebuild_connection_string(connection_str) + client: AsyncIOMotorClient = AsyncIOMotorClient(connection_str) + client.get_io_loop = asyncio.get_event_loop + await init_beanie( + database=client.browsers, + document_models=[BrowserProcess], # type: ignore + ) + + return init + + +init = __beanie_initializer() diff --git a/ocrdmonitor/main.py b/ocrdmonitor/main.py index 409c26d..c30568b 100644 --- a/ocrdmonitor/main.py +++ b/ocrdmonitor/main.py @@ -2,4 +2,4 @@ from ocrdmonitor.server.app import create_app settings = Settings() -app = create_app(settings) +app = create_app(settings) \ No newline at end of file diff --git a/ocrdmonitor/server/app.py b/ocrdmonitor/server/app.py index df1a593..98358f5 100644 --- a/ocrdmonitor/server/app.py +++ b/ocrdmonitor/server/app.py @@ -1,14 +1,17 @@ import logging from pathlib import Path -from fastapi import FastAPI, Request, Response -from fastapi.responses import RedirectResponse +from fastapi import FastAPI, Request, Response, status +from fastapi.encoders import jsonable_encoder +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse, RedirectResponse from fastapi.staticfiles import StaticFiles from fastapi.templating import Jinja2Templates from ocrdmonitor.ocrdcontroller import OcrdController from ocrdmonitor.server.index import create_index from ocrdmonitor.server.jobs import create_jobs +from ocrdmonitor.server.lifespan import lifespan from ocrdmonitor.server.logs import create_logs from ocrdmonitor.server.logview import create_logview from ocrdmonitor.server.settings import Settings @@ -21,7 +24,7 @@ def create_app(settings: Settings) -> FastAPI: - app = FastAPI() + app = FastAPI(lifespan=lifespan(settings.ocrd_browser)) templates = Jinja2Templates(TEMPLATE_DIR) app.mount("/static", StaticFiles(directory=STATIC_DIR), name="static") @@ -30,6 +33,19 @@ async def swallow_exceptions(request: Request, err: Exception) -> Response: logging.error(err) return RedirectResponse("/") + @app.exception_handler(RequestValidationError) + async def validation_exception( + request: Request, exc: RequestValidationError + ) -> Response: + logging.error(f"Unprocessable entity on route {request.url}") + logging.error("Error details:") + logging.error(exc.errors()) + logging.error(exc.body) + return JSONResponse( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + content=jsonable_encoder({"detail": exc.errors(), "body": exc.body}), + ) + app.include_router(create_index(templates)) app.include_router( create_jobs( @@ -40,13 +56,7 @@ async def swallow_exceptions(request: Request, err: Exception) -> Response: ), ) ) - app.include_router( - create_workspaces( - templates, - settings.ocrd_browser.factory(), - settings.ocrd_browser.workspace_dir, - ) - ) + app.include_router(create_workspaces(templates, settings.ocrd_browser)) app.include_router(create_logs(templates, settings.ocrd_browser.workspace_dir)) app.include_router(create_workflows(templates)) app.include_router(create_logview(templates, settings.ocrd_logview.port)) diff --git a/ocrdmonitor/server/lifespan.py b/ocrdmonitor/server/lifespan.py new file mode 100644 index 0000000..9a3cfb6 --- /dev/null +++ b/ocrdmonitor/server/lifespan.py @@ -0,0 +1,36 @@ +import asyncio +from contextlib import asynccontextmanager +from typing import AsyncContextManager, AsyncIterator, Callable + +from fastapi import FastAPI + +from ocrdbrowser import OcrdBrowser +from ocrdmonitor.browserprocess import BrowserProcessRepository +from ocrdmonitor.server.settings import OcrdBrowserSettings + + +Lifespan = Callable[[FastAPI], AsyncContextManager[None]] + + +def lifespan(browser_settings: OcrdBrowserSettings) -> Lifespan: + @asynccontextmanager + async def _lifespan(_: FastAPI) -> AsyncIterator[None]: + await clean_unreachable_browsers(browser_settings) + yield + + return _lifespan + + +async def clean_unreachable_browsers(browser_settings: OcrdBrowserSettings) -> None: + repo = await browser_settings.repository() + all_browsers = await repo.find() + async with asyncio.TaskGroup() as group: + for browser in all_browsers: + group.create_task(ping_or_delete(repo, browser)) + + +async def ping_or_delete(repo: BrowserProcessRepository, browser: OcrdBrowser) -> None: + try: + await browser.client().get("/") + except ConnectionError: + await repo.delete(browser) diff --git a/ocrdmonitor/server/proxy.py b/ocrdmonitor/server/proxy.py deleted file mode 100644 index b83d710..0000000 --- a/ocrdmonitor/server/proxy.py +++ /dev/null @@ -1,37 +0,0 @@ -from __future__ import annotations - -import asyncio - -from fastapi import Response -from ocrdbrowser import Channel - -from .redirect import BrowserRedirect - - -async def forward(redirect: BrowserRedirect, url: str) -> Response: - redirect_url = redirect.redirect_url(url) - resource = await redirect.browser.client().get(redirect_url) - return Response(content=resource) - - -async def tunnel( - source: Channel, - target: Channel, - timeout: float = 0.001, -) -> None: - await _tunnel_one_way(source, target, timeout) - await _tunnel_one_way(target, source, timeout) - - -async def _tunnel_one_way( - source: Channel, - target: Channel, - timeout: float, -) -> None: - try: - source_data = await asyncio.wait_for(source.receive_bytes(), timeout) - await target.send_bytes(source_data) - except asyncio.exceptions.TimeoutError: - # a timeout is rather common if no data is being sent, - # so we are simply ignoring this exception - pass diff --git a/ocrdmonitor/server/redirect.py b/ocrdmonitor/server/redirect.py deleted file mode 100644 index 226c6e6..0000000 --- a/ocrdmonitor/server/redirect.py +++ /dev/null @@ -1,107 +0,0 @@ -from __future__ import annotations - -from pathlib import Path -from typing import Callable - -from ocrdbrowser import OcrdBrowser - - -def removeprefix(string: str, prefix: str) -> str: - def __removeprefix(prefix: str) -> str: - if string.startswith(prefix): - len_prefix = len(prefix) - return string[len_prefix:] - - return string - - _removeprefix: Callable[[str], str] = getattr( - string, "removeprefix", __removeprefix - ) - return _removeprefix(prefix) - - -def removesuffix(string: str, suffix: str) -> str: - def __removesuffix(suffix: str) -> str: - if string.endswith(suffix): - len_suffix = len(suffix) - return string[-len_suffix:] - - return string - - _removesuffix: Callable[[str], str] = getattr( - string, "removesuffix", __removesuffix - ) - - return _removesuffix(suffix) - - -class BrowserRedirect: - def __init__(self, workspace: Path, browser: OcrdBrowser) -> None: - self._workspace = workspace - self._browser = browser - - @property - def browser(self) -> OcrdBrowser: - return self._browser - - @property - def workspace(self) -> Path: - return self._workspace - - def redirect_url(self, url: str) -> str: - url = removeprefix(url, str(self._workspace)) - url = removeprefix(url, "/") - address = removesuffix(self._browser.address(), "/") - return removesuffix(address + "/" + url, "/") - - def matches(self, path: str) -> bool: - return path.startswith(str(self.workspace)) - - -class RedirectMap: - def __init__(self) -> None: - self._redirects: dict[str, set[BrowserRedirect]] = {} - - def add( - self, session_id: str, workspace: Path, server: OcrdBrowser - ) -> BrowserRedirect: - try: - redirect = self.get(session_id, workspace) - return redirect - except KeyError: - redirect = BrowserRedirect(workspace, server) - self._redirects.setdefault(session_id, set()).add(redirect) - return redirect - - def remove(self, session_id: str, workspace: Path) -> None: - redirect = self.get(session_id, workspace) - self._redirects[session_id].remove(redirect) - - def get(self, session_id: str, workspace: Path) -> BrowserRedirect: - redirect = next( - ( - redirect - for redirect in self._redirects.get(session_id, set()) - if redirect.matches(str(workspace)) - ), - None, - ) - - return self._instance_or_raise(redirect) - - def _instance_or_raise(self, redirect: BrowserRedirect | None) -> BrowserRedirect: - if redirect is None: - raise KeyError("No redirect found") - - return redirect - - def has_redirect_to_workspace(self, session_id: str, workspace: Path) -> bool: - try: - self.get(session_id, workspace) - return True - except KeyError: - return False - - def __contains__(self, session_and_workspace: tuple[str, Path]) -> bool: - session_id, workspace = session_and_workspace - return self.has_redirect_to_workspace(session_id, workspace) diff --git a/ocrdmonitor/server/settings.py b/ocrdmonitor/server/settings.py index 2ccfc27..ab617aa 100644 --- a/ocrdmonitor/server/settings.py +++ b/ocrdmonitor/server/settings.py @@ -1,21 +1,35 @@ from __future__ import annotations -import asyncio -import atexit +import functools from pathlib import Path -from typing import Literal +from typing import Callable, Literal, Type from pydantic import BaseModel, BaseSettings, validator from ocrdbrowser import ( + DockerOcrdBrowser, DockerOcrdBrowserFactory, OcrdBrowserFactory, SubProcessOcrdBrowserFactory, + SubProcessOcrdBrowser, ) +from ocrdmonitor import dbmodel +from ocrdmonitor.browserprocess import BrowserProcessRepository from ocrdmonitor.ocrdcontroller import RemoteServer from ocrdmonitor.sshremote import SSHRemote +BrowserType = Type[SubProcessOcrdBrowser] | Type[DockerOcrdBrowser] +CreatingFactories: dict[str, Callable[[set[int]], OcrdBrowserFactory]] = { + "native": SubProcessOcrdBrowserFactory, + "docker": functools.partial(DockerOcrdBrowserFactory, "http://localhost"), +} + +RestoringFactories: dict[str, BrowserType] = { + "native": SubProcessOcrdBrowser, + "docker": DockerOcrdBrowser, +} + class OcrdControllerSettings(BaseModel): job_dir: Path @@ -36,19 +50,18 @@ class OcrdBrowserSettings(BaseModel): workspace_dir: Path mode: Literal["native", "docker"] = "native" port_range: tuple[int, int] + db_connection_string: str - def factory(self) -> OcrdBrowserFactory: - port_range_set = set(range(*self.port_range)) - if self.mode == "native": - return SubProcessOcrdBrowserFactory(port_range_set) - else: - factory = DockerOcrdBrowserFactory("http://localhost", port_range_set) + async def repository(self) -> BrowserProcessRepository: + # if not self._repository_initialized: + await dbmodel.init(self.db_connection_string) - @atexit.register - def stop_containers() -> None: - asyncio.get_event_loop().run_until_complete(factory.stop_all()) + restore = RestoringFactories[self.mode] + return dbmodel.MongoBrowserProcessRepository(restore) - return factory + def factory(self) -> OcrdBrowserFactory: + port_range_set = set(range(*self.port_range)) + return CreatingFactories[self.mode](port_range_set) @validator("port_range", pre=True) def validator(cls, value: str | tuple[int, int]) -> tuple[int, int]: diff --git a/ocrdmonitor/server/workspaces.py b/ocrdmonitor/server/workspaces.py deleted file mode 100644 index 0dfabab..0000000 --- a/ocrdmonitor/server/workspaces.py +++ /dev/null @@ -1,109 +0,0 @@ -from __future__ import annotations - -import uuid -from pathlib import Path - - -import ocrdbrowser -import ocrdmonitor.server.proxy as proxy -from fastapi import APIRouter, Cookie, Request, Response, WebSocket, WebSocketDisconnect -from fastapi.templating import Jinja2Templates -from ocrdbrowser import ChannelClosed, OcrdBrowser, OcrdBrowserFactory, workspace -from ocrdmonitor.server.redirect import RedirectMap - - -def create_workspaces( - templates: Jinja2Templates, factory: OcrdBrowserFactory, workspace_dir: Path -) -> APIRouter: - router = APIRouter(prefix="/workspaces") - - redirects = RedirectMap() - - @router.get("/", name="workspaces.list") - def list_workspaces(request: Request) -> Response: - spaces = [ - Path(space).relative_to(workspace_dir) - for space in workspace.list_all(workspace_dir) - ] - - return templates.TemplateResponse( - "list_workspaces.html.j2", - {"request": request, "workspaces": spaces}, - ) - - @router.get("/browse/{workspace:path}", name="workspaces.browse") - async def browser(request: Request, workspace: Path) -> Response: - session_id = request.cookies.setdefault("session_id", str(uuid.uuid4())) - response = Response() - response.set_cookie("session_id", session_id) - - if (session_id, workspace) not in redirects: - browser = await launch_browser(session_id, workspace) - redirects.add(session_id, workspace, browser) - - return response - - @router.get("/open/{workspace:path}", name="workspaces.open") - def open_workspace(request: Request, workspace: str) -> Response: - return templates.TemplateResponse( - "workspace.html.j2", - {"request": request, "workspace": workspace}, - ) - - @router.get("/ping/{workspace:path}", name="workspaces.ping") - async def ping_workspace( - request: Request, workspace: Path, session_id: str = Cookie(default=None) - ) -> Response: - redirect = redirects.get(session_id, workspace) - try: - await proxy.forward(redirect, str(workspace)) - return Response(status_code=200) - except ConnectionError: - return Response(status_code=502) - - # NOTE: It is important that the route path here ends with a slash, otherwise - # the reverse routing will not work as broadway.js uses window.location - # which points to the last component with a trailing slash. - @router.get("/view/{workspace:path}/", name="workspaces.view") - async def workspace_reverse_proxy( - request: Request, workspace: Path, session_id: str = Cookie(default=None) - ) -> Response: - redirect = redirects.get(session_id, workspace) - try: - return await proxy.forward(redirect, str(workspace)) - except ConnectionError: - return templates.TemplateResponse( - "view_workspace_failed.html.j2", - {"request": request, "workspace": workspace}, - ) - - @router.websocket("/view/{workspace:path}/socket", name="workspaces.view.socket") - async def workspace_socket_proxy( - websocket: WebSocket, workspace: Path, session_id: str = Cookie(default=None) - ) -> None: - redirect = redirects.get(session_id, workspace) - await websocket.accept(subprotocol="broadway") - await communicate_with_browser_until_closed(websocket, redirect.browser) - - async def communicate_with_browser_until_closed( - websocket: WebSocket, browser: OcrdBrowser - ) -> None: - async with browser.client().open_channel() as channel: - try: - while True: - await proxy.tunnel(channel, websocket) - except ChannelClosed: - await stop_browser(browser) - except WebSocketDisconnect: - pass - - async def launch_browser(session_id: str, workspace: Path) -> OcrdBrowser: - full_workspace_path = workspace_dir / workspace - return await ocrdbrowser.launch(str(full_workspace_path), session_id, factory) - - async def stop_browser(browser: OcrdBrowser) -> None: - await browser.stop() - key = Path(browser.workspace()).relative_to(workspace_dir) - redirects.remove(browser.owner(), key) - - return router diff --git a/ocrdmonitor/server/workspaces/__init__.py b/ocrdmonitor/server/workspaces/__init__.py new file mode 100644 index 0000000..9535bcc --- /dev/null +++ b/ocrdmonitor/server/workspaces/__init__.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +from pathlib import Path + +from fastapi import APIRouter +from fastapi.templating import Jinja2Templates + +from ocrdmonitor.server.settings import OcrdBrowserSettings + +from ._launchroutes import register_launchroutes +from ._listroutes import register_listroutes +from ._proxyroutes import register_proxyroutes + + +def create_workspaces( + templates: Jinja2Templates, browser_settings: OcrdBrowserSettings +) -> APIRouter: + router = APIRouter(prefix="/workspaces") + + WORKSPACE_DIR = browser_settings.workspace_dir + + def full_workspace(workspace: Path | str) -> str: + return str(WORKSPACE_DIR / workspace) + + register_listroutes(router, templates, browser_settings) + register_launchroutes(router, templates, browser_settings, full_workspace) + register_proxyroutes(router, templates, browser_settings, full_workspace) + + return router diff --git a/ocrdmonitor/server/workspaces/_browsercommunication.py b/ocrdmonitor/server/workspaces/_browsercommunication.py new file mode 100644 index 0000000..13d66e8 --- /dev/null +++ b/ocrdmonitor/server/workspaces/_browsercommunication.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import asyncio +import logging +from difflib import SequenceMatcher +from typing import Awaitable, Callable + +from fastapi import Response + +from ocrdbrowser import Channel, ChannelClosed, OcrdBrowser + + +async def forward(browser: OcrdBrowser, partial_workspace: str) -> Response: + url = _get_redirect_url(browser, partial_workspace) + resource = await browser.client().get(url) + return Response(content=resource) + + +def _get_redirect_url(browser: OcrdBrowser, partial_workspace: str) -> str: + matcher = SequenceMatcher(None, browser.workspace(), partial_workspace) + match = matcher.find_longest_match() + return partial_workspace[match.size :] + + +CloseCallback = Callable[[OcrdBrowser], Awaitable[None]] + + +async def communicate_until_closed( + websocket: Channel, browser: OcrdBrowser, close_callback: CloseCallback +) -> None: + async with browser.client().open_channel() as channel: + try: + while True: + await _tunnel(channel, websocket) + except ChannelClosed: + await close_callback(browser) + except Exception as err: + logging.error( + f""" + An exception occurred during communication with the browser {repr(browser)}. + The exception was {repr(err)} + """ + ) + + +async def _tunnel( + source: Channel, + target: Channel, + timeout: float = 0.001, +) -> None: + await _tunnel_one_way(source, target, timeout) + await _tunnel_one_way(target, source, timeout) + + +async def _tunnel_one_way( + source: Channel, + target: Channel, + timeout: float, +) -> None: + try: + source_data = await asyncio.wait_for(source.receive_bytes(), timeout) + await target.send_bytes(source_data) + except asyncio.exceptions.TimeoutError: + # a timeout is rather common if no data is being sent, + # so we are simply ignoring this exception + pass diff --git a/ocrdmonitor/server/workspaces/_launchroutes.py b/ocrdmonitor/server/workspaces/_launchroutes.py new file mode 100644 index 0000000..e43f867 --- /dev/null +++ b/ocrdmonitor/server/workspaces/_launchroutes.py @@ -0,0 +1,49 @@ +import uuid +from pathlib import Path +from typing import Callable + +from fastapi import APIRouter, Cookie, Depends, Request, Response +from fastapi.templating import Jinja2Templates + +from ocrdbrowser import OcrdBrowserFactory +from ocrdmonitor.browserprocess import BrowserProcessRepository +from ocrdmonitor.server.settings import OcrdBrowserSettings + + +def session_response(session_id: str) -> Response: + response = Response() + response.set_cookie("session_id", session_id) + return response + + +def register_launchroutes( + router: APIRouter, + templates: Jinja2Templates, + browser_settings: OcrdBrowserSettings, + full_workspace: Callable[[str | Path], str], +) -> None: + @router.get("/open/{workspace:path}", name="workspaces.open") + def open_workspace(request: Request, workspace: str) -> Response: + session_id = request.cookies.setdefault("session_id", str(uuid.uuid4())) + response = templates.TemplateResponse( + "workspace.html.j2", + {"request": request, "session_id": session_id, "workspace": workspace}, + ) + response.set_cookie("session_id", session_id) + return response + + @router.get("/browse/{workspace:path}", name="workspaces.browse") + async def browser( + workspace: Path, + session_id: str = Cookie(), + factory: OcrdBrowserFactory = Depends(browser_settings.factory), + repository: BrowserProcessRepository = Depends(browser_settings.repository), + ) -> Response: + full_path = full_workspace(workspace) + existing_browsers = await repository.find(owner=session_id, workspace=full_path) + + if not existing_browsers: + browser = await factory(session_id, full_path) + await repository.insert(browser) + + return session_response(session_id) diff --git a/ocrdmonitor/server/workspaces/_listroutes.py b/ocrdmonitor/server/workspaces/_listroutes.py new file mode 100644 index 0000000..b2dbd5e --- /dev/null +++ b/ocrdmonitor/server/workspaces/_listroutes.py @@ -0,0 +1,23 @@ +from pathlib import Path + +from fastapi import APIRouter, Request, Response +from fastapi.templating import Jinja2Templates + +from ocrdbrowser import workspace +from ocrdmonitor.server.settings import OcrdBrowserSettings + + +def register_listroutes( + router: APIRouter, templates: Jinja2Templates, browser_settings: OcrdBrowserSettings +) -> None: + @router.get("/", name="workspaces.list") + def list_workspaces(request: Request) -> Response: + spaces = [ + Path(space).relative_to(browser_settings.workspace_dir) + for space in workspace.list_all(browser_settings.workspace_dir) + ] + + return templates.TemplateResponse( + "list_workspaces.html.j2", + {"request": request, "workspaces": spaces}, + ) diff --git a/ocrdmonitor/server/workspaces/_proxyroutes.py b/ocrdmonitor/server/workspaces/_proxyroutes.py new file mode 100644 index 0000000..1ff040d --- /dev/null +++ b/ocrdmonitor/server/workspaces/_proxyroutes.py @@ -0,0 +1,127 @@ +from __future__ import annotations +import asyncio +import logging + +from pathlib import Path +from typing import Callable + +from fastapi import APIRouter, Cookie, Depends, Request, Response, WebSocket +from fastapi.templating import Jinja2Templates + +from ocrdbrowser import OcrdBrowser +from ocrdmonitor.browserprocess import BrowserProcessRepository +from ocrdmonitor.server.settings import OcrdBrowserSettings + +from ._browsercommunication import CloseCallback, communicate_until_closed, forward + + +async def stop_and_remove_browser( + repository: BrowserProcessRepository, browser: OcrdBrowser +) -> None: + async with asyncio.TaskGroup() as group: + group.create_task(browser.stop()) + group.create_task(repository.delete(browser)) + logging.info(f"Stopping browser {browser.workspace()}") + + +async def first_owned_browser_in_workspace( + session_id: str, workspace: str, repository: BrowserProcessRepository +) -> OcrdBrowser | None: + def in_workspace(browser: OcrdBrowser) -> bool: + return workspace.startswith(browser.workspace()) + + browsers_in_workspace = filter( + in_workspace, await repository.find(owner=session_id) + ) + return next(browsers_in_workspace, None) + + +def browser_closed_callback(repository: BrowserProcessRepository) -> CloseCallback: + async def _callback(browser: OcrdBrowser) -> None: + await stop_and_remove_browser(repository, browser) + + return _callback + + +def get_session_id(request: Request, session_id: str | None) -> str: + return session_id or request.cookies["session_id"] + + +def register_proxyroutes( + router: APIRouter, + templates: Jinja2Templates, + browser_settings: OcrdBrowserSettings, + full_workspace: Callable[[str | Path], str], +) -> None: + @router.get("/ping/{workspace:path}", name="workspaces.ping") + async def ping_workspace( + workspace: Path, + session_id: str = Cookie(), + repository: BrowserProcessRepository = Depends(browser_settings.repository), + ) -> Response: + browser = await repository.first( + owner=session_id, workspace=full_workspace(workspace) + ) + + if not browser: + return Response(status_code=404) + + try: + await forward(browser, str(workspace)) + return Response(status_code=200) + except ConnectionError: + return Response(status_code=502) + + # NOTE: It is important that the route path here ends with a slash, otherwise + # the reverse routing will not work as broadway.js uses window.location + # which points to the last component with a trailing slash. + @router.get("/view/{workspace:path}/", name="workspaces.view") + async def workspace_reverse_proxy( + request: Request, + workspace: Path, + session_id: str = Cookie(default=None), + repository: BrowserProcessRepository = Depends(browser_settings.repository), + ) -> Response: + # The session_id cookie is not always properly injected for some reason + # Therefore we try to get it from the request if it is None + session_id = get_session_id(request, session_id) + + browser = await first_owned_browser_in_workspace( + session_id, full_workspace(workspace), repository + ) + + if not browser: + return Response( + content=f"No browser found for {workspace} and session ID {session_id}", + status_code=404, + ) + try: + return await forward(browser, str(workspace)) + except ConnectionError: + await stop_and_remove_browser(repository, browser) + return templates.TemplateResponse( + "view_workspace_failed.html.j2", + {"request": request, "workspace": workspace}, + ) + + @router.websocket("/view/{workspace:path}/socket", name="workspaces.view.socket") + async def workspace_socket_proxy( + websocket: WebSocket, + workspace: Path, + session_id: str = Cookie(), + repository: BrowserProcessRepository = Depends(browser_settings.repository), + ) -> None: + browser = await repository.first( + owner=session_id, workspace=full_workspace(workspace) + ) + + if browser is None: + await websocket.close(reason="No browser found") + return + + await websocket.accept(subprotocol="broadway") + await communicate_until_closed( + websocket, + browser, + close_callback=browser_closed_callback(repository), + ) diff --git a/pdm.lock b/pdm.lock index 927db5f..9131652 100644 --- a/pdm.lock +++ b/pdm.lock @@ -23,6 +23,19 @@ version = "22.2.0" requires_python = ">=3.6" summary = "Classes Without Boilerplate" +[[package]] +name = "beanie" +version = "1.18.0" +requires_python = ">=3.7,<4.0" +summary = "Asynchronous Python ODM for MongoDB" +dependencies = [ + "click>=7", + "lazy-model>=0.0.3", + "motor<4.0,>=2.5", + "pydantic>=1.10.0", + "toml", +] + [[package]] name = "beautifulsoup4" version = "4.12.0" @@ -94,6 +107,12 @@ name = "distlib" version = "0.3.6" summary = "Distribution utilities" +[[package]] +name = "dnspython" +version = "2.3.0" +requires_python = ">=3.7,<4.0" +summary = "DNS toolkit" + [[package]] name = "docker" version = "6.0.1" @@ -174,15 +193,33 @@ dependencies = [ "MarkupSafe>=2.0", ] +[[package]] +name = "lazy-model" +version = "0.0.5" +requires_python = ">=3.7,<4.0" +summary = "" +dependencies = [ + "pydantic>=1.9.0", +] + [[package]] name = "markupsafe" version = "2.1.2" requires_python = ">=3.7" summary = "Safely add untrusted strings to HTML/XML markup." +[[package]] +name = "motor" +version = "3.1.2" +requires_python = ">=3.7" +summary = "Non-blocking MongoDB driver for Tornado or asyncio" +dependencies = [ + "pymongo<5,>=4.1", +] + [[package]] name = "mypy" -version = "1.1.1" +version = "1.3.0" requires_python = ">=3.7" summary = "Optional static typing for Python" dependencies = [ @@ -252,6 +289,15 @@ dependencies = [ "python-dotenv>=0.10.4", ] +[[package]] +name = "pymongo" +version = "4.3.3" +requires_python = ">=3.7" +summary = "Python driver for MongoDB " +dependencies = [ + "dnspython<3.0.0,>=1.16.0", +] + [[package]] name = "pytest" version = "7.2.2" @@ -344,6 +390,12 @@ dependencies = [ "wrapt", ] +[[package]] +name = "toml" +version = "0.10.2" +requires_python = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +summary = "Python Library for Tom's Obvious, Minimal Language" + [[package]] name = "types-beautifulsoup4" version = "4.12.0.0" @@ -422,8 +474,10 @@ requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" summary = "Module for decorators, wrappers and monkey patching." [metadata] -lock_version = "4.1" -content_hash = "sha256:41531e3c394e45bd34b4d98fb20ef889335291b731a867850f266b07050d5f4e" +lock_version = "4.2" +cross_platform = true +groups = ["default", "dev", "nox"] +content_hash = "sha256:19d49bce6b07f9f1bacc395b0a137186149b00ad97e508f457767f79d7935f86" [metadata.files] "anyio 3.6.2" = [ @@ -438,6 +492,10 @@ content_hash = "sha256:41531e3c394e45bd34b4d98fb20ef889335291b731a867850f266b070 {url = "https://files.pythonhosted.org/packages/21/31/3f468da74c7de4fcf9b25591e682856389b3400b4b62f201e65f15ea3e07/attrs-22.2.0.tar.gz", hash = "sha256:c9227bfc2f01993c03f68db37d1d15c9690188323c067c641f1a35ca58185f99"}, {url = "https://files.pythonhosted.org/packages/fb/6e/6f83bf616d2becdf333a1640f1d463fef3150e2e926b7010cb0f81c95e88/attrs-22.2.0-py3-none-any.whl", hash = "sha256:29e95c7f6778868dbd49170f98f8818f78f3dc5e0e37c0b1f474e3561b240836"}, ] +"beanie 1.18.0" = [ + {url = "https://files.pythonhosted.org/packages/54/95/dbf00d5776e1a6f761262003e9abbc14706952e16018e8d6449a845a4c4d/beanie-1.18.0.tar.gz", hash = "sha256:1e1205d41176fe5b010cf04964c827841fe9c2c9cffc5ba5a29f66fb2f9c2e68"}, + {url = "https://files.pythonhosted.org/packages/d9/8b/a371c3fb5db326675720f66ac65f22f94f0b4c3830ccc65e0acecd352d2a/beanie-1.18.0-py3-none-any.whl", hash = "sha256:31f8eff8fe436e420766df457dc5bfd6025e927c9f8914adbc59ebff37fe6d7c"}, +] "beautifulsoup4 4.12.0" = [ {url = "https://files.pythonhosted.org/packages/c5/4c/b5b7d6e1d4406973fb7f4e5df81c6f07890fa82548ac3b945deed1df9d48/beautifulsoup4-4.12.0.tar.gz", hash = "sha256:c5fceeaec29d09c84970e47c65f2f0efe57872f7cff494c9691a26ec0ff13234"}, {url = "https://files.pythonhosted.org/packages/ee/a7/06b189a2e280e351adcef25df532af3c59442123187e228b960ab3238687/beautifulsoup4-4.12.0-py3-none-any.whl", hash = "sha256:2130a5ad7f513200fae61a17abb5e338ca980fa28c439c0571014bc0217e9591"}, @@ -570,6 +628,10 @@ content_hash = "sha256:41531e3c394e45bd34b4d98fb20ef889335291b731a867850f266b070 {url = "https://files.pythonhosted.org/packages/58/07/815476ae605bcc5f95c87a62b95e74a1bce0878bc7a3119bc2bf4178f175/distlib-0.3.6.tar.gz", hash = "sha256:14bad2d9b04d3a36127ac97f30b12a19268f211063d8f8ee4f47108896e11b46"}, {url = "https://files.pythonhosted.org/packages/76/cb/6bbd2b10170ed991cf64e8c8b85e01f2fb38f95d1bc77617569e0b0b26ac/distlib-0.3.6-py2.py3-none-any.whl", hash = "sha256:f35c4b692542ca110de7ef0bea44d73981caeb34ca0b9b6b2e6d7790dda8f80e"}, ] +"dnspython 2.3.0" = [ + {url = "https://files.pythonhosted.org/packages/12/86/d305e87555430ff4630d729420d97dece3b16efcbf2b7d7e974d11b0d86c/dnspython-2.3.0-py3-none-any.whl", hash = "sha256:89141536394f909066cabd112e3e1a37e4e654db00a25308b0f130bc3152eb46"}, + {url = "https://files.pythonhosted.org/packages/91/8b/522301c50ca1f78b09c2ca116ffb0fd797eadf6a76085d376c01f9dd3429/dnspython-2.3.0.tar.gz", hash = "sha256:224e32b03eb46be70e12ef6d64e0be123a64e621ab4c0822ff6d450d52a540b9"}, +] "docker 6.0.1" = [ {url = "https://files.pythonhosted.org/packages/79/26/6609b51ecb418e12d1534d00b888ce7e108f38b47dc6cd589598d5c6aaa2/docker-6.0.1.tar.gz", hash = "sha256:896c4282e5c7af5c45e8b683b0b0c33932974fe6e50fc6906a0a83616ab3da97"}, {url = "https://files.pythonhosted.org/packages/d5/b3/a5e41798a6d4b92880998e0d9e6980e57c5d039f7f7144f87627a6b19084/docker-6.0.1-py3-none-any.whl", hash = "sha256:dbcb3bd2fa80dca0788ed908218bf43972772009b881ed1e20dfc29a65e49782"}, @@ -606,6 +668,10 @@ content_hash = "sha256:41531e3c394e45bd34b4d98fb20ef889335291b731a867850f266b070 {url = "https://files.pythonhosted.org/packages/7a/ff/75c28576a1d900e87eb6335b063fab47a8ef3c8b4d88524c4bf78f670cce/Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, {url = "https://files.pythonhosted.org/packages/bc/c3/f068337a370801f372f2f8f6bad74a5c140f6fda3d9de154052708dd3c65/Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, ] +"lazy-model 0.0.5" = [ + {url = "https://files.pythonhosted.org/packages/0c/dd/0ef5eaa54e502c3e3229420f0c619bdd5a556417bab1ed74c1d8b23dd3e6/lazy-model-0.0.5.tar.gz", hash = "sha256:2d98f9dfe275012477555a439dceb56364793a0f266758d1a33267d68e8fbc76"}, + {url = "https://files.pythonhosted.org/packages/16/19/7d72b219dd73dd3e5c2212fcc54d58b255bd62d9469da39c830f8a41cc70/lazy_model-0.0.5-py3-none-any.whl", hash = "sha256:8b4fc5eac99029f84b11b21e81a6894911a475f25e53227b7e44833e62e26553"}, +] "markupsafe 2.1.2" = [ {url = "https://files.pythonhosted.org/packages/02/2c/18d55e5df6a9ea33709d6c33e08cb2e07d39e20ad05d8c6fbf9c9bcafd54/MarkupSafe-2.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:835fb5e38fd89328e9c81067fd642b3593c33e1e17e2fdbf77f5676abb14a156"}, {url = "https://files.pythonhosted.org/packages/04/cf/9464c3c41b7cdb8df660cda75676697e7fb49ce1be7691a1162fc88da078/MarkupSafe-2.1.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:085fd3201e7b12809f9e6e9bc1e5c96a368c8523fad5afb02afe3c051ae4afcc"}, @@ -658,33 +724,37 @@ content_hash = "sha256:41531e3c394e45bd34b4d98fb20ef889335291b731a867850f266b070 {url = "https://files.pythonhosted.org/packages/ea/60/2400ba59cf2465fa136487ee7299f52121a9d04b2cf8539ad43ad10e70e8/MarkupSafe-2.1.2-cp311-cp311-win_amd64.whl", hash = "sha256:e55e40ff0cc8cc5c07996915ad367fa47da6b3fc091fdadca7f5403239c5fec3"}, {url = "https://files.pythonhosted.org/packages/f9/aa/ebcd114deab08f892b1d70badda4436dbad1747f9e5b72cffb3de4c7129d/MarkupSafe-2.1.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7313ce6a199651c4ed9d7e4cfb4aa56fe923b1adf9af3b420ee14e6d9a73df65"}, ] -"mypy 1.1.1" = [ - {url = "https://files.pythonhosted.org/packages/2a/28/8485aad67750b3374443d28bad3eed947737cf425a640ea4be4ac70a7827/mypy-1.1.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce61663faf7a8e5ec6f456857bfbcec2901fbdb3ad958b778403f63b9e606a1b"}, - {url = "https://files.pythonhosted.org/packages/30/da/808ceaf2bcf23a9e90156c7b11b41add8dd5a009ee48159ec820d04d97bd/mypy-1.1.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9401e33814cec6aec8c03a9548e9385e0e228fc1b8b0a37b9ea21038e64cdd8a"}, - {url = "https://files.pythonhosted.org/packages/44/9d/d23fa5d12bacbe7beea5fb6315b3325beabbe438e7e14d38c82b71609818/mypy-1.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:39c7119335be05630611ee798cc982623b9e8f0cff04a0b48dfc26100e0b97af"}, - {url = "https://files.pythonhosted.org/packages/47/9f/34f6a2254f7d39b8c4349b8ac480c233d37c377faf2c67c6ef925b3af0ab/mypy-1.1.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:59bbd71e5c58eed2e992ce6523180e03c221dcd92b52f0e792f291d67b15a71c"}, - {url = "https://files.pythonhosted.org/packages/61/99/4a844dcacbc4990a8312236bf74a55910ee9a05db69dee7d6fb7a7ffe6c2/mypy-1.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dbb19c9f662e41e474e0cff502b7064a7edc6764f5262b6cd91d698163196799"}, - {url = "https://files.pythonhosted.org/packages/62/54/be80f8d01f5cf72f774a77f9f750527a6fa733f09f78b1da30e8fa3914e6/mypy-1.1.1.tar.gz", hash = "sha256:ae9ceae0f5b9059f33dbc62dea087e942c0ccab4b7a003719cb70f9b8abfa32f"}, - {url = "https://files.pythonhosted.org/packages/64/63/6a04ca7a8b7f34811cada43ed6119736a7f4a07c5e1cbd8eec0e0f4962d5/mypy-1.1.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d809f88734f44a0d44959d795b1e6f64b2bbe0ea4d9cc4776aa588bb4229fc1c"}, - {url = "https://files.pythonhosted.org/packages/65/cc/ae5032abc06949e7a8c68f9885883fdb745c96bcf137cd4fa7225d50b647/mypy-1.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:2888ce4fe5aae5a673386fa232473014056967f3904f5abfcf6367b5af1f612a"}, - {url = "https://files.pythonhosted.org/packages/67/d3/1323311369eae97da4c7f47f266c55f7bdc22e74e4e2e1691be511ab8a91/mypy-1.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:69b35d1dcb5707382810765ed34da9db47e7f95b3528334a3c999b0c90fe523f"}, - {url = "https://files.pythonhosted.org/packages/7e/32/1b161731d19580c55d3d7c04b8ace80dc7cf42d852adf750f348a485068f/mypy-1.1.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b7c7b708fe9a871a96626d61912e3f4ddd365bf7f39128362bc50cbd74a634d5"}, - {url = "https://files.pythonhosted.org/packages/8a/fd/b610256224e01da4c4f315d11f62d39d815e97439a58d49d60aa4f55a60b/mypy-1.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:61bf08362e93b6b12fad3eab68c4ea903a077b87c90ac06c11e3d7a09b56b9c1"}, - {url = "https://files.pythonhosted.org/packages/8c/3d/a8d518bb06952484ada20897878a7a14741536f43514dcfecfac0676aa01/mypy-1.1.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c1c10fa12df1232c936830839e2e935d090fc9ee315744ac33b8a32216b93707"}, - {url = "https://files.pythonhosted.org/packages/91/63/55d0e62829f739f47978f1d8eb965ca8c40261841e47491ad297c84921c5/mypy-1.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:5cb14ff9919b7df3538590fc4d4c49a0f84392237cbf5f7a816b4161c061829e"}, - {url = "https://files.pythonhosted.org/packages/a4/0b/3a30f50287e42a4230320fa2eac25eb3017d38a7c31f083d407ab627607c/mypy-1.1.1-py3-none-any.whl", hash = "sha256:4e4e8b362cdf99ba00c2b218036002bdcdf1e0de085cdb296a49df03fb31dfc4"}, - {url = "https://files.pythonhosted.org/packages/b8/06/3d72d1b316ceec347874c4285fad8bf17e3fb21bb7848c1a942df239e44a/mypy-1.1.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d64c28e03ce40d5303450f547e07418c64c241669ab20610f273c9e6290b4b0b"}, - {url = "https://files.pythonhosted.org/packages/b8/72/385f3aeaaf262325454ac7f569eb81ac623464871df23d9778c864d04c6c/mypy-1.1.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:19ba15f9627a5723e522d007fe708007bae52b93faab00f95d72f03e1afa9598"}, - {url = "https://files.pythonhosted.org/packages/b9/e5/71eef5239219ee2f4d85e2ca6368d736705a3b874023b57f7237b977839c/mypy-1.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5b5f81b40d94c785f288948c16e1f2da37203c6006546c5d947aab6f90aefef2"}, - {url = "https://files.pythonhosted.org/packages/be/d5/5588a2ee0d77189626a57b555b6b006dda6d5b0083f16c6be0c2d761cd7b/mypy-1.1.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4b398d8b1f4fba0e3c6463e02f8ad3346f71956b92287af22c9b12c3ec965a9f"}, - {url = "https://files.pythonhosted.org/packages/bf/2d/45a526f248719ee32ecf1261564247a2e717a9c6167de5eb67d53599c4df/mypy-1.1.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21b437be1c02712a605591e1ed1d858aba681757a1e55fe678a15c2244cd68a5"}, - {url = "https://files.pythonhosted.org/packages/c0/d6/17ba6f8749722b8f61c6ab680769658f0bc63c293556149e2bf400b1f1a2/mypy-1.1.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:315ac73cc1cce4771c27d426b7ea558fb4e2836f89cb0296cbe056894e3a1f78"}, - {url = "https://files.pythonhosted.org/packages/d3/35/a0892864f1c128dc6449ee69897f9db7a64de2c16f41c14640dd22251b1b/mypy-1.1.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:0a28a76785bf57655a8ea5eb0540a15b0e781c807b5aa798bd463779988fa1d5"}, - {url = "https://files.pythonhosted.org/packages/d9/ab/d6d3884c3f432898458e2ade712988a7d1da562c1a363f2003b31677acd8/mypy-1.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:26cdd6a22b9b40b2fd71881a8a4f34b4d7914c679f154f43385ca878a8297389"}, - {url = "https://files.pythonhosted.org/packages/e1/a6/331cff5f7476904a2ebe6ed7cee2310b6be583ff6d45609ea0e0d67fd39d/mypy-1.1.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:64cc3afb3e9e71a79d06e3ed24bb508a6d66f782aff7e56f628bf35ba2e0ba51"}, - {url = "https://files.pythonhosted.org/packages/ed/89/85a04f32135fe4e35fd59d47100c939c7425fcb29868894c4b7a6171e065/mypy-1.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:a380c041db500e1410bb5b16b3c1c35e61e773a5c3517926b81dfdab7582be54"}, - {url = "https://files.pythonhosted.org/packages/f5/35/da01ef5831ceaf99a673e018d06ff1622ec460e4164b5e900ddaeceb52e1/mypy-1.1.1-cp37-cp37m-win_amd64.whl", hash = "sha256:ef6a01e563ec6a4940784c574d33f6ac1943864634517984471642908b30b6f7"}, - {url = "https://files.pythonhosted.org/packages/f6/57/93e676773f91141127329a56e2238eac506a78f6fb0ae0650a53fcc1355d/mypy-1.1.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2b0c373d071593deefbcdd87ec8db91ea13bd8f1328d44947e88beae21e8d5e9"}, +"motor 3.1.2" = [ + {url = "https://files.pythonhosted.org/packages/82/96/ae017cd62761d2fd2cc1eabfc902c3b4e3768fe994fc6a2f474694a56910/motor-3.1.2.tar.gz", hash = "sha256:80c08477c09e70db4f85c99d484f2bafa095772f1d29b3ccb253270f9041da9a"}, + {url = "https://files.pythonhosted.org/packages/f9/c3/22a695d0e6c373d0a33036de7fdc084068d896e948d11b691c88b6c1672f/motor-3.1.2-py3-none-any.whl", hash = "sha256:4bfc65230853ad61af447088527c1197f91c20ee957cfaea3144226907335716"}, +] +"mypy 1.3.0" = [ + {url = "https://files.pythonhosted.org/packages/09/7b/8eb0d648352c61b08cb364d278b5c12c3f1c5841724fdd2929d7172b7eaf/mypy-1.3.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:f9dca1e257d4cc129517779226753dbefb4f2266c4eaad610fc15c6a7e14283e"}, + {url = "https://files.pythonhosted.org/packages/11/41/d24f93eefc89c650782bf1f9acfdb02a32f327b841058a5b0ce5857b60af/mypy-1.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4c99c3ecf223cf2952638da9cd82793d8f3c0c5fa8b6ae2b2d9ed1e1ff51ba85"}, + {url = "https://files.pythonhosted.org/packages/25/c7/4735f81858a727e170279144600881fe3299aa7589ed585af6b788ea4556/mypy-1.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6e33bb8b2613614a33dff70565f4c803f889ebd2f859466e42b46e1df76018dd"}, + {url = "https://files.pythonhosted.org/packages/2b/27/4a26f91301804969194ee0dc9393843f10566d7fdf192ce11fc0218a989d/mypy-1.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c1eb485cea53f4f5284e5baf92902cd0088b24984f4209e25981cc359d64448d"}, + {url = "https://files.pythonhosted.org/packages/3c/5d/b87339c1fdfec7d13899cd7ad2ee992801695114c1cf9e1645da264cd437/mypy-1.3.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:74bc9b6e0e79808bf8678d7678b2ae3736ea72d56eede3820bd3849823e7f305"}, + {url = "https://files.pythonhosted.org/packages/47/f6/25c154bb1c479f2047093f0580c2c35ffc1ff007d52b7e50020cca60c010/mypy-1.3.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:76ec771e2342f1b558c36d49900dfe81d140361dd0d2df6cd71b3db1be155409"}, + {url = "https://files.pythonhosted.org/packages/4c/10/530d2df4d57f46f77b8211cf9bbe090baacff02e7076f21f1bf08148d541/mypy-1.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ddae0f39ca146972ff6bb4399f3b2943884a774b8771ea0a8f50e971f5ea5ba8"}, + {url = "https://files.pythonhosted.org/packages/55/e1/90487a3ea5a88b8f5c9d7fbf6f5fa7fcc8633d0132ce8364810a1da901c9/mypy-1.3.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:cbc07246253b9e3d7d74c9ff948cd0fd7a71afcc2b77c7f0a59c26e9395cb152"}, + {url = "https://files.pythonhosted.org/packages/5b/fb/0b1c90c635319b98dd65c6d6d6347413e42397e94057993011eeedeffbd9/mypy-1.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:8c5979d0deb27e0f4479bee18ea0f83732a893e81b78e62e2dda3e7e518c92ee"}, + {url = "https://files.pythonhosted.org/packages/6a/d0/4681d84878cecfd911752016ab30566366f6de7296fdc977b746eb68bf45/mypy-1.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7d23370d2a6b7a71dc65d1266f9a34e4cde9e8e21511322415db4b26f46f6b8c"}, + {url = "https://files.pythonhosted.org/packages/6a/d9/48de5203f4b6287a98fadcc47072b1bc69e3faaa39cba59a3a600b05a42c/mypy-1.3.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:6e42d29e324cdda61daaec2336c42512e59c7c375340bd202efa1fe0f7b8f8ca"}, + {url = "https://files.pythonhosted.org/packages/7e/75/021af7f0683ea19b9ad6a436e1b5c7cb39899c0f7b31040fa69b2395421e/mypy-1.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:a22435632710a4fcf8acf86cbd0d69f68ac389a3892cb23fbad176d1cddaf228"}, + {url = "https://files.pythonhosted.org/packages/86/56/08c5ff6b2139f301d9aa56cb8e7b2a24d4faa6fc3e94234dfe7eeecc9c44/mypy-1.3.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:faff86aa10c1aa4a10e1a301de160f3d8fc8703b88c7e98de46b531ff1276a9a"}, + {url = "https://files.pythonhosted.org/packages/88/0e/646696eb8fe7658b752009a495054a0214ae8e659e9cbcde8181f16ae999/mypy-1.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:658fe7b674769a0770d4b26cb4d6f005e88a442fe82446f020be8e5f5efb2fae"}, + {url = "https://files.pythonhosted.org/packages/8d/c8/681f4a19c62aa71bdc9ad3a4bc9a0fb8846bd0b5a8bc1b29d261c8025f80/mypy-1.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:550a8b3a19bb6589679a7c3c31f64312e7ff482a816c96e0cecec9ad3a7564dd"}, + {url = "https://files.pythonhosted.org/packages/90/b6/a2d2ba604982af6034e3fcad17a464a66127be47f07b4587beec76e8f80b/mypy-1.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:44797d031a41516fcf5cbfa652265bb994e53e51994c1bd649ffcd0c3a7eccbf"}, + {url = "https://files.pythonhosted.org/packages/b1/ce/8d87f684bb7e2a520cfa9cd17b8dc686a83143bb12a3e1ac4ad6d8d4825c/mypy-1.3.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c5d2cc54175bab47011b09688b418db71403aefad07cbcd62d44010543fc143f"}, + {url = "https://files.pythonhosted.org/packages/b1/e1/399e3dfeb2842e4a2634866e4ef8b69151d465b7a5ceb648d7f1296f17d0/mypy-1.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:95d8d31a7713510685b05fbb18d6ac287a56c8f6554d88c19e73f724a445448a"}, + {url = "https://files.pythonhosted.org/packages/b8/36/6628916f94bb0816e1719117e1962750413ab408f83673ce7d571caf3960/mypy-1.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1c4c42c60a8103ead4c1c060ac3cdd3ff01e18fddce6f1016e08939647a0e703"}, + {url = "https://files.pythonhosted.org/packages/ba/ac/1c280246fc0c5239409f31e1a321f178ba11a9c6e5eaaf6d56f9ff627cdf/mypy-1.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e86c2c6852f62f8f2b24cb7a613ebe8e0c7dc1402c61d36a609174f63e0ff017"}, + {url = "https://files.pythonhosted.org/packages/c9/c5/f3e4ed59e08e3a728a15da198317edfcd13b7dc2215d52b5d85fce716285/mypy-1.3.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:87df44954c31d86df96c8bd6e80dfcd773473e877ac6176a8e29898bfb3501cb"}, + {url = "https://files.pythonhosted.org/packages/cd/b9/6abe1cd8ac8e70f12f43eebe6427814f9d36142d331eae5cc5bba77585a2/mypy-1.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:d0b6c62206e04061e27009481cb0ec966f7d6172b5b936f3ead3d74f29fe3dcf"}, + {url = "https://files.pythonhosted.org/packages/d8/c6/de2e214a42b63d7ea0abef9f02a6da69cad6d532165bb7a8cc8291099a0c/mypy-1.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:473117e310febe632ddf10e745a355714e771ffe534f06db40702775056614c4"}, + {url = "https://files.pythonhosted.org/packages/d9/79/82d452b409d7610944ba3a1a6079987d3ed6062cb8fe5c8850f26dafb6e0/mypy-1.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ebc95f8386314272bbc817026f8ce8f4f0d2ef7ae44f947c4664efac9adec929"}, + {url = "https://files.pythonhosted.org/packages/e3/f7/1fed3b24abb75f244fa6bc60ea03cd9d3d8ad225a4cfda7533042fe6d831/mypy-1.3.0-py3-none-any.whl", hash = "sha256:a8763e72d5d9574d45ce5881962bc8e9046bf7b375b0abf031f3e6811732a897"}, + {url = "https://files.pythonhosted.org/packages/f9/88/3bfe07521fb9e74b449cbc4367434067ec70bfd8a24c652fa3e0f9597389/mypy-1.3.0.tar.gz", hash = "sha256:e1f4d16e296f5135624b34e8fb741eb0eadedca90862405b1f1fde2040b9bd11"}, ] "mypy-extensions 1.0.0" = [ {url = "https://files.pythonhosted.org/packages/2a/e2/5d3f6ada4297caebe1a2add3b126fe800c96f56dbe5d1988a2cbe0b267aa/mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, @@ -748,6 +818,82 @@ content_hash = "sha256:41531e3c394e45bd34b4d98fb20ef889335291b731a867850f266b070 {url = "https://files.pythonhosted.org/packages/f5/09/3f2ad426d20d2d353432f1c76290fa3c9863e2c04e05382ccca2aeade4c3/pydantic-1.10.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:bbd5c531b22928e63d0cb1868dee76123456e1de2f1cb45879e9e7a3f3f1779b"}, {url = "https://files.pythonhosted.org/packages/f5/56/64028e205064748d6015a1afd6111c06f2b90982636850a3e157a7180ed5/pydantic-1.10.6-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:60184e80aac3b56933c71c48d6181e630b0fbc61ae455a63322a66a23c14731a"}, ] +"pymongo 4.3.3" = [ + {url = "https://files.pythonhosted.org/packages/01/10/e7157fcda1db4f759c858f8d9dc001112eb630136894056bb29f332137c3/pymongo-4.3.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:bb869707d8e30645ed6766e44098600ca6cdf7989c22a3ea2b7966bb1d98d4b2"}, + {url = "https://files.pythonhosted.org/packages/05/17/185c96a98d3d91ad3cdfbc9bc91ad8bea697cfaf1b3ca314f52006f71d2b/pymongo-4.3.3-cp310-cp310-win32.whl", hash = "sha256:dc0cff74cd36d7e1edba91baa09622c35a8a57025f2f2b7a41e3f83b1db73186"}, + {url = "https://files.pythonhosted.org/packages/0e/8f/1009913e8ad51390966811e0163ed6df2dfa43a6f632ac35f53e51b2321b/pymongo-4.3.3-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:e2961b05f9c04a53da8bfc72f1910b6aec7205fcf3ac9c036d24619979bbee4b"}, + {url = "https://files.pythonhosted.org/packages/0e/9f/a4986f0a86fc017599bf4c8912c01005a27c536acd221041234e0cb9739a/pymongo-4.3.3-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:4ed00f96e147f40b565fe7530d1da0b0f3ab803d5dd5b683834500fa5d195ec4"}, + {url = "https://files.pythonhosted.org/packages/10/58/cdf21baff3328e6ba3b960918cd48302c3973e97ea4dcfbdf6ae5bf18408/pymongo-4.3.3-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:6fcfbf435eebf8a1765c6d1f46821740ebe9f54f815a05c8fc30d789ef43cb12"}, + {url = "https://files.pythonhosted.org/packages/11/a3/8f7b87dbb9fd496f14c596bb02487fdb44dbb58e3c39da3f0eb0199b1523/pymongo-4.3.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cdb87309de97c63cb9a69132e1cb16be470e58cffdfbad68fdd1dc292b22a840"}, + {url = "https://files.pythonhosted.org/packages/22/18/68b8a63f289df40df27623c99779acd9eb6c007a4546700e676e07d7c2d6/pymongo-4.3.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:08fc250b5552ee97ceeae0f52d8b04f360291285fc7437f13daa516ce38fdbc6"}, + {url = "https://files.pythonhosted.org/packages/26/38/33270a35e265c1936ab3ea6863c02b9e3292ca013df9bd1e5ab1ed231ec7/pymongo-4.3.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5effd87c7d363890259eac16c56a4e8da307286012c076223997f8cc4a8c435b"}, + {url = "https://files.pythonhosted.org/packages/29/3f/230c83a6be6e037f4558c9b7a2b8dc6de55ebc68662b0a13f9ff800614f7/pymongo-4.3.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eac0a143ef4f28f49670bf89cb15847eb80b375d55eba401ca2f777cd425f338"}, + {url = "https://files.pythonhosted.org/packages/2c/5c/ab73b2fc15fd9930f07bce865c3f0d98fe90211b92889831a746d61d3830/pymongo-4.3.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:12f3621a46cdc7a9ba8080422262398a91762a581d27e0647746588d3f995c88"}, + {url = "https://files.pythonhosted.org/packages/2c/c7/302a0fa990e5c2e1b137b94a5dfc174437a77872990c6c05b21779fe9502/pymongo-4.3.3-cp39-cp39-manylinux2014_ppc64le.whl", hash = "sha256:c1a70c51da9fa95bd75c167edb2eb3f3c4d27bc4ddd29e588f21649d014ec0b7"}, + {url = "https://files.pythonhosted.org/packages/2e/d8/d35fcd7fd6d9b55ab6b317182884938d34c64c91dce9ff5cf3548ca5cd30/pymongo-4.3.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:81d1a7303bd02ca1c5be4aacd4db73593f573ba8e0c543c04c6da6275fd7a47e"}, + {url = "https://files.pythonhosted.org/packages/2f/f0/33804cfc9113e0405063f0a777d213d9c006512cb06681a258ae559b3a8c/pymongo-4.3.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f3055510fdfdb1775bc8baa359783022f70bb553f2d46e153c094dfcb08578ff"}, + {url = "https://files.pythonhosted.org/packages/38/68/928d7ce22719cfa255fb973b34aed6f04ac3ea89049ce69e3b092c30a60f/pymongo-4.3.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1074f1a6f23e28b983c96142f2d45be03ec55d93035b471c26889a7ad2365db3"}, + {url = "https://files.pythonhosted.org/packages/39/22/e5acdce322f6aed2c6b06b8afae19c0fdf01031db1f7dbaeb34df60396c1/pymongo-4.3.3-cp39-cp39-manylinux1_i686.whl", hash = "sha256:016c412118e1c23fef3a1eada4f83ae6e8844fd91986b2e066fc1b0013cdd9ae"}, + {url = "https://files.pythonhosted.org/packages/39/97/3a04c850755723d64555ae29fdec2d4eafe9f2a12c22d4dc5e41e846423d/pymongo-4.3.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704d939656e21b073bfcddd7228b29e0e8a93dd27b54240eaafc0b9a631629a6"}, + {url = "https://files.pythonhosted.org/packages/3c/30/3d7e6336cfc795655a7193d77853972c5b502f58e1992205ad1b9bd28128/pymongo-4.3.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:55b6163dac53ef1e5d834297810c178050bd0548a4136cd4e0f56402185916ca"}, + {url = "https://files.pythonhosted.org/packages/40/e3/dda96a2280058e08bc0dabeddf86bd3513e601f579134f2107680585636b/pymongo-4.3.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e7fac06a539daef4fcf5d8288d0d21b412f9b750454cd5a3cf90484665db442a"}, + {url = "https://files.pythonhosted.org/packages/42/0c/d2ad12aec55acdc4099134a8c87912d8fe01e2e1e5969b5d6c3485b99284/pymongo-4.3.3-cp37-cp37m-win_amd64.whl", hash = "sha256:54c377893f2cbbffe39abcff5ff2e917b082c364521fa079305f6f064e1a24a9"}, + {url = "https://files.pythonhosted.org/packages/45/2f/70f2e110a77dcb5490fe000aa380397968a09b8528f878aa1eadc0b11920/pymongo-4.3.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:74731c9e423c93cbe791f60c27030b6af6a948cef67deca079da6cd1bb583a8e"}, + {url = "https://files.pythonhosted.org/packages/48/b3/048d832794acb914cf8cf396089a29301ee79417e18f068f38a1eace9408/pymongo-4.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:cafa52873ae12baa512a8721afc20de67a36886baae6a5f394ddef0ce9391f91"}, + {url = "https://files.pythonhosted.org/packages/49/de/9005f70242f651fe4758a162eedbda13c9e55713083c345574c17cf8aa8f/pymongo-4.3.3-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:9b87b23570565a6ddaa9244d87811c2ee9cffb02a753c8a2da9c077283d85845"}, + {url = "https://files.pythonhosted.org/packages/4a/92/9c11924649a557d95283882a4bcb67cfc32d6cb1528064a53c0bdb0540d7/pymongo-4.3.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:a9c2885b4a8e6e39db5662d8b02ca6dcec796a45e48c2de12552841f061692ba"}, + {url = "https://files.pythonhosted.org/packages/4b/ce/c6c6875dc14410952d3ff2e7960fb0498b1d9e70c483e5ce788c01fad54e/pymongo-4.3.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c0640b4e9d008e13956b004d1971a23377b3d45491f87082161c92efb1e6c0d6"}, + {url = "https://files.pythonhosted.org/packages/4e/ca/6c1cb5c69715c13312852d91cb62c175e7da58c91c428447db1a2364c646/pymongo-4.3.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d3a51901066696c4af38c6c63a1f0aeffd5e282367ff475de8c191ec9609b56d"}, + {url = "https://files.pythonhosted.org/packages/4f/2c/2da01e59e47cec96df562f0fe8ed6e1dd8b01a0ff8acd6d8ea1b59aaf82a/pymongo-4.3.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d86c35d94b5499689354ccbc48438a79f449481ee6300f3e905748edceed78e7"}, + {url = "https://files.pythonhosted.org/packages/4f/a9/32799279229f74f4d477f6c122dbbb4173f7d6d158bb9f7adf582c2ada20/pymongo-4.3.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6dd1cf2995fdbd64fc0802313e8323f5fa18994d51af059b5b8862b73b5e53f0"}, + {url = "https://files.pythonhosted.org/packages/60/2f/6b18e099cfabf8fbe86ec201f53afa73a8b80e2e9dcbdef52429492d236e/pymongo-4.3.3-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:8fd6e191b92a10310f5a6cfe10d6f839d79d192fb02480bda325286bd1c7b385"}, + {url = "https://files.pythonhosted.org/packages/63/0e/ac6759051f18adf5506fe0c458bc12d03d9e94d2dc83087b21dc21888154/pymongo-4.3.3-cp38-cp38-win_amd64.whl", hash = "sha256:d5571b6978750601f783cea07fb6b666837010ca57e5cefa389c1d456f6222e2"}, + {url = "https://files.pythonhosted.org/packages/63/74/51b2ec1b760169cbb19637913b86b6851dd9a57f95fe67adb7b0d1037469/pymongo-4.3.3-cp310-cp310-manylinux2014_ppc64le.whl", hash = "sha256:ffcc8394123ea8d43fff8e5d000095fe7741ce3f8988366c5c919c4f5eb179d3"}, + {url = "https://files.pythonhosted.org/packages/63/c8/a6e9f789cfbafc8293b5d94b0fa66b7a8854c6e74a04a74bc7585381ddd8/pymongo-4.3.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:be1d2ce7e269215c3ee9a215e296b7a744aff4f39233486d2c4d77f5f0c561a6"}, + {url = "https://files.pythonhosted.org/packages/66/b6/8e554ee180a28aa3f99200eb1ab60ab180fbea1a55f47166a6da2fd93299/pymongo-4.3.3-cp38-cp38-manylinux2014_i686.whl", hash = "sha256:d07d06dba5b5f7d80f9cc45501456e440f759fe79f9895922ed486237ac378a8"}, + {url = "https://files.pythonhosted.org/packages/69/d4/9cd99a5d98353b6c10595ec969c087d63a93ce60741b52463a9fcb2114ad/pymongo-4.3.3-cp39-cp39-win32.whl", hash = "sha256:dc24d245026a72d9b4953729d31813edd4bd4e5c13622d96e27c284942d33f24"}, + {url = "https://files.pythonhosted.org/packages/6a/6c/246b69b8fc3071e9ff1f42480fbc29835b95e910655604b66bef0a282e78/pymongo-4.3.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c184ec5be465c0319440734491e1aa4709b5f3ba75fdfc9dbbc2ae715a7f6829"}, + {url = "https://files.pythonhosted.org/packages/71/c7/c129dcde11ec97fe485cfc7a837284a0300bc4647a1bcb1e63f1ce050732/pymongo-4.3.3-cp37-cp37m-manylinux2014_s390x.whl", hash = "sha256:4d00b91c77ceb064c9b0459f0d6ea5bfdbc53ea9e17cf75731e151ef25a830c7"}, + {url = "https://files.pythonhosted.org/packages/74/7a/140e4c739319c3ee1163aa65bc91414ddf5b3c6376af19375e2dead1fbb5/pymongo-4.3.3-cp310-cp310-manylinux2014_s390x.whl", hash = "sha256:943f208840777f34312c103a2d1caab02d780c4e9be26b3714acf6c4715ba7e1"}, + {url = "https://files.pythonhosted.org/packages/74/a8/fe9d9c1f7d3a12b3d5c2b26fb267671a02f42b68ddd69d20105b6d87798b/pymongo-4.3.3-cp37-cp37m-manylinux2014_i686.whl", hash = "sha256:c09956606c08c4a7c6178a04ba2dd9388fcc5db32002ade9c9bc865ab156ab6d"}, + {url = "https://files.pythonhosted.org/packages/76/05/de90f39846ec83fe9e2099c7993266bb1a154f3a0777e78121f56fe08ee7/pymongo-4.3.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ef888f48eb9203ee1e04b9fb27429017b290fb916f1e7826c2f7808c88798394"}, + {url = "https://files.pythonhosted.org/packages/7d/33/aa74d9e5067bdd7b68cbe54ea5cad427883131d100c20d6a31ff0625a214/pymongo-4.3.3-cp39-cp39-manylinux2014_s390x.whl", hash = "sha256:8a06a0c02f5606330e8f2e2f3b7949877ca7e4024fa2bff5a4506bec66c49ec7"}, + {url = "https://files.pythonhosted.org/packages/81/37/c5c765526adb3f452ea4033d5d4e960514d53857b32c85fc2dfcac7aad86/pymongo-4.3.3-cp39-cp39-manylinux2014_i686.whl", hash = "sha256:b38a96b3eed8edc515b38257f03216f382c4389d022a8834667e2bc63c0c0c31"}, + {url = "https://files.pythonhosted.org/packages/81/5d/6d34f7b3cffe3efe38cac65de60beba7f1a14b8f5b64d27354bce33b924d/pymongo-4.3.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:47f7aa217b25833cd6f0e72b0d224be55393c2692b4f5e0561cb3beeb10296e9"}, + {url = "https://files.pythonhosted.org/packages/81/f1/5d56b0ffdda842298334135ac181032ee4624bc57101a538d67ba8958695/pymongo-4.3.3-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:6c2216d8b6a6d019c6f4b1ad55f890e5e77eb089309ffc05b6911c09349e7474"}, + {url = "https://files.pythonhosted.org/packages/89/24/52d65bbb0cf038d73b49c9d1f6b251500d807ed2579aaa55cf5d788513be/pymongo-4.3.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:fd7bb378d82b88387dc10227cfd964f6273eb083e05299e9b97cbe075da12d11"}, + {url = "https://files.pythonhosted.org/packages/8b/8f/93649909ec1ba88fee224884884b4e10ac26c0ca00c58f1781036476d30d/pymongo-4.3.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:599d3f6fbef31933b96e2d906b0f169b3371ff79ea6aaf6ecd76c947a3508a3d"}, + {url = "https://files.pythonhosted.org/packages/92/45/47134bdc3d628fa02945545c9d0cca1d7b349c507734860cf3614da77cb0/pymongo-4.3.3-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:0c466710871d0026c190fc4141e810cf9d9affbf4935e1d273fbdc7d7cda6143"}, + {url = "https://files.pythonhosted.org/packages/93/da/d58cdba6e4c896300d1c939119c0911948a7edd94e10cc75048142e56160/pymongo-4.3.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c2fdc855149efe7cdcc2a01ca02bfa24761c640203ea94df467f3baf19078be"}, + {url = "https://files.pythonhosted.org/packages/96/48/8baccdb480d0ceb2799d1b6d2da780b6f174635c64f82fa27bc8fbb9d660/pymongo-4.3.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7c051fe37c96b9878f37fa58906cb53ecd13dcb7341d3a85f1e2e2f6b10782d9"}, + {url = "https://files.pythonhosted.org/packages/97/9f/0156a752e50cfbc767a182c80a7e94174a772a94cb72a52f2660fc373c77/pymongo-4.3.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7b16250238de8dafca225647608dddc7bbb5dce3dd53b4d8e63c1cc287394c2f"}, + {url = "https://files.pythonhosted.org/packages/98/4d/4423858f2587a3c15c9b40a70e3672e0902667874f24e77e5388d848715d/pymongo-4.3.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7761cacb8745093062695b11574effea69db636c2fd0a9269a1f0183712927b4"}, + {url = "https://files.pythonhosted.org/packages/9a/31/482f7401e7bbbeb66ab6b4ac263e2b50435f4329cce1e72378972d48f6b5/pymongo-4.3.3.tar.gz", hash = "sha256:34e95ffb0a68bffbc3b437f2d1f25fc916fef3df5cdeed0992da5f42fae9b807"}, + {url = "https://files.pythonhosted.org/packages/a0/53/f8b2099b2d8dcec0e4070455b6b7a9ea5088ee07a745b0c6a711d55a5357/pymongo-4.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:fc28e8d85d392a06434e9a934908d97e2cf453d69488d2bcd0bfb881497fd975"}, + {url = "https://files.pythonhosted.org/packages/a3/c6/ff88fce93529c9418c80854ecaf013254ab0b1d59f8f4fa2702419352d18/pymongo-4.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:b8a03af1ce79b902a43f5f694c4ca8d92c2a4195db0966f08f266549e2fc49bc"}, + {url = "https://files.pythonhosted.org/packages/a9/8c/5ae0d794ff1771dd2a298f1a7d0889455a65874a98120171a002c8cb741a/pymongo-4.3.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:52896e22115c97f1c829db32aa2760b0d61839cfe08b168c2b1d82f31dbc5f55"}, + {url = "https://files.pythonhosted.org/packages/b5/a2/a566780a2baeb108ae4b7e87add2c022090a39f728e9c808dee4c8b1efde/pymongo-4.3.3-cp37-cp37m-manylinux2014_ppc64le.whl", hash = "sha256:b0cfe925610f2fd59555bb7fc37bd739e4b197d33f2a8b2fae7b9c0c6640318c"}, + {url = "https://files.pythonhosted.org/packages/c4/3d/51e3ed544c4d4a0dbcafe197d582f9e922e73ea185bd5a19486c7c297308/pymongo-4.3.3-cp38-cp38-manylinux2014_s390x.whl", hash = "sha256:34b040e095e1671df0c095ec0b04fc4ebb19c4c160f87c2b55c079b16b1a6b00"}, + {url = "https://files.pythonhosted.org/packages/c4/b5/e2d246016d15c949736c9a4b4da4ec8e2045b661504be4b749c34188c2a5/pymongo-4.3.3-cp311-cp311-win32.whl", hash = "sha256:524d78673518dcd352a91541ecd2839c65af92dc883321c2109ef6e5cd22ef23"}, + {url = "https://files.pythonhosted.org/packages/c5/be/64441bc6f65ddca82ecb1231348c89257272c023a44658d59f044877a498/pymongo-4.3.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c29e758f0e734e1e90357ae01ec9c6daf19ff60a051192fe110d8fb25c62600e"}, + {url = "https://files.pythonhosted.org/packages/c6/1f/cd1d6d21620125693cd6d21eb9264b885df553f3c51cb778b06fe96d6abd/pymongo-4.3.3-cp38-cp38-manylinux2014_ppc64le.whl", hash = "sha256:711bc52cb98e7892c03e9b669bebd89c0a890a90dbc6d5bb2c47f30239bac6e9"}, + {url = "https://files.pythonhosted.org/packages/c9/02/77f30505aa009f329ec935a2e0e856889e19568c1c6c7af4dbffe894c27e/pymongo-4.3.3-cp38-cp38-manylinux1_i686.whl", hash = "sha256:dca34367a4e77fcab0693e603a959878eaf2351585e7d752cac544bc6b2dee46"}, + {url = "https://files.pythonhosted.org/packages/d4/4d/cdfde31b4545d2f0aaabae9a9acd0dda6384f3d02b4f7a4b6a483f4bf749/pymongo-4.3.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5134d33286c045393c7beb51be29754647cec5ebc051cf82799c5ce9820a2ca2"}, + {url = "https://files.pythonhosted.org/packages/d6/58/a39537805ca205b4b65503765ca110224a409e777c1825fd6c8108ec9fd0/pymongo-4.3.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:341221e2f2866a5960e6f8610f4cbac0bb13097f3b1a289aa55aba984fc0d969"}, + {url = "https://files.pythonhosted.org/packages/dc/bc/1d69ee98cc0b50278f9c6044666a5dbd8b296e8bd3af733066f6bb8bc597/pymongo-4.3.3-cp37-cp37m-win32.whl", hash = "sha256:49210feb0be8051a64d71691f0acbfbedc33e149f0a5d6e271fddf6a12493fed"}, + {url = "https://files.pythonhosted.org/packages/df/2c/572e43db59a870b8df3332b94bd29ee7246bcba8cbb071b61174ecd1c834/pymongo-4.3.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:316498b642c00401370b2156b5233b256f9b33799e0a8d9d0b8a7da217a20fca"}, + {url = "https://files.pythonhosted.org/packages/e1/ea/ca13d38405cea315683b085cfcf661cf48be9a9a786dcead86d9454fcc18/pymongo-4.3.3-cp37-cp37m-macosx_10_6_intel.whl", hash = "sha256:39b03045c71f761aee96a12ebfbc2f4be89e724ff6f5e31c2574c1a0e2add8bd"}, + {url = "https://files.pythonhosted.org/packages/e2/7c/a076b118f1b7aea6c8dc548d45441801b86486bf67765589112e28ea188d/pymongo-4.3.3-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:c6258a3663780ae47ba73d43eb63c79c40ffddfb764e09b56df33be2f9479837"}, + {url = "https://files.pythonhosted.org/packages/e7/e1/e2c577333ee346b411db65d9f62c746eca8b1062c55afdb5d2fb8ebc23fe/pymongo-4.3.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fa7e202feb683dad74f00dea066690448d0cfa310f8a277db06ec8eb466601b5"}, + {url = "https://files.pythonhosted.org/packages/ee/2a/223a77aab2d1d9f2ca86b1db60578f25ebd2f1c0f558fcf46d05457865d1/pymongo-4.3.3-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:cd6a4afb20fb3c26a7bfd4611a0bbb24d93cbd746f5eb881f114b5e38fd55501"}, + {url = "https://files.pythonhosted.org/packages/f0/25/5331b822a0e2486efe75c741fa9dcb500b67ecfb0223f26179afa60f1c17/pymongo-4.3.3-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:3b93043b14ba7eb08c57afca19751658ece1cfa2f0b7b1fb5c7a41452fbb8482"}, + {url = "https://files.pythonhosted.org/packages/f2/4a/68ab4706a992fd7b01ec53a9e2138733972895b260578e544221845770dd/pymongo-4.3.3-cp310-cp310-manylinux1_i686.whl", hash = "sha256:66413c50d510e5bcb0afc79880d1693a2185bcea003600ed898ada31338c004e"}, + {url = "https://files.pythonhosted.org/packages/f3/87/f2ccd99ea5184d9a9013acca92f3060e29253038df8003148b1a643e6165/pymongo-4.3.3-cp310-cp310-manylinux2014_i686.whl", hash = "sha256:695939036a320f4329ccf1627edefbbb67cc7892b8222d297b0dd2313742bfee"}, + {url = "https://files.pythonhosted.org/packages/f4/d6/3088b63536c74c4e9cf687916712843e7d4abfc981eca3e264ec801372af/pymongo-4.3.3-cp38-cp38-win32.whl", hash = "sha256:a6cd6f1db75eb07332bd3710f58f5fce4967eadbf751bad653842750a61bda62"}, + {url = "https://files.pythonhosted.org/packages/f8/ef/bd801e889305bc48ca3210569ea613d66a52c717578a465ac2792cec709a/pymongo-4.3.3-cp310-cp310-manylinux2014_x86_64.whl", hash = "sha256:01f7cbe88d22440b6594c955e37312d932fd632ffed1a86d0c361503ca82cc9d"}, + {url = "https://files.pythonhosted.org/packages/fa/6a/bf5391534a10cfb4a2b4a9e6697f17115fc460da8041ec67835c23d2ff59/pymongo-4.3.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a966d5304b7d90c45c404914e06bbf02c5bf7e99685c6c12f0047ef2aa837142"}, + {url = "https://files.pythonhosted.org/packages/fc/28/1b934e5839bf12b022782561c803ee63149737d6c5f9627d3299cd28516f/pymongo-4.3.3-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:7d43ac9c7eeda5100fb0a7152fab7099c9cf9e5abd3bb36928eb98c7d7a339c6"}, +] "pytest 7.2.2" = [ {url = "https://files.pythonhosted.org/packages/b2/68/5321b5793bd506961bd40bdbdd0674e7de4fb873ee7cab33dd27283ad513/pytest-7.2.2-py3-none-any.whl", hash = "sha256:130328f552dcfac0b1cec75c12e3f005619dc5f874f0a06e8ff7263f0ee6225e"}, {url = "https://files.pythonhosted.org/packages/b9/29/311895d9cd3f003dd58e8fdea36dd895ba2da5c0c90601836f7de79f76fe/pytest-7.2.2.tar.gz", hash = "sha256:c99ab0c73aceb050f68929bc93af19ab6db0558791c6a0715723abe9d0ade9d4"}, @@ -799,6 +945,10 @@ content_hash = "sha256:41531e3c394e45bd34b4d98fb20ef889335291b731a867850f266b070 "testcontainers 3.7.1" = [ {url = "https://files.pythonhosted.org/packages/b3/37/38c595414d764cb1d9f3a0c907878c4146a21505ab974c63bcf3d8145807/testcontainers-3.7.1-py2.py3-none-any.whl", hash = "sha256:7f48cef4bf0ccd78f1a4534d4b701a003a3bace851f24eae58a32f9e3f0aeba0"}, ] +"toml 0.10.2" = [ + {url = "https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {url = "https://files.pythonhosted.org/packages/be/ba/1f744cdc819428fc6b5084ec34d9b30660f6f9daaf70eead706e3203ec3c/toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] "types-beautifulsoup4 4.12.0.0" = [ {url = "https://files.pythonhosted.org/packages/92/0b/39afb220c7d8328c5c887007e17c950eda2c2e9300132b69e923e81ff033/types_beautifulsoup4-4.12.0.0-py3-none-any.whl", hash = "sha256:43c23852a6ef0053632b9a308fc3488831c0f3e02c0f4b4478a28703217cf683"}, {url = "https://files.pythonhosted.org/packages/a4/23/9a9131dedfbd64354fabedef74c8b69092afa4c65720b8fb35df18ded18b/types-beautifulsoup4-4.12.0.0.tar.gz", hash = "sha256:3859e70d3118d65d12ebfca109304de4bf52383e6f99f941c114fd1153bb6cc1"}, diff --git a/pyproject.toml b/pyproject.toml index c245e85..9a3dbff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,13 +17,14 @@ dependencies = [ "websockets>=10.4", "uvicorn>=0.19.0", "httpx>=0.23.3", + "beanie>=1.18.0", ] requires-python = ">=3.11" license = { text = "MIT" } [project.optional-dependencies] dev = [ "beautifulsoup4>=4.11.1", - "mypy>=1.1.1", + "mypy>=1.3.0", "pytest>=7.2.2", "pytest-asyncio>=0.21.0", "testcontainers>=3.7.1", @@ -31,17 +32,17 @@ dev = [ "types-requests>=2.28.11.15", "types-beautifulsoup4>=4.12.0.0", ] -nox = [ - "nox>=2022.11.21", -] +nox = ["nox>=2022.11.21"] [tool.mypy] plugins = ["pydantic.mypy"] +[tool.ruff] +line-length = 100 + [tool.pytest.ini_options] markers = [ - "integration: mark test as integration test", - "needs_docker: marks tests that need access to Docker in order to run" + "integration: mark test as integration test" ] [tool.pdm.scripts] @@ -49,7 +50,7 @@ test = "pytest tests -m 'not integration'" test-integration = "pytest tests" [[tool.mypy.overrides]] -module = "testcontainers.*" +module = ["testcontainers.*", "motor.motor_asyncio"] ignore_missing_imports = true [build-system] diff --git a/tests/decorators.py b/tests/decorators.py new file mode 100644 index 0000000..ecdc42d --- /dev/null +++ b/tests/decorators.py @@ -0,0 +1,14 @@ +from typing import Any, Callable + + +Decorator = Callable[[Callable[..., Any]], Callable[..., Any]] + + +def compose(*decorators: Decorator) -> Decorator: + def decorated(fn: Callable[..., Any]) -> Callable[..., Any]: + for deco in reversed(decorators): + fn = deco(fn) + + return fn + + return decorated diff --git a/tests/markers.py b/tests/markers.py new file mode 100644 index 0000000..39c7cfa --- /dev/null +++ b/tests/markers.py @@ -0,0 +1,24 @@ +import shutil + +import pytest + + +def browse_ocrd_not_available() -> bool: + browse_ocrd = shutil.which("browse-ocrd") + broadway = shutil.which("broadwayd") + return not all((browse_ocrd, broadway)) + + +def docker_not_available() -> bool: + return not bool(shutil.which("docker")) + + +skip_if_no_docker = pytest.mark.skipif( + docker_not_available(), + reason="Skipping because Docker is not available", +) + +skip_if_no_browse_ocrd = pytest.mark.skipif( + browse_ocrd_not_available(), + reason="Skipping because browse-ocrd or broadwayd are not available", +) diff --git a/tests/ocrdbrowser/test_browser_launch.py b/tests/ocrdbrowser/test_browser_launch.py new file mode 100644 index 0000000..95cfa1e --- /dev/null +++ b/tests/ocrdbrowser/test_browser_launch.py @@ -0,0 +1,109 @@ +import asyncio +import functools +from typing import AsyncIterator, Callable, NamedTuple + +import pytest +import pytest_asyncio + +from ocrdbrowser import ( + DockerOcrdBrowserFactory, + NoPortsAvailableError, + OcrdBrowserFactory, + SubProcessOcrdBrowserFactory, +) +from tests import markers +from tests.decorators import compose + + +create_docker_browser_factory = functools.partial( + DockerOcrdBrowserFactory, "http://localhost" +) + +browser_factory_test = compose( + pytest.mark.asyncio, + pytest.mark.integration, + pytest.mark.parametrize( + "create_browser_factory", + ( + pytest.param( + create_docker_browser_factory, + marks=markers.skip_if_no_docker, + ), + pytest.param( + SubProcessOcrdBrowserFactory, marks=markers.skip_if_no_browse_ocrd + ), + ), + ), +) + + +class DockerProcessKiller(NamedTuple): + kill: str = "docker stop" + ps: str = "docker ps" + + +class NativeProcessKiller(NamedTuple): + kill: str = "kill" + ps: str = "ps" + + +async def kill_processes( + killer: NativeProcessKiller | DockerProcessKiller, name_filter: str +) -> None: + kill_cmd, ps_cmd = killer + cmd = await asyncio.create_subprocess_shell( + f"{kill_cmd} $({ps_cmd} | grep {name_filter} | awk '{{ print $1 }}')", + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + await cmd.wait() + + +@pytest_asyncio.fixture(autouse=True) +async def stop_browsers() -> AsyncIterator[None]: + yield + + async with asyncio.TaskGroup() as group: + group.create_task(kill_processes(DockerProcessKiller(), "ocrd-browser")) + group.create_task(kill_processes(NativeProcessKiller(), "broadwayd")) + group.create_task(kill_processes(NativeProcessKiller(), "browse-ocrd")) + + +CreateBrowserFactory = Callable[[set[int]], OcrdBrowserFactory] + + +@browser_factory_test +async def test__factory__launches_new_browser_instance( + create_browser_factory: CreateBrowserFactory, +) -> None: + sut = create_browser_factory({9000}) + browser = await sut("the-owner", "tests/workspaces/a_workspace") + + client = browser.client() + response = await client.get("/") + assert response is not None + + +@browser_factory_test +async def test__launching_on_an_allocated_port__raises_unavailable_port_error( + create_browser_factory: CreateBrowserFactory, +) -> None: + _factory = create_browser_factory({9000}) + await _factory("first-owner", "tests/workspaces/a_workspace") + + sut = create_browser_factory({9000}) + with pytest.raises(NoPortsAvailableError): + await sut("second-owner", "tests/workspaces/a_workspace") + + +@browser_factory_test +async def test__one_port_allocated__launches_on_next_available( + create_browser_factory: CreateBrowserFactory, +) -> None: + _factory = create_browser_factory({9000}) + await _factory("other-owner", "tests/workspaces/a_workspace") + + sut = create_browser_factory({9000, 9001}) + browser = await sut("second-other-owner", "tests/workspaces/a_workspace") + + assert browser.address() == "http://localhost:9001" diff --git a/tests/ocrdbrowser/test_launch.py b/tests/ocrdbrowser/test_launch.py deleted file mode 100644 index 8739168..0000000 --- a/tests/ocrdbrowser/test_launch.py +++ /dev/null @@ -1,51 +0,0 @@ -from typing import cast - -import pytest - -import ocrdbrowser -from tests.testdoubles import BrowserSpy, IteratingBrowserTestDoubleFactory - - -@pytest.mark.asyncio -async def test__workspace__launch__spawns_new_ocrd_browser() -> None: - owner = "the-owner" - workspace = "path/to/workspace" - process = await ocrdbrowser.launch( - workspace, owner, IteratingBrowserTestDoubleFactory() - ) - - process = cast(BrowserSpy, process) - assert process.is_running is True - assert process.owner() == owner - assert process.workspace() == workspace - - -@pytest.mark.asyncio -async def test__workspace__launch_for_different_owners__both_processes_running() -> None: - factory = IteratingBrowserTestDoubleFactory() - - first_process = await ocrdbrowser.launch("first-path", "first-owner", factory) - second_process = await ocrdbrowser.launch( - "second-path", "second-owner", factory, {first_process} - ) - - processes = {first_process, second_process} - assert all(cast(BrowserSpy, process).is_running for process in processes) - assert {p.owner() for p in processes} == {"first-owner", "second-owner"} - assert {p.workspace() for p in processes} == {"first-path", "second-path"} - - -@pytest.mark.asyncio -async def test__workspace__launch_for_same_owner_and_workspace__does_not_start_new_process() -> ( - None -): - owner = "the-owner" - workspace = "the-workspace" - factory = IteratingBrowserTestDoubleFactory() - - first_process = await ocrdbrowser.launch(workspace, owner, factory) - second_process = await ocrdbrowser.launch( - workspace, owner, factory, {first_process} - ) - - assert first_process is second_process diff --git a/tests/ocrdmonitor/server/conftest.py b/tests/ocrdmonitor/server/conftest.py index 8905966..8197e28 100644 --- a/tests/ocrdmonitor/server/conftest.py +++ b/tests/ocrdmonitor/server/conftest.py @@ -1 +1,5 @@ -from .fixtures import app, launch_monitor +from .fixtures.fixtureconfig import ( + app, # noqa: F401 + browser_fixture, # noqa: F401 + repository_fixture, # noqa: F401 +) diff --git a/tests/ocrdmonitor/server/decorators.py b/tests/ocrdmonitor/server/decorators.py new file mode 100644 index 0000000..53be862 --- /dev/null +++ b/tests/ocrdmonitor/server/decorators.py @@ -0,0 +1,23 @@ +import pytest +from tests import markers +from tests.decorators import compose + +from tests.ocrdmonitor.server.fixtures.repository import ( + inmemory_repository, + mongodb_repository, +) + + +use_custom_repository = compose( + pytest.mark.asyncio, + pytest.mark.parametrize( + "repository", + ( + pytest.param(inmemory_repository), + pytest.param( + mongodb_repository, + marks=(pytest.mark.integration, markers.skip_if_no_docker), + ), + ), + ), +) diff --git a/tests/ocrdmonitor/server/fixtures.py b/tests/ocrdmonitor/server/fixtures.py deleted file mode 100644 index 8a5b5cd..0000000 --- a/tests/ocrdmonitor/server/fixtures.py +++ /dev/null @@ -1,60 +0,0 @@ -from contextlib import asynccontextmanager -from pathlib import Path -from typing import AsyncIterator, Iterator -from unittest.mock import patch - -import pytest -import uvicorn -from fastapi.testclient import TestClient - -from ocrdmonitor.server.app import create_app -from ocrdmonitor.server.settings import ( - OcrdBrowserSettings, - OcrdControllerSettings, - OcrdLogViewSettings, - Settings, -) -from tests.testdoubles import BackgroundProcess, BrowserTestDoubleFactory - -JOB_DIR = Path(__file__).parent / "ocrd.jobs" -WORKSPACE_DIR = Path("tests") / "workspaces" - - -def create_settings() -> Settings: - return Settings( - ocrd_browser=OcrdBrowserSettings( - workspace_dir=WORKSPACE_DIR, - port_range=(9000, 9100), - ), - ocrd_controller=OcrdControllerSettings( - job_dir=JOB_DIR, - host="", - user="", - ), - ocrd_logview=OcrdLogViewSettings(port=8022), - ) - - -@asynccontextmanager -async def patch_factory( - factory: BrowserTestDoubleFactory, -) -> AsyncIterator[BrowserTestDoubleFactory]: - async with factory: - with patch.object(OcrdBrowserSettings, "factory", lambda _: factory): - yield factory - - -@pytest.fixture -def app() -> TestClient: - return TestClient(create_app(create_settings())) - - -def _launch_app() -> None: - app = create_app(create_settings()) - uvicorn.run(app, port=3000) - - -@pytest.fixture -def launch_monitor() -> Iterator[None]: - with BackgroundProcess(_launch_app): - yield diff --git a/tests/ocrdmonitor/server/fixtures/__init__.py b/tests/ocrdmonitor/server/fixtures/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/ocrdmonitor/server/fixtures/environment.py b/tests/ocrdmonitor/server/fixtures/environment.py new file mode 100644 index 0000000..134eba9 --- /dev/null +++ b/tests/ocrdmonitor/server/fixtures/environment.py @@ -0,0 +1,131 @@ +from dataclasses import dataclass +from types import TracebackType +from typing import ( + Any, + AsyncContextManager, + Callable, + ContextManager, + Self, + Type, +) + +from fastapi.testclient import TestClient + +from ocrdmonitor.browserprocess import BrowserProcessRepository, BrowserRestoringFactory +from ocrdmonitor.server.app import create_app +from tests.ocrdmonitor.server.fixtures.factory import patch_factory +from tests.ocrdmonitor.server.fixtures.repository import ( + inmemory_repository, + patch_repository, +) +from tests.ocrdmonitor.server.fixtures.settings import create_settings +from tests.testdoubles import ( + BrowserRegistry, + BrowserSpy, + BrowserTestDouble, + IteratingBrowserTestDoubleFactory, + RegistryBrowserFactory, + RestoringRegistryBrowserFactory, +) + + +@dataclass +class Environment: + repository: BrowserProcessRepository + app: TestClient + + +BrowserConstructor = Callable[[], BrowserTestDouble] +RepositoryInitializer = Callable[ + [BrowserRestoringFactory], + AsyncContextManager[BrowserProcessRepository], +] + + +class Fixture: + def __init__(self) -> None: + self.browser_constructor: BrowserConstructor = BrowserSpy + self.repo_constructor: RepositoryInitializer = inmemory_repository + self.existing_browsers: list[BrowserTestDouble] = [] + self.session_id = "" + + self._open_contexts: list[ContextManager[Any] | AsyncContextManager[Any]] = [] + + def with_browser_type(self, browser_constructor: BrowserConstructor) -> Self: + self.browser_constructor = browser_constructor + return self + + def with_repository_type(self, repo_constructor: RepositoryInitializer) -> Self: + self.repo_constructor = repo_constructor + return self + + def with_running_browsers(self, *browsers: BrowserTestDouble) -> Self: + self.existing_browsers = list(browsers) + return self + + def with_session_id(self, session_id: str) -> Self: + self.session_id = session_id + return self + + async def __aenter__(self) -> Environment: + registry = BrowserRegistry({}) + repository = await self._patch_repository(registry) + await self._patch_factory(registry) + await self._insert_running_browsers(registry, repository) + app = self._build_app() + + return Environment(repository=repository, app=app) + + async def _patch_repository( + self, registry: BrowserRegistry + ) -> BrowserProcessRepository: + repository = await self._init_repo(registry) + patcher = patch_repository(repository) + self._open_contexts.append(patcher) + await patcher.__aenter__() + return repository + + async def _init_repo(self, registry: BrowserRegistry) -> BrowserProcessRepository: + restoring_factory = RestoringRegistryBrowserFactory(registry) + repo_ctx = self.repo_constructor(restoring_factory) + self._open_contexts.append(repo_ctx) + repository = await repo_ctx.__aenter__() + return repository + + async def _patch_factory(self, registry: BrowserRegistry) -> None: + creating_factory = RegistryBrowserFactory( + IteratingBrowserTestDoubleFactory(default_browser=self.browser_constructor), + registry, + ) + patcher = patch_factory(creating_factory) + await patcher.__aenter__() + self._open_contexts.append(patcher) + + def _build_app(self) -> TestClient: + app = TestClient(create_app(create_settings())) + app.__enter__() + if self.session_id: + app.cookies["session_id"] = self.session_id + + self._open_contexts.append(app) + return app + + async def _insert_running_browsers( + self, registry: BrowserRegistry, repository: BrowserProcessRepository + ) -> None: + for browser in self.existing_browsers: + registry[browser.address()] = browser + await repository.insert(browser) + await browser.start() + + async def __aexit__( + self, + exc_type: Type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: + for ctx in self._open_contexts: + if isinstance(ctx, AsyncContextManager): + await ctx.__aexit__(exc_type, exc_value, traceback) + else: + ctx.__exit__(exc_type, exc_value, traceback) diff --git a/tests/ocrdmonitor/server/fixtures/factory.py b/tests/ocrdmonitor/server/fixtures/factory.py new file mode 100644 index 0000000..68f8815 --- /dev/null +++ b/tests/ocrdmonitor/server/fixtures/factory.py @@ -0,0 +1,15 @@ +from contextlib import asynccontextmanager +from typing import AsyncIterator +from unittest.mock import patch + +from ocrdmonitor.server.settings import OcrdBrowserSettings +from tests.testdoubles import BrowserTestDoubleFactory + + +@asynccontextmanager +async def patch_factory( + factory: BrowserTestDoubleFactory, +) -> AsyncIterator[BrowserTestDoubleFactory]: + async with factory: + with patch.object(OcrdBrowserSettings, "factory", lambda _: factory): + yield factory diff --git a/tests/ocrdmonitor/server/fixtures/fixtureconfig.py b/tests/ocrdmonitor/server/fixtures/fixtureconfig.py new file mode 100644 index 0000000..44f3fe9 --- /dev/null +++ b/tests/ocrdmonitor/server/fixtures/fixtureconfig.py @@ -0,0 +1,40 @@ +from typing import AsyncIterator + +import pytest +import pytest_asyncio +from fastapi.testclient import TestClient + +from tests import markers +from tests.testdoubles import BrowserFake, BrowserSpy + +from .environment import Fixture, RepositoryInitializer +from .repository import inmemory_repository, mongodb_repository + + +@pytest.fixture( + params=[ + inmemory_repository, + pytest.param( + mongodb_repository, + marks=(pytest.mark.integration, markers.skip_if_no_docker), + ), + ] +) +def repository_fixture(request: pytest.FixtureRequest) -> Fixture: + repository: RepositoryInitializer = request.param + return Fixture().with_repository_type(repository) + + +@pytest.fixture( + params=[BrowserSpy, pytest.param(BrowserFake, marks=pytest.mark.integration)] +) +def browser_fixture( + repository_fixture: Fixture, request: pytest.FixtureRequest +) -> Fixture: + return repository_fixture.with_browser_type(request.param) + + +@pytest_asyncio.fixture +async def app() -> AsyncIterator[TestClient]: + async with Fixture() as env: + yield env.app diff --git a/tests/ocrdmonitor/server/fixtures/repository.py b/tests/ocrdmonitor/server/fixtures/repository.py new file mode 100644 index 0000000..4a1d6bb --- /dev/null +++ b/tests/ocrdmonitor/server/fixtures/repository.py @@ -0,0 +1,35 @@ +from contextlib import asynccontextmanager +from typing import AsyncIterator +from unittest.mock import patch + +from testcontainers.mongodb import MongoDbContainer + +from ocrdmonitor import dbmodel +from ocrdmonitor.browserprocess import BrowserProcessRepository, BrowserRestoringFactory +from ocrdmonitor.server.settings import OcrdBrowserSettings +from tests.testdoubles import InMemoryBrowserProcessRepository + + +@asynccontextmanager +async def mongodb_repository( + restoring_factory: BrowserRestoringFactory, +) -> AsyncIterator[dbmodel.MongoBrowserProcessRepository]: + with MongoDbContainer() as container: + await dbmodel.init(container.get_connection_url(), force_initialize=True) + yield dbmodel.MongoBrowserProcessRepository(restoring_factory) + + +@asynccontextmanager +async def inmemory_repository( + restoring_factory: BrowserRestoringFactory, +) -> AsyncIterator[InMemoryBrowserProcessRepository]: + yield InMemoryBrowserProcessRepository(restoring_factory) + + +@asynccontextmanager +async def patch_repository(repository: BrowserProcessRepository) -> AsyncIterator[None]: + async def _repository(_: OcrdBrowserSettings) -> BrowserProcessRepository: + return repository + + with patch.object(OcrdBrowserSettings, "repository", _repository): + yield diff --git a/tests/ocrdmonitor/server/fixtures/settings.py b/tests/ocrdmonitor/server/fixtures/settings.py new file mode 100644 index 0000000..aab973b --- /dev/null +++ b/tests/ocrdmonitor/server/fixtures/settings.py @@ -0,0 +1,27 @@ +from pathlib import Path + +from ocrdmonitor.server.settings import ( + OcrdBrowserSettings, + OcrdControllerSettings, + OcrdLogViewSettings, + Settings, +) + +JOB_DIR = Path(__file__).parent / "ocrd.jobs" +WORKSPACE_DIR = Path("tests") / "workspaces" + + +def create_settings() -> Settings: + return Settings( + ocrd_browser=OcrdBrowserSettings( + workspace_dir=WORKSPACE_DIR, + port_range=(9000, 9100), + db_connection_string="", + ), + ocrd_controller=OcrdControllerSettings( + job_dir=JOB_DIR, + host="", + user="", + ), + ocrd_logview=OcrdLogViewSettings(port=8022), + ) diff --git a/tests/ocrdmonitor/server/test_job_endpoint.py b/tests/ocrdmonitor/server/test_job_endpoint.py index 58fa120..930adc8 100644 --- a/tests/ocrdmonitor/server/test_job_endpoint.py +++ b/tests/ocrdmonitor/server/test_job_endpoint.py @@ -8,12 +8,13 @@ import pytest from fastapi.testclient import TestClient from httpx import Response + from ocrdmonitor.ocrdcontroller import RemoteServer from ocrdmonitor.ocrdjob import OcrdJob from ocrdmonitor.processstatus import ProcessState, ProcessStatus from ocrdmonitor.server.settings import OcrdControllerSettings from tests.ocrdmonitor.server import scraping -from tests.ocrdmonitor.server.fixtures import JOB_DIR +from tests.ocrdmonitor.server.fixtures.settings import JOB_DIR from tests.ocrdmonitor.test_jobs import JOB_TEMPLATE, jobfile_content_for diff --git a/tests/ocrdmonitor/server/test_settings.py b/tests/ocrdmonitor/server/test_settings.py index 369bb34..d71bf79 100644 --- a/tests/ocrdmonitor/server/test_settings.py +++ b/tests/ocrdmonitor/server/test_settings.py @@ -17,6 +17,7 @@ mode="native", workspace_dir="path/to/workdir", port_range=(9000, 9100), + db_connection_string="user@mongo:mongodb:1234" ), ocrd_controller=OcrdControllerSettings( job_dir="path/to/jobdir", diff --git a/tests/ocrdmonitor/server/test_startup.py b/tests/ocrdmonitor/server/test_startup.py new file mode 100644 index 0000000..0db4568 --- /dev/null +++ b/tests/ocrdmonitor/server/test_startup.py @@ -0,0 +1,22 @@ +import pytest + +from tests.ocrdmonitor.server.fixtures.environment import Fixture +from tests.testdoubles import BrowserSpy, unreachable_browser + + +@pytest.mark.asyncio +async def test__browsers_in_db__on_startup__cleans_unreachables_from_db( + repository_fixture: Fixture, +) -> None: + session_id = "the-owner" + reachable = BrowserSpy(owner=session_id, address="http://reachable.com") + unreachable = unreachable_browser( + owner=session_id, address="http://unreachable.com" + ) + + fixture = repository_fixture.with_running_browsers( + reachable, unreachable + ).with_session_id(session_id) + + async with fixture as env: + assert await env.repository.count() == 1 diff --git a/tests/ocrdmonitor/server/test_workspace_endpoint.py b/tests/ocrdmonitor/server/test_workspace_endpoint.py index e901414..8287c41 100644 --- a/tests/ocrdmonitor/server/test_workspace_endpoint.py +++ b/tests/ocrdmonitor/server/test_workspace_endpoint.py @@ -1,89 +1,91 @@ from __future__ import annotations -from typing import AsyncIterator, cast +import asyncio +from typing import AsyncIterator import pytest import pytest_asyncio from fastapi.testclient import TestClient from httpx import Response +from tests import markers -from ocrdbrowser import ChannelClosed from tests.ocrdmonitor.server import scraping -from tests.ocrdmonitor.server.fixtures import WORKSPACE_DIR, patch_factory -from tests.testdoubles import BrowserFake -from tests.testdoubles._browserfactory import ( - BrowserTestDoubleFactory, - IteratingBrowserTestDoubleFactory, - SingletonBrowserTestDoubleFactory, +from tests.ocrdmonitor.server.decorators import use_custom_repository +from tests.ocrdmonitor.server.fixtures.environment import ( + Environment, + Fixture, + RepositoryInitializer, ) -from tests.testdoubles import Browser_Heading, BrowserSpy, BrowserTestDouble - - -class DisconnectingChannel: - async def send_bytes(self, data: bytes) -> None: - raise ChannelClosed() - - async def receive_bytes(self) -> bytes: - raise ChannelClosed() - - -@pytest_asyncio.fixture( - params=(BrowserSpy, pytest.param(BrowserFake, marks=pytest.mark.integration)) +from tests.ocrdmonitor.server.fixtures.repository import ( + inmemory_repository, + mongodb_repository, +) +from tests.ocrdmonitor.server.fixtures.settings import WORKSPACE_DIR +from tests.testdoubles import ( + Browser_Heading, + BrowserFake, + BrowserSpy, + browser_with_disconnecting_channel, + unreachable_browser, ) -async def iterating_factory( - request: pytest.FixtureRequest, -) -> AsyncIterator[BrowserTestDoubleFactory]: - async with patch_factory( - IteratingBrowserTestDoubleFactory(default_browser=request.param) - ) as factory: - yield factory - - -@pytest_asyncio.fixture -async def singleton_browser_spy() -> AsyncIterator[BrowserSpy]: - browser_spy = BrowserSpy() - async with patch_factory(SingletonBrowserTestDoubleFactory(browser_spy)): - yield browser_spy @pytest.fixture( - params=(BrowserSpy, pytest.param(BrowserFake, marks=pytest.mark.integration)) + params=[ + inmemory_repository, + pytest.param( + mongodb_repository, + marks=(pytest.mark.integration, markers.skip_if_no_docker), + ), + ] ) -def browser( - iterating_factory: IteratingBrowserTestDoubleFactory, - request: pytest.FixtureRequest, -) -> BrowserTestDouble: - browser_type = request.param - browser = cast(BrowserTestDouble, browser_type()) - iterating_factory.add(browser) +def repository_fixture(request: pytest.FixtureRequest) -> Fixture: + repository: RepositoryInitializer = request.param + return Fixture().with_repository_type(repository) - return browser +@pytest.fixture( + params=[BrowserSpy, pytest.param(BrowserFake, marks=pytest.mark.integration)] +) +def browser_fixture( + repository_fixture: Fixture, request: pytest.FixtureRequest +) -> Fixture: + return repository_fixture.with_browser_type(request.param) -@pytest.fixture -def disconnecting_browser( - iterating_factory: IteratingBrowserTestDoubleFactory, -) -> BrowserSpy: - disconnecting_browser = BrowserSpy() - disconnecting_browser.configure_client(channel=DisconnectingChannel()) - iterating_factory.add(disconnecting_browser) - return disconnecting_browser +@pytest_asyncio.fixture +async def app(browser_fixture: Fixture) -> AsyncIterator[TestClient]: + async with browser_fixture as env: + yield env.app def assert_is_browser_response(actual: Response) -> None: assert scraping.parse_texts(actual.content, "h1") == [Browser_Heading] -def view_workspace(app: TestClient, workspace: str) -> Response: - _ = app.get(f"/workspaces/browse/{workspace}") - response = app.get(f"/workspaces/view/{workspace}") +def interact_with_workspace(app: TestClient, workspace: str) -> Response: + open_workspace(app, workspace) + response = view_workspace(app, workspace) with app.websocket_connect(f"/workspaces/view/{workspace}/socket"): pass - return response +def open_workspace(app: TestClient, workspace: str) -> Response: + _ = app.get(f"/workspaces/open/{workspace}") + return app.get(f"/workspaces/browse/{workspace}") + + +def view_workspace(app: TestClient, workspace: str) -> Response: + return app.get(f"/workspaces/view/{workspace}") + + +@pytest_asyncio.fixture +async def defaultenv(browser_fixture: Fixture) -> AsyncIterator[Environment]: + async with browser_fixture as env: + yield env + + def test__workspaces__shows_the_workspace_names_starting_from_workspace_root( app: TestClient, ) -> None: @@ -93,20 +95,29 @@ def test__workspaces__shows_the_workspace_names_starting_from_workspace_root( assert set(texts) == {"a_workspace", "another workspace", "nested/workspace"} -def test__browse_workspace__passes_full_workspace_path_to_ocrdbrowser( - browser: BrowserTestDouble, - app: TestClient, +@use_custom_repository +async def test__browse_workspace__passes_full_workspace_path_to_ocrdbrowser( + repository: RepositoryInitializer, ) -> None: - response = app.get("/workspaces/browse/a_workspace") + workspace = "a_workspace" + full_workspace = str(WORKSPACE_DIR / workspace) + browser = BrowserSpy() + fixture = ( + Fixture().with_repository_type(repository).with_browser_type(lambda: browser) + ) - assert browser.is_running is True - assert browser.workspace() == str(WORKSPACE_DIR / "a_workspace") - assert response.status_code == 200 + async with fixture as env: + response = open_workspace(env.app, workspace) + assert browser.is_running is True + assert browser.workspace() == full_workspace + assert response.status_code == 200 -@pytest.mark.usefixtures("iterating_factory") -def test__browse_workspace__assigns_and_tracks_session_id(app: TestClient) -> None: - response = app.get("/workspaces/browse/a_workspace") + +def test__browse_workspace__assigns_and_tracks_session_id( + app: TestClient, +) -> None: + response = open_workspace(app, "a_workspace") first_session_id = response.cookies.get("session_id") response = app.get("/workspaces/browse/a_workspace") @@ -116,61 +127,160 @@ def test__browse_workspace__assigns_and_tracks_session_id(app: TestClient) -> No assert first_session_id == second_session_id -def test__opened_workspace__when_socket_disconnects_on_broadway_side__shuts_down_browser( - disconnecting_browser: BrowserSpy, - app: TestClient, +@pytest.mark.asyncio +async def test__opened_workspace__when_socket_disconnects__shuts_down_browser( + browser_fixture: Fixture, ) -> None: - _ = view_workspace(app, "a_workspace") + session_id = "the-owner" + disconnecting_browser = browser_with_disconnecting_channel( + session_id, str(WORKSPACE_DIR / "a_workspace") + ) + + fixture = browser_fixture.with_running_browsers( + disconnecting_browser + ).with_session_id(session_id) + + async with fixture as env: + _ = interact_with_workspace(env.app, "a_workspace") assert disconnecting_browser.is_running is False -def test__disconnected_workspace__when_opening_again__starts_new_browser( - disconnecting_browser: BrowserTestDouble, - browser: BrowserTestDouble, - app: TestClient, +@pytest.mark.asyncio +async def test__disconnected_workspace__when_opening_again__viewing_proxies_requests_to_browser( + browser_fixture: Fixture, ) -> None: + session_id = "the-owner" workspace = "a_workspace" - _ = view_workspace(app, workspace) + full_workspace = str(WORKSPACE_DIR / workspace) + fixture = browser_fixture.with_running_browsers( + browser_with_disconnecting_channel(session_id, full_workspace) + ).with_session_id(session_id) - _ = view_workspace(app, workspace) + async with fixture as env: + _ = interact_with_workspace(env.app, workspace) - assert disconnecting_browser.is_running is False - assert browser.is_running is True + actual = interact_with_workspace(env.app, workspace) + assert_is_browser_response(actual) -@pytest.mark.usefixtures("disconnecting_browser") -def test__disconnected_workspace__when_opening_again__viewing_proxies_requests_to_browser( - app: TestClient, + +@pytest.mark.asyncio +async def test__when_requesting_resource__returns_resource_from_workspace( + browser_fixture: Fixture, ) -> None: + session_id = "the-owner" workspace = "a_workspace" - _ = view_workspace(app, workspace) + resource = "/some_resource" + resource_in_workspace = workspace + "/some_resource" + full_workspace = str(WORKSPACE_DIR / workspace) - actual = view_workspace(app, workspace) + def echo_bytes(path: str) -> bytes: + return path.encode() - assert_is_browser_response(actual) + browser = BrowserSpy(session_id, full_workspace) + browser.configure_client(response_factory=echo_bytes) + + fixture = browser_fixture.with_running_browsers(browser).with_session_id(session_id) + + async with fixture as env: + open_workspace(env.app, workspace) + + actual = view_workspace(env.app, resource_in_workspace) + + assert actual.content == resource.encode() -@pytest.mark.usefixtures("iterating_factory") def test__browsed_workspace_is_ready__when_pinging__returns_ok( app: TestClient, ) -> None: workspace = "a_workspace" - _ = view_workspace(app, workspace) + _ = interact_with_workspace(app, workspace) result = app.get(f"/workspaces/ping/{workspace}") assert result.status_code == 200 -def test__browsed_workspace_not_ready__when_pinging__returns_bad_gateway( - singleton_browser_spy: BrowserSpy, - app: TestClient, +@pytest.mark.asyncio +async def test__browsed_workspace_not_ready__when_pinging__returns_bad_gateway( + repository_fixture: Fixture, ) -> None: - singleton_browser_spy.configure_client(response=ConnectionError) workspace = "a_workspace" - _ = view_workspace(app, workspace) + fixture = repository_fixture.with_browser_type(unreachable_browser) - result = app.get(f"/workspaces/ping/{workspace}") + async with fixture as env: + open_workspace(env.app, workspace) + + result = env.app.get(f"/workspaces/ping/{workspace}") assert result.status_code == 502 + + +@pytest.mark.asyncio +async def test__browsing_workspace__stores_browser_in_repository( + defaultenv: Environment, +) -> None: + _ = interact_with_workspace(defaultenv.app, "a_workspace") + + found_browsers = list( + await defaultenv.repository.find(workspace=str(WORKSPACE_DIR / "a_workspace")) + ) + + assert len(found_browsers) == 1 + + +@pytest.mark.asyncio +async def test__error_connecting_to_workspace__removes_browser_from_repository( + repository_fixture: Fixture, +) -> None: + fixture = repository_fixture.with_browser_type(unreachable_browser) + async with fixture as env: + open_workspace(env.app, "a_workspace") + _ = view_workspace(env.app, "a_workspace") + + browsers = await env.repository.find( + workspace=str(WORKSPACE_DIR / "a_workspace") + ) + + assert len(list(browsers)) == 0 + + +@pytest.mark.asyncio +async def test__when_socket_to_workspace_disconnects__removes_browser_from_repository( + repository_fixture: Fixture, +) -> None: + # NOTE: it seems something is weird with the event loop in this test + # Searching for browsers inside the with block happens BEFORE the browser is deleted + # I'm not sure if this is a bug in the FastAPI TestClient or if we're doing something wrong here + # We apply a little hack and sleep for .1 seconds, handing control back to the event loop + + fixture = repository_fixture.with_browser_type(browser_with_disconnecting_channel) + + async with fixture as env: + _ = interact_with_workspace(env.app, "a_workspace") + await asyncio.sleep(0.1) + + browsers = await env.repository.find( + workspace=str(WORKSPACE_DIR / "a_workspace") + ) + + assert len(list(browsers)) == 0 + + +@pytest.mark.asyncio +async def test__browser_stored_in_repo__when_browsing_workspace_redirects_to_restored_browser( + browser_fixture: Fixture, +) -> None: + session_id = "the-owner" + workspace = "a_workspace" + full_workspace = str(WORKSPACE_DIR / workspace) + browser = BrowserSpy(session_id, full_workspace) + browser.configure_client(response=b"RESTORED BROWSER") + + fixture = browser_fixture.with_running_browsers(browser).with_session_id(session_id) + + async with fixture as env: + response = interact_with_workspace(env.app, "a_workspace") + + assert response.content == b"RESTORED BROWSER" diff --git a/tests/ocrdmonitor/test_redirect.py b/tests/ocrdmonitor/test_redirect.py deleted file mode 100644 index a5abe07..0000000 --- a/tests/ocrdmonitor/test_redirect.py +++ /dev/null @@ -1,73 +0,0 @@ -from pathlib import Path - -import pytest -from ocrdmonitor.server.redirect import BrowserRedirect - -from tests.testdoubles._browserspy import BrowserSpy - - -def server_stub(address: str) -> BrowserSpy: - return BrowserSpy(address=address) - - -SERVER_ADDRESS = "http://example.com:8080" - - -def test__redirect_for_empty_url_returns_server_address() -> None: - workspace = Path("path/to/workspace") - browser = server_stub("http://example.com:8080") - sut = BrowserRedirect(workspace, browser) - - assert sut.redirect_url("") == browser.address() - - -@pytest.mark.parametrize("address", [SERVER_ADDRESS, SERVER_ADDRESS + "/"]) -@pytest.mark.parametrize("filename", ["file.js", "/file.js"]) -def test__redirect_to_file_in_workspace__returns_server_slash_file( - address: str, - filename: str, -) -> None: - workspace = Path("path/to/workspace") - browser = server_stub(address) - sut = BrowserRedirect(workspace, browser) - - assert sut.redirect_url(filename) == url(address, filename) - - -def test__redirect_from_workspace__returns_server_address() -> None: - workspace = Path("path/to/workspace") - browser = server_stub("http://example.com:8080") - sut = BrowserRedirect(workspace, browser) - - assert sut.redirect_url(str(workspace)) == browser.address() - - -def test__redirect_with_workspace__is_a_match() -> None: - workspace = Path("path/to/workspace") - browser = server_stub("") - sut = BrowserRedirect(workspace, browser) - - assert sut.matches(str(workspace)) is True - - -def test__an_empty_path__does_not_match() -> None: - workspace = Path("path/to/workspace") - browser = server_stub("") - sut = BrowserRedirect(workspace, browser) - - assert sut.matches("") is False - - -def test__a_path_starting_with_workspace__is_a_match() -> None: - workspace = Path("path/to/workspace") - browser = server_stub("") - sut = BrowserRedirect(workspace, browser) - - sub_path = workspace / "sub" / "path" / "file.txt" - assert sut.matches(str(sub_path)) is True - - -def url(server_address: str, subpath: str) -> str: - server_address = server_address.removesuffix("/") - subpath = subpath.removeprefix("/") - return server_address + "/" + subpath diff --git a/tests/ocrdmonitor/test_sshremote.py b/tests/ocrdmonitor/test_sshremote.py index ab648be..92a69c2 100644 --- a/tests/ocrdmonitor/test_sshremote.py +++ b/tests/ocrdmonitor/test_sshremote.py @@ -6,6 +6,7 @@ from ocrdmonitor.processstatus import ProcessState from ocrdmonitor.sshremote import SSHRemote +from tests import markers from tests.ocrdmonitor.sshcontainer import ( get_process_group_from_container, SSHConfig, @@ -17,7 +18,7 @@ @pytest.mark.asyncio @pytest.mark.integration -@pytest.mark.needs_docker +@markers.skip_if_no_docker async def test_ps_over_ssh__returns_list_of_process_status( openssh_server: DockerContainer, ) -> None: diff --git a/tests/testdoubles/__init__.py b/tests/testdoubles/__init__.py index 9832aee..6b5bfa3 100644 --- a/tests/testdoubles/__init__.py +++ b/tests/testdoubles/__init__.py @@ -1,13 +1,23 @@ from ._backgroundprocess import BackgroundProcess -from ._broadwayfake import broadway_fake, FAKE_HOST_ADDRESS +from ._broadwayfake import FAKE_HOST_ADDRESS, broadway_fake from ._browserfactory import ( BrowserTestDouble, BrowserTestDoubleFactory, IteratingBrowserTestDoubleFactory, - SingletonBrowserTestDoubleFactory, ) from ._browserfake import BrowserFake -from ._browserspy import BrowserSpy, Browser_Heading +from ._browserprocessrepository import InMemoryBrowserProcessRepository +from ._browserspy import ( + Browser_Heading, + BrowserSpy, + browser_with_disconnecting_channel, + unreachable_browser, +) +from ._registrybrowserfactory import ( + BrowserRegistry, + RegistryBrowserFactory, + RestoringRegistryBrowserFactory, +) __all__ = [ "BackgroundProcess", @@ -18,6 +28,12 @@ "BrowserTestDouble", "BrowserTestDoubleFactory", "FAKE_HOST_ADDRESS", - "SingletonBrowserTestDoubleFactory", "IteratingBrowserTestDoubleFactory", + "InMemoryBrowserProcessRepository", + "BrowserRegistry", + "ProxyBrowser", + "RegistryBrowserFactory", + "RestoringRegistryBrowserFactory", + "browser_with_disconnecting_channel", + "unreachable_browser", ] diff --git a/tests/testdoubles/_backgroundprocess.py b/tests/testdoubles/_backgroundprocess.py index 159682e..a2dc529 100644 --- a/tests/testdoubles/_backgroundprocess.py +++ b/tests/testdoubles/_backgroundprocess.py @@ -25,6 +25,13 @@ def __exit__(self, *args: Any, **kwargs: Any) -> None: def is_running(self) -> bool: return self._process is not None and self._process.is_alive() + @property + def pid(self) -> int | None: + if not self._process: + return None + + return self._process.pid + def launch(self) -> None: if self.is_running: return diff --git a/tests/testdoubles/_broadwayfake.py b/tests/testdoubles/_broadwayfake.py index 5d7b07a..9824abc 100644 --- a/tests/testdoubles/_broadwayfake.py +++ b/tests/testdoubles/_broadwayfake.py @@ -8,7 +8,7 @@ FAKE_HOST_IP = "127.0.0.1" -FAKE_HOST_PORT = 7000 +FAKE_HOST_PORT = 8000 FAKE_HOST_ADDRESS = f"{FAKE_HOST_IP}:{FAKE_HOST_PORT}" diff --git a/tests/testdoubles/_browserfactory.py b/tests/testdoubles/_browserfactory.py index 6a3611d..6807a02 100644 --- a/tests/testdoubles/_browserfactory.py +++ b/tests/testdoubles/_browserfactory.py @@ -1,8 +1,9 @@ import asyncio from types import TracebackType -from typing import Callable, Protocol, Self, Type +from typing import AsyncContextManager, Callable, Protocol, Self, Type + +from ocrdbrowser import OcrdBrowser, OcrdBrowserFactory -from ocrdbrowser import OcrdBrowser from ._browserspy import BrowserSpy @@ -10,31 +11,14 @@ class BrowserTestDouble(OcrdBrowser, Protocol): def set_owner_and_workspace(self, owner: str, workspace: str) -> None: ... + async def start(self) -> None: + ... + @property def is_running(self) -> bool: ... -class SingletonBrowserTestDoubleFactory: - def __init__(self, browser: BrowserTestDouble | None = None) -> None: - self._browser = browser or BrowserSpy() - - def __call__(self, owner: str, workspace_path: str) -> OcrdBrowser: - self._browser.set_owner_and_workspace(owner, workspace_path) - return self._browser - - async def __aenter__(self) -> Self: - return self - - async def __aexit__( - self, - exc_type: Type[BaseException] | None, - exc_value: BaseException | None, - traceback: TracebackType | None, - ) -> None: - await self._browser.stop() - - class IteratingBrowserTestDoubleFactory: def __init__( self, @@ -49,9 +33,10 @@ def __init__( def add(self, process: BrowserTestDouble) -> None: self._processes.append(process) - def __call__(self, owner: str, workspace_path: str) -> OcrdBrowser: + async def __call__(self, owner: str, workspace_path: str) -> OcrdBrowser: browser = next(self._proc_iter, self._default_browser()) browser.set_owner_and_workspace(owner, workspace_path) + await browser.start() self._created.append(browser) return browser @@ -69,6 +54,7 @@ async def __aexit__( group.create_task(browser.stop()) -BrowserTestDoubleFactory = ( - SingletonBrowserTestDoubleFactory | IteratingBrowserTestDoubleFactory -) +class BrowserTestDoubleFactory( + OcrdBrowserFactory, AsyncContextManager[OcrdBrowserFactory], Protocol +): + pass diff --git a/tests/testdoubles/_browserfake.py b/tests/testdoubles/_browserfake.py index 3fcd9b9..b3c89c5 100644 --- a/tests/testdoubles/_browserfake.py +++ b/tests/testdoubles/_browserfake.py @@ -21,6 +21,9 @@ def set_owner_and_workspace(self, owner: str, workspace: str) -> None: self._workspace = workspace self._browser = broadway_fake(workspace) + def process_id(self) -> str: + return str(self._browser.pid) + def address(self) -> str: return f"http://{FAKE_HOST_ADDRESS}" diff --git a/tests/testdoubles/_browserprocessrepository.py b/tests/testdoubles/_browserprocessrepository.py new file mode 100644 index 0000000..eb8a09e --- /dev/null +++ b/tests/testdoubles/_browserprocessrepository.py @@ -0,0 +1,78 @@ +from typing import Collection, NamedTuple + +from ocrdbrowser import OcrdBrowser +from ocrdmonitor.browserprocess import BrowserRestoringFactory + +from ._browserspy import BrowserSpy + + +class BrowserEntry(NamedTuple): + owner: str + workspace: str + address: str + process_id: str + + +class InMemoryBrowserProcessRepository: + def __init__( + self, restoring_factory: BrowserRestoringFactory | None = None + ) -> None: + self._processes: list[BrowserEntry] = [] + self.restoring_factory: BrowserRestoringFactory = ( + restoring_factory or BrowserSpy + ) + + async def insert(self, browser: OcrdBrowser) -> None: + entry = BrowserEntry( + browser.owner(), + browser.workspace(), + browser.address(), + browser.process_id(), + ) + + self._processes.append(entry) + + async def delete(self, browser: OcrdBrowser) -> None: + entry = BrowserEntry( + browser.owner(), + browser.workspace(), + browser.address(), + browser.process_id(), + ) + + self._processes.remove(entry) + + async def find( + self, + *, + owner: str | None = None, + workspace: str | None = None, + ) -> Collection[OcrdBrowser]: + def match(browser: BrowserEntry) -> bool: + matches = True + if owner is not None: + matches = matches and browser.owner == owner + + if workspace is not None: + matches = matches and browser.workspace == workspace + + return matches + + print(self._processes) + return [ + self.restoring_factory( + process_id=browser.process_id, + owner=browser.owner, + workspace=browser.workspace, + address=browser.address, + ) + for browser in self._processes + if match(browser) + ] + + async def first(self, *, owner: str, workspace: str) -> OcrdBrowser | None: + results = await self.find(owner=owner, workspace=workspace) + return next(iter(results), None) + + async def count(self) -> int: + return len(self._processes) diff --git a/tests/testdoubles/_browserspy.py b/tests/testdoubles/_browserspy.py index 212e081..58a27de 100644 --- a/tests/testdoubles/_browserspy.py +++ b/tests/testdoubles/_browserspy.py @@ -2,9 +2,9 @@ from contextlib import asynccontextmanager from textwrap import dedent -from typing import AsyncGenerator, Type +from typing import AsyncGenerator, Callable, Type -from ocrdbrowser import Channel, OcrdBrowserClient +from ocrdbrowser import Channel, ChannelClosed, OcrdBrowserClient Browser_Heading = "OCRD BROWSER" @@ -26,14 +26,29 @@ async def receive_bytes(self) -> bytes: return bytes() +class DisconnectingChannel: + async def send_bytes(self, data: bytes) -> None: + raise ChannelClosed() + + async def receive_bytes(self) -> bytes: + raise ChannelClosed() + + class BrowserClientStub: def __init__( - self, response: bytes | Type[Exception] = b"", channel: Channel | None = None + self, + response: bytes | Type[Exception] = b"", + channel: Channel | None = None, + response_factory: Callable[[str], bytes] | None = None, ) -> None: self.channel = channel or ChannelDummy() self.response = response or html_template.encode() + self.response_factory = response_factory async def get(self, resource: str) -> bytes: + if self.response_factory is not None: + return self.response_factory(resource) + if not isinstance(self.response, bytes): raise self.response @@ -48,25 +63,33 @@ class BrowserSpy: def __init__( self, owner: str = "", - workspace_path: str = "", + workspace: str = "", address: str = "http://unreachable.example.com", + process_id: str = "1234", running: bool = False, ) -> None: - self.is_running = running self._address = address + self._process_id = process_id + self.is_running = running self.owner_name = owner - self.workspace_path = workspace_path + self.workspace_path = workspace self._client = BrowserClientStub() def configure_client( - self, response: bytes | Type[Exception] = b"", channel: Channel | None = None + self, + response: bytes | Type[Exception] = b"", + channel: Channel | None = None, + response_factory: Callable[[str], bytes] | None = None, ) -> None: - self._client = BrowserClientStub(response, channel) + self._client = BrowserClientStub(response, channel, response_factory) def set_owner_and_workspace(self, owner: str, workspace: str) -> None: self.owner_name = owner self.workspace_path = workspace + def process_id(self) -> str: + return self._process_id + def address(self) -> str: return self._address @@ -92,5 +115,28 @@ def __repr__(self) -> str: workspace: {self.workspace()} owner: {self.owner()} running: {self.is_running} + process id: {self._process_id} """ ) + + +def browser_with_disconnecting_channel( + owner: str = "", + workspace: str = "", + address: str = "http://unreachable.example.com", + process_id: str = "1234", +) -> BrowserSpy: + spy = BrowserSpy(owner, workspace, address, process_id) + spy.configure_client(response=b"Disconnected", channel=DisconnectingChannel()) + return spy + + +def unreachable_browser( + owner: str = "", + workspace: str = "", + address: str = "http://unreachable.example.com", + process_id: str = "1234", +) -> BrowserSpy: + spy = BrowserSpy(owner, workspace, address, process_id) + spy.configure_client(response=ConnectionError) + return spy diff --git a/tests/testdoubles/_registrybrowserfactory.py b/tests/testdoubles/_registrybrowserfactory.py new file mode 100644 index 0000000..82b9f06 --- /dev/null +++ b/tests/testdoubles/_registrybrowserfactory.py @@ -0,0 +1,54 @@ +from types import TracebackType +from typing import NewType, Self, Type, cast + +from ocrdbrowser import OcrdBrowser + +from ._browserfactory import ( + BrowserTestDouble, + BrowserTestDoubleFactory, + IteratingBrowserTestDoubleFactory, +) + +BrowserRegistry = NewType("BrowserRegistry", dict[str, BrowserTestDouble]) + + +class RegistryBrowserFactory: + @classmethod + def iteratingfactory(cls: Type[Self], browser_registry: BrowserRegistry) -> Self: + return cls(IteratingBrowserTestDoubleFactory(), browser_registry) + + def __init__( + self, + internal_factory: BrowserTestDoubleFactory, + browser_registry: BrowserRegistry, + ) -> None: + self._factory = internal_factory + self._registry = browser_registry + + async def __aenter__(self) -> Self: + await self._factory.__aenter__() + return self + + async def __aexit__( + self, + exc_type: Type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: + await self._factory.__aexit__(exc_type, exc_value, traceback) + + async def __call__(self, owner: str, workspace_path: str) -> OcrdBrowser: + browser = await self._factory(owner, workspace_path) + self._registry[browser.address()] = cast(BrowserTestDouble, browser) + return browser + + +class RestoringRegistryBrowserFactory: + def __init__(self, browser_registry: BrowserRegistry) -> None: + self._registry = browser_registry + + def __call__( + self, owner: str, workspace: str, address: str, process_id: str + ) -> BrowserTestDouble: + browser = self._registry[address] + return browser diff --git a/tests/workspaces/a_workspace/mets.xml b/tests/workspaces/a_workspace/mets.xml index e69de29..82bb527 100644 --- a/tests/workspaces/a_workspace/mets.xml +++ b/tests/workspaces/a_workspace/mets.xml @@ -0,0 +1,25 @@ + + + + + Your Organization Name + + UniqueIdentifier123 + + + + + + + + \ No newline at end of file