diff --git a/index.rst b/index.rst index ead699b2a..8c02832fe 100644 --- a/index.rst +++ b/index.rst @@ -13,7 +13,6 @@ testcontainers-python testcontainers-python facilitates the use of Docker containers for functional and integration testing. The collection of packages currently supports the following features. .. toctree:: - :maxdepth: 1 core/README modules/index @@ -60,15 +59,12 @@ Installation ------------ The suite of testcontainers packages is available on `PyPI `_, -and the package can be installed using :code:`pip`. +and individual packages can be installed using :code:`pip`. -Version `4.0.0` onwards we do not support the `testcontainers-*` packages as it is unsustainable to maintain ownership. +Version `4.0.0` onwards we do not support the `testcontainers-*` packages as it is unsutainable to maintain ownership. Instead packages can be installed by specifying `extras `__, e.g., :code:`pip install testcontainers[postgres]`. -Please note, that community modules are supported on a best-effort basis and breaking changes DO NOT create major versions in the package. -Therefore, only the package core is strictly following SemVer. If your workflow is broken by a minor update, please look at the changelogs for guidance. - Custom Containers ----------------- @@ -84,75 +80,40 @@ For common use cases, you can also use the generic containers provided by the `t Docker in Docker (DinD) ----------------------- -When trying to launch Testcontainers from within a Docker container, e.g., in continuous integration testing, two things have to be provided: +When trying to launch a testcontainer from within a Docker container, e.g., in continuous integration testing, two things have to be provided: 1. The container has to provide a docker client installation. Either use an image that has docker pre-installed (e.g. the `official docker images `_) or install the client from within the `Dockerfile` specification. 2. The container has to have access to the docker daemon which can be achieved by mounting `/var/run/docker.sock` or setting the `DOCKER_HOST` environment variable as part of your `docker run` command. -Private Docker registry ------------------------ - -Using a private docker registry requires the `DOCKER_AUTH_CONFIG` environment variable to be set. -`official documentation `_ - -The value of this variable should be a JSON string containing the authentication information for the registry. - -Example: - -.. code-block:: bash - - DOCKER_AUTH_CONFIG='{"auths": {"https://myregistry.com": {"auth": "dXNlcm5hbWU6cGFzc3dvcmQ="}}}' - -In order to generate the JSON string, you can use the following command: - -.. code-block:: bash - - echo -n '{"auths": {"": {"auth": "'$(echo -n ":" | base64 -w 0)'"}}}' - -Fetching passwords from cloud providers: - -.. code-block:: bash - - ECR_PASSWORD = $(aws ecr get-login-password --region eu-west-1) - GCP_PASSWORD = $(gcloud auth print-access-token) - AZURE_PASSWORD = $(az acr login --name --expose-token --output tsv) - - Configuration ------------- -+-------------------------------------------+---------------------------------------------------+------------------------------------------+ -| Env Variable | Example | Description | -+===========================================+===================================================+==========================================+ -| ``TESTCONTAINERS_DOCKER_SOCKET_OVERRIDE`` | ``/var/run/docker.sock`` | Path to Docker's socket used by ryuk | -+-------------------------------------------+---------------------------------------------------+------------------------------------------+ -| ``TESTCONTAINERS_RYUK_PRIVILEGED`` | ``false`` | Run ryuk as a privileged container | -+-------------------------------------------+---------------------------------------------------+------------------------------------------+ -| ``TESTCONTAINERS_RYUK_DISABLED`` | ``false`` | Disable ryuk | -+-------------------------------------------+---------------------------------------------------+------------------------------------------+ -| ``RYUK_CONTAINER_IMAGE`` | ``testcontainers/ryuk:0.7.0`` | Custom image for ryuk | -+-------------------------------------------+---------------------------------------------------+------------------------------------------+ -| ``DOCKER_AUTH_CONFIG`` | ``{"auths": {"": {"auth": ""}}}`` | Custom registry auth config | -+-------------------------------------------+---------------------------------------------------+------------------------------------------+ ++-------------------------------------------+-------------------------------+------------------------------------------+ +| Env Variable | Example | Description | ++===========================================+===============================+==========================================+ +| ``TESTCONTAINERS_DOCKER_SOCKET_OVERRIDE`` | ``/var/run/docker.sock`` | Path to Docker's socket used by ryuk | ++-------------------------------------------+-------------------------------+------------------------------------------+ +| ``TESTCONTAINERS_RYUK_PRIVILEGED`` | ``false`` | Run ryuk as a privileged container | ++-------------------------------------------+-------------------------------+------------------------------------------+ +| ``TESTCONTAINERS_RYUK_DISABLED`` | ``false`` | Disable ryuk | ++-------------------------------------------+-------------------------------+------------------------------------------+ +| ``RYUK_CONTAINER_IMAGE`` | ``testcontainers/ryuk:0.7.0`` | Custom image for ryuk | ++-------------------------------------------+-------------------------------+------------------------------------------+ Development and Contributing ---------------------------- -We recommend you use a `Poetry `_ for development. -After having installed `poetry`, you can run the following snippet to set up your local dev environment. +We recommend you use a `virtual environment `_ for development (:code:`python>=3.7` is required). After setting up your virtual environment, you can install all dependencies and test the installation by running the following snippet. .. code-block:: bash - make install + poetry install --all-extras + make /tests Package Structure ^^^^^^^^^^^^^^^^^ -Testcontainers is a collection of `implicit namespace packages `__ -to decouple the development of different extensions, -e.g., :code:`testcontainers[mysql]` and :code:`testcontainers[postgres]` for MySQL and PostgreSQL database containers, respectively. - -The folder structure is as follows: +Testcontainers is a collection of `implicit namespace packages `__ to decouple the development of different extensions, e.g., :code:`testcontainers-mysql` and :code:`testcontainers-postgres` for MySQL and PostgreSQL database containers, respectively. The folder structure is as follows. .. code-block:: bash @@ -172,11 +133,10 @@ The folder structure is as follows: ... # README for this feature. README.rst + # Setup script for this feature. + setup.py Contributing a New Feature ^^^^^^^^^^^^^^^^^^^^^^^^^^ -You want to contribute a new feature or container? Great! -- We recommend you first `open an issue `_ -- Then follow the suggestions from the team -- We also have a Pull Request `template `_ for new containers! +You want to contribute a new feature or container? Great! You can do that in six steps as outlined `here __`. diff --git a/modules/cosmosdb/README.rst b/modules/cosmosdb/README.rst new file mode 100644 index 000000000..802cffa4e --- /dev/null +++ b/modules/cosmosdb/README.rst @@ -0,0 +1,5 @@ +.. autoclass:: testcontainers.cosmosdb.CosmosDBMongoEndpointContainer +.. title:: testcontainers.cosmosdb.CosmosDBMongoEndpointContainer + +.. autoclass:: testcontainers.cosmosdb.CosmosDBNoSQLEndpointContainer +.. title:: testcontainers.cosmosdb.CosmosDBNoSQLEndpointContainer diff --git a/modules/cosmosdb/testcontainers/cosmosdb/__init__.py b/modules/cosmosdb/testcontainers/cosmosdb/__init__.py new file mode 100644 index 000000000..619ddb3b4 --- /dev/null +++ b/modules/cosmosdb/testcontainers/cosmosdb/__init__.py @@ -0,0 +1,4 @@ +from .mongodb import CosmosDBMongoEndpointContainer +from .nosql import CosmosDBNoSQLEndpointContainer + +__all__ = ["CosmosDBMongoEndpointContainer", "CosmosDBNoSQLEndpointContainer"] diff --git a/modules/cosmosdb/testcontainers/cosmosdb/_emulator.py b/modules/cosmosdb/testcontainers/cosmosdb/_emulator.py new file mode 100644 index 000000000..161a01c29 --- /dev/null +++ b/modules/cosmosdb/testcontainers/cosmosdb/_emulator.py @@ -0,0 +1,110 @@ +import os +import socket +import ssl +from collections.abc import Iterable +from distutils.util import strtobool +from urllib.error import HTTPError, URLError +from urllib.request import urlopen + +from typing_extensions import Self + +from testcontainers.core.container import DockerContainer +from testcontainers.core.waiting_utils import wait_container_is_ready, wait_for_logs + +from . import _grab as grab + +__all__ = ["CosmosDBEmulatorContainer"] + +EMULATOR_PORT = 8081 + + +class CosmosDBEmulatorContainer(DockerContainer): + """ + Abstract class for CosmosDB Emulator endpoints. + + Concrete implementations for each endpoint is provided by a separate class: + NoSQLEmulatorContainer and MongoDBEmulatorContainer. + """ + + def __init__( + self, + image: str = os.getenv( + "AZURE_COSMOS_EMULATOR_IMAGE", "mcr.microsoft.com/cosmosdb/linux/azure-cosmos-emulator:latest" + ), + partition_count: int = os.getenv("AZURE_COSMOS_EMULATOR_PARTITION_COUNT", None), + enable_data_persistence: bool = strtobool(os.getenv("AZURE_COSMOS_EMULATOR_ENABLE_DATA_PERSISTENCE", "false")), + key: str = os.getenv( + "AZURE_COSMOS_EMULATOR_KEY", + "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==", + ), + bind_ports: bool = strtobool(os.getenv("AZURE_COSMOS_EMULATOR_BIND_PORTS", "true")), + endpoint_ports: Iterable[int] = [], + **other_kwargs, + ): + super().__init__(image=image, **other_kwargs) + self.endpoint_ports = endpoint_ports + self.partition_count = partition_count + self.key = key + self.enable_data_persistence = enable_data_persistence + self.bind_ports = bind_ports + + @property + def host(self) -> str: + """ + Emulator host + """ + return self.get_container_host_ip() + + @property + def server_certificate_pem(self) -> bytes: + """ + PEM-encoded server certificate + """ + return self._cert_pem_bytes + + def start(self) -> Self: + self._configure() + super().start() + self._wait_until_ready() + self._cert_pem_bytes = self._download_cert() + return self + + def _configure(self) -> None: + all_ports = {EMULATOR_PORT, *self.endpoint_ports} + if self.bind_ports: + for port in all_ports: + self.with_bind_ports(port, port) + else: + self.with_exposed_ports(*all_ports) + + ( + self.with_env("AZURE_COSMOS_EMULATOR_PARTITION_COUNT", str(self.partition_count)) + .with_env("AZURE_COSMOS_EMULATOR_IP_ADDRESS_OVERRIDE", socket.gethostbyname(socket.gethostname())) + .with_env("AZURE_COSMOS_EMULATOR_ENABLE_DATA_PERSISTENCE", str(self.enable_data_persistence)) + .with_env("AZURE_COSMOS_EMULATOR_KEY", str(self.key)) + ) + + def _wait_until_ready(self) -> Self: + wait_for_logs(container=self, predicate="Started\\s*$") + + if self.bind_ports: + self._wait_for_url(f"https://{self.host}:{EMULATOR_PORT}/_explorer/index.html") + self._wait_for_query_success() + + return self + + def _download_cert(self) -> bytes: + with grab.file( + self.get_wrapped_container(), + "/tmp/cosmos/appdata/.system/profiles/Client/AppData/Local/CosmosDBEmulator/emulator.pem", + ) as cert: + return cert.read() + + @wait_container_is_ready(HTTPError, URLError) + def _wait_for_url(self, url: str) -> Self: + with urlopen(url, context=ssl._create_unverified_context()) as response: + response.read() + return self + + def _wait_for_query_success(self) -> None: + pass diff --git a/modules/cosmosdb/testcontainers/cosmosdb/_grab.py b/modules/cosmosdb/testcontainers/cosmosdb/_grab.py new file mode 100644 index 000000000..e1895019a --- /dev/null +++ b/modules/cosmosdb/testcontainers/cosmosdb/_grab.py @@ -0,0 +1,26 @@ +import tarfile +import tempfile +from contextlib import contextmanager +from os import path +from pathlib import Path + +from docker.models.containers import Container + + +@contextmanager +def file(container: Container, target: str): + target_path = Path(target) + assert target_path.is_absolute(), "target must be an absolute path" + + with tempfile.TemporaryDirectory() as tmp: + archive = Path(tmp) / "grabbed.tar" + + # download from container as tar archive + with open(archive, "wb") as f: + tar_bits, _ = container.get_archive(target) + for chunk in tar_bits: + f.write(chunk) + + # extract target file from tar archive + with tarfile.TarFile(archive) as tar: + yield tar.extractfile(path.basename(target)) diff --git a/modules/cosmosdb/testcontainers/cosmosdb/mongodb.py b/modules/cosmosdb/testcontainers/cosmosdb/mongodb.py new file mode 100644 index 000000000..82e8c096b --- /dev/null +++ b/modules/cosmosdb/testcontainers/cosmosdb/mongodb.py @@ -0,0 +1,47 @@ +import os + +from ._emulator import CosmosDBEmulatorContainer + +__all__ = ["CosmosDBMongoEndpointContainer"] + +ENDPOINT_PORT = 10255 + + +class CosmosDBMongoEndpointContainer(CosmosDBEmulatorContainer): + """ + CosmosDB MongoDB enpoint Emulator. + + Example: + + .. code-block:: python + + >>> from testcontainers.cosmosdb import CosmosDBMongoEndpointContainer + + >>> with CosmosDBMongoEndpointContainer(mongodb_version="4.0") as emulator: + ... print(f"Point your MongoDB client at {emulator.host}:{emulator.port} using key {emulator.key}") + ... print(f"and eiher disable TLS server auth or trust the server's self signed cert (emulator.server_certificate_pem)") + + """ + + def __init__( + self, + mongodb_version: str, + image: str = os.getenv( + "AZURE_COSMOS_EMULATOR_IMAGE", "mcr.microsoft.com/cosmosdb/linux/azure-cosmos-emulator:mongodb" + ), + **other_kwargs, + ): + super().__init__(image=image, endpoint_ports=[ENDPOINT_PORT], **other_kwargs) + assert mongodb_version is not None, "A MongoDB version is required to use the MongoDB Endpoint" + self.mongodb_version = mongodb_version + + @property + def port(self) -> str: + """ + The exposed port to the MongoDB endpoint + """ + return self.get_exposed_port(ENDPOINT_PORT) + + def _configure(self) -> None: + super()._configure() + self.with_env("AZURE_COSMOS_EMULATOR_ENABLE_MONGODB_ENDPOINT", self.mongodb_version) diff --git a/modules/cosmosdb/testcontainers/cosmosdb/nosql.py b/modules/cosmosdb/testcontainers/cosmosdb/nosql.py new file mode 100644 index 000000000..f78469674 --- /dev/null +++ b/modules/cosmosdb/testcontainers/cosmosdb/nosql.py @@ -0,0 +1,69 @@ +from azure.core.exceptions import ServiceRequestError +from azure.cosmos import CosmosClient as SyncCosmosClient +from azure.cosmos.aio import CosmosClient as AsyncCosmosClient + +from testcontainers.core.waiting_utils import wait_container_is_ready + +from ._emulator import CosmosDBEmulatorContainer + +__all__ = ["CosmosDBNoSQLEndpointContainer"] + +NOSQL_PORT = 8081 + + +class CosmosDBNoSQLEndpointContainer(CosmosDBEmulatorContainer): + """ + CosmosDB NoSQL enpoint Emulator. + + Example: + + .. code-block:: python + + >>> from testcontainers.cosmosdb import CosmosDBNoSQLEndpointContainer + >>> with CosmosDBNoSQLEndpointContainer() as emulator: + ... db = emulator.insecure_sync_client().create_database_if_not_exists("test") + + .. code-block:: python + + >>> from testcontainers.cosmosdb import CosmosDBNoSQLEndpointContainer + >>> from azure.cosmos import CosmosClient + + >>> with CosmosDBNoSQLEndpointContainer() as emulator: + ... client = CosmosClient(url=emulator.url, credential=emulator.key, connection_verify=False) + ... db = client.create_database_if_not_exists("test") + + """ + + def __init__(self, **kwargs): + super().__init__(endpoint_ports=[NOSQL_PORT], **kwargs) + + @property + def port(self) -> str: + """ + The exposed port to the NoSQL endpoint + """ + return self.get_exposed_port(NOSQL_PORT) + + @property + def url(self) -> str: + """ + The url to the NoSQL endpoint + """ + return f"https://{self.host}:{self.port}" + + def insecure_async_client(self): + """ + Returns an asynchronous CosmosClient instance + """ + return AsyncCosmosClient(url=self.url, credential=self.key, connection_verify=False) + + def insecure_sync_client(self): + """ + Returns a synchronous CosmosClient instance + """ + return SyncCosmosClient(url=self.url, credential=self.key, connection_verify=False) + + @wait_container_is_ready(ServiceRequestError) + def _wait_for_query_success(self) -> None: + with self.insecure_sync_client() as c: + list(c.list_databases()) diff --git a/modules/cosmosdb/tests/test_emulator.py b/modules/cosmosdb/tests/test_emulator.py new file mode 100644 index 000000000..542ddd11c --- /dev/null +++ b/modules/cosmosdb/tests/test_emulator.py @@ -0,0 +1,8 @@ +import pytest +from testcontainers.cosmosdb._emulator import CosmosDBEmulatorContainer + + +def test_runs(): + with CosmosDBEmulatorContainer(partition_count=1, bind_ports=False) as emulator: + assert emulator.server_certificate_pem is not None + assert emulator.get_exposed_port(8081) is not None diff --git a/modules/cosmosdb/tests/test_mongodb.py b/modules/cosmosdb/tests/test_mongodb.py new file mode 100644 index 000000000..a50ee82ea --- /dev/null +++ b/modules/cosmosdb/tests/test_mongodb.py @@ -0,0 +1,16 @@ +import pytest +from testcontainers.cosmosdb import CosmosDBMongoEndpointContainer + + +def test_requires_a_version(): + with pytest.raises(AssertionError, match="A MongoDB version is required"): + CosmosDBMongoEndpointContainer(mongodb_version=None) + + # instanciates + CosmosDBMongoEndpointContainer(mongodb_version="4.0") + + +def test_runs(): + with CosmosDBMongoEndpointContainer(mongodb_version="4.0", partition_count=1, bind_ports=False) as emulator: + assert emulator.env["AZURE_COSMOS_EMULATOR_ENABLE_MONGODB_ENDPOINT"] == "4.0" + assert emulator.get_exposed_port(10255) is not None, "The MongoDB endpoint's port should be exposed" diff --git a/modules/cosmosdb/tests/test_nosql.py b/modules/cosmosdb/tests/test_nosql.py new file mode 100644 index 000000000..a9460a1b0 --- /dev/null +++ b/modules/cosmosdb/tests/test_nosql.py @@ -0,0 +1,7 @@ +import pytest +from testcontainers.cosmosdb import CosmosDBNoSQLEndpointContainer + + +def test_runs(): + with CosmosDBNoSQLEndpointContainer(partition_count=1, bind_ports=False) as emulator: + assert emulator.get_exposed_port(8081) is not None, "The NoSQL endpoint's port should be exposed" diff --git a/poetry.lock b/poetry.lock index aa5fdc29b..90a83f33f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -175,6 +175,21 @@ typing-extensions = ">=4.6.0" [package.extras] aio = ["aiohttp (>=3.0)"] +[[package]] +name = "azure-cosmos" +version = "4.7.0" +description = "Microsoft Azure Cosmos Client Library for Python" +optional = true +python-versions = ">=3.8" +files = [ + {file = "azure-cosmos-4.7.0.tar.gz", hash = "sha256:72d714033134656302a2e8957c4b93590673bd288b0ca60cb123e348ae99a241"}, + {file = "azure_cosmos-4.7.0-py3-none-any.whl", hash = "sha256:03d8c7740ddc2906fb16e07b136acc0fe6a6a02656db46c5dd6f1b127b58cc96"}, +] + +[package.dependencies] +azure-core = ">=1.25.1" +typing-extensions = ">=4.6.0" + [[package]] name = "azure-storage-blob" version = "12.19.1" @@ -4462,6 +4477,7 @@ cassandra = [] chroma = ["chromadb-client"] clickhouse = ["clickhouse-driver"] cockroachdb = [] +cosmosdb = ["azure-cosmos"] elasticsearch = [] generic = ["httpx"] google = ["google-cloud-datastore", "google-cloud-pubsub"] @@ -4497,4 +4513,4 @@ weaviate = ["weaviate-client"] [metadata] lock-version = "2.0" python-versions = ">=3.9,<4.0" -content-hash = "e07f8edf8cefba872bbf48dcfa187163cefb00a60122daa62de8891b61fc55de" +content-hash = "2b87af7b69af2cc83f8198ab0fcfef7ceaf8411a8300c4ca72c0521e5d966445" diff --git a/pyproject.toml b/pyproject.toml index 0b6088954..c7a398d7b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ packages = [ { include = "testcontainers", from = "modules/chroma" }, { include = "testcontainers", from = "modules/clickhouse" }, { include = "testcontainers", from = "modules/cockroachdb" }, + { include = "testcontainers", from = "modules/cosmosdb" }, { include = "testcontainers", from = "modules/elasticsearch" }, { include = "testcontainers", from = "modules/generic" }, { include = "testcontainers", from = "modules/testmoduleimport"}, @@ -106,12 +107,14 @@ chromadb-client = { version = "*", optional = true } qdrant-client = { version = "*", optional = true } bcrypt = { version = "*", optional = true } httpx = { version = "*", optional = true } +azure-cosmos = { version = "*", optional = true } [tool.poetry.extras] arangodb = ["python-arango"] azurite = ["azure-storage-blob"] cassandra = [] clickhouse = ["clickhouse-driver"] +cosmosdb = ["azure-cosmos"] cockroachdb = [] elasticsearch = [] generic = ["httpx"]