From fc7c4669d9493e0ecd4b9021f3ef389280c72f18 Mon Sep 17 00:00:00 2001 From: Vemund Santi Date: Tue, 2 Apr 2024 14:43:11 +0200 Subject: [PATCH 01/20] Include configuration and waiting hooks in base DockerContainer class --- core/testcontainers/core/container.py | 11 +++++++++++ core/testcontainers/core/generic.py | 11 +---------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/core/testcontainers/core/container.py b/core/testcontainers/core/container.py index 3e1e1ba1..995ff29a 100644 --- a/core/testcontainers/core/container.py +++ b/core/testcontainers/core/container.py @@ -75,7 +75,14 @@ def maybe_emulate_amd64(self) -> "DockerContainer": return self.with_kwargs(platform="linux/amd64") return self + def _configure(self) -> None: + pass + + def _wait_until_ready(self) -> None: + pass + def start(self): + self._configure() if not RYUK_DISABLED and self.image != RYUK_IMAGE: logger.debug("Creating Ryuk container") Reaper.get_instance() @@ -92,6 +99,10 @@ def start(self): **self._kwargs, ) logger.info("Container started: %s", self._container.short_id) + + self._wait_until_ready() + logger.info("Container ready: %s", self._container.short_id) + return self def stop(self, force=True, delete_volume=True) -> None: diff --git a/core/testcontainers/core/generic.py b/core/testcontainers/core/generic.py index a3bff96e..b4ff8b62 100644 --- a/core/testcontainers/core/generic.py +++ b/core/testcontainers/core/generic.py @@ -32,7 +32,7 @@ class DbContainer(DockerContainer): """ @wait_container_is_ready(*ADDITIONAL_TRANSIENT_ERRORS) - def _connect(self) -> None: + def _wait_until_ready(self): import sqlalchemy engine = sqlalchemy.create_engine(self.get_connection_url()) @@ -64,12 +64,3 @@ def _create_connection_url( if dbname: url = f"{url}/{dbname}" return url - - def start(self) -> "DbContainer": - self._configure() - super().start() - self._connect() - return self - - def _configure(self) -> None: - raise NotImplementedError From 2578a97ec903140109980e5ea99996bc9101b231 Mon Sep 17 00:00:00 2001 From: Vemund Santi Date: Tue, 2 Apr 2024 14:43:42 +0200 Subject: [PATCH 02/20] Inherit directly from DockerContainer in ArangoDB module --- .../testcontainers/arangodb/__init__.py | 22 +++++++++++-------- modules/arangodb/tests/test_arangodb.py | 13 ++++------- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/modules/arangodb/testcontainers/arangodb/__init__.py b/modules/arangodb/testcontainers/arangodb/__init__.py index a7c95465..896c3579 100644 --- a/modules/arangodb/testcontainers/arangodb/__init__.py +++ b/modules/arangodb/testcontainers/arangodb/__init__.py @@ -2,16 +2,18 @@ ArangoDB container support. """ -import typing from os import environ +from typing import Optional + +from typing_extensions import override from testcontainers.core.config import TIMEOUT -from testcontainers.core.generic import DbContainer +from testcontainers.core.container import DockerContainer from testcontainers.core.utils import raise_for_deprecated_parameter from testcontainers.core.waiting_utils import wait_for_logs -class ArangoDbContainer(DbContainer): +class ArangoDbContainer(DockerContainer): """ ArangoDB container. @@ -26,7 +28,7 @@ class ArangoDbContainer(DbContainer): >>> from testcontainers.arangodb import ArangoDbContainer >>> from arango import ArangoClient - >>> with ArangoDbContainer("arangodb:3.11.8") as arango: + >>> with ArangoDbContainer("arangodb:3.12.0") as arango: ... client = ArangoClient(hosts=arango.get_connection_url()) ... ... # Connect @@ -42,8 +44,8 @@ def __init__( image: str = "arangodb:latest", port: int = 8529, arango_root_password: str = "passwd", - arango_no_auth: typing.Optional[bool] = None, - arango_random_root_password: typing.Optional[bool] = None, + arango_no_auth: Optional[bool] = None, + arango_random_root_password: Optional[bool] = None, **kwargs, ) -> None: """ @@ -78,6 +80,7 @@ def __init__( ) ) + @override def _configure(self) -> None: self.with_env("ARANGO_ROOT_PASSWORD", self.arango_root_password) if self.arango_no_auth: @@ -85,9 +88,10 @@ def _configure(self) -> None: if self.arango_random_root_password: self.with_env("ARANGO_RANDOM_ROOT_PASSWORD", "1") + @override + def _wait_until_ready(self) -> None: + wait_for_logs(self, predicate="is ready for business", timeout=TIMEOUT) + def get_connection_url(self) -> str: port = self.get_exposed_port(self.port) return f"http://{self.get_container_host_ip()}:{port}" - - def _connect(self) -> None: - wait_for_logs(self, predicate="is ready for business", timeout=TIMEOUT) diff --git a/modules/arangodb/tests/test_arangodb.py b/modules/arangodb/tests/test_arangodb.py index f526bf38..5d66d06a 100644 --- a/modules/arangodb/tests/test_arangodb.py +++ b/modules/arangodb/tests/test_arangodb.py @@ -48,14 +48,9 @@ def arango_test_ops(arango_client, expeced_version, username="root", password="" assert len(student_names) == students_to_insert_cnt -def test_docker_run_arango(): - """ - Test ArangoDB container with default settings. - """ - image = f"{ARANGODB_IMAGE_NAME}:{IMAGE_VERSION}" - arango_root_password = "passwd" - - with ArangoDbContainer(image) as arango: +@pytest.mark.parametrize("version", ["3.12.0", "3.11.8", "3.10.13"]) +def test_docker_run_arango(version: str): + with ArangoDbContainer(f"arangodb:{version}") as arango: client = ArangoClient(hosts=arango.get_connection_url()) # Test invalid auth @@ -63,7 +58,7 @@ def test_docker_run_arango(): with pytest.raises(DatabaseCreateError): sys_db.create_database("test") - arango_test_ops(arango_client=client, expeced_version=IMAGE_VERSION, password=arango_root_password) + arango_test_ops(arango_client=client, expeced_version=version, password="passwd") def test_docker_run_arango_without_auth(): From fa0d34a35af7b87dff32a7790972ff7030d8b1a5 Mon Sep 17 00:00:00 2001 From: Vemund Santi Date: Tue, 2 Apr 2024 14:54:38 +0200 Subject: [PATCH 03/20] Use new _wait_until_ready hook in Azureite module --- modules/azurite/testcontainers/azurite/__init__.py | 12 +++--------- modules/azurite/tests/test_azurite.py | 4 ++-- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/modules/azurite/testcontainers/azurite/__init__.py b/modules/azurite/testcontainers/azurite/__init__.py index 969fcf35..6e10e86b 100644 --- a/modules/azurite/testcontainers/azurite/__init__.py +++ b/modules/azurite/testcontainers/azurite/__init__.py @@ -33,10 +33,9 @@ class AzuriteContainer(DockerContainer): >>> from testcontainers.azurite import AzuriteContainer >>> from azure.storage.blob import BlobServiceClient - >>> with AzuriteContainer() as azurite_container: - ... connection_string = azurite_container.get_connection_string() + >>> with AzuriteContainer("mcr.microsoft.com/azure-storage/azurite:3.29.0") as azurite_container: ... client = BlobServiceClient.from_connection_string( - ... connection_string, + ... azurite_container.get_connection_string(), ... api_version="2019-12-12" ... ) """ @@ -102,12 +101,7 @@ def get_connection_string(self) -> str: return connection_string - def start(self) -> "AzuriteContainer": - super().start() - self._connect() - return self - @wait_container_is_ready(OSError) - def _connect(self) -> None: + def _wait_until_ready(self) -> None: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: s.connect((self.get_container_host_ip(), int(self.get_exposed_port(next(iter(self.ports)))))) diff --git a/modules/azurite/tests/test_azurite.py b/modules/azurite/tests/test_azurite.py index 74230ab1..a1527b3f 100644 --- a/modules/azurite/tests/test_azurite.py +++ b/modules/azurite/tests/test_azurite.py @@ -4,9 +4,9 @@ def test_docker_run_azurite(): - with AzuriteContainer() as azurite_container: + with AzuriteContainer("mcr.microsoft.com/azure-storage/azurite:3.29.0") as azurite_container: blob_service_client = BlobServiceClient.from_connection_string( - azurite_container.get_connection_string(), api_version="2019-12-12" + azurite_container.get_connection_string(), api_version="2023-11-03" ) blob_service_client.create_container("test-container") From 482df602624ffa1c9c6cfc616d2485ddbd3f2bb4 Mon Sep 17 00:00:00 2001 From: Vemund Santi Date: Tue, 2 Apr 2024 14:59:21 +0200 Subject: [PATCH 04/20] Use new _wait_until_ready hook in Cassandra module --- modules/cassandra/testcontainers/cassandra/__init__.py | 10 ++++------ modules/cassandra/tests/test_cassandra.py | 9 ++++++--- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/modules/cassandra/testcontainers/cassandra/__init__.py b/modules/cassandra/testcontainers/cassandra/__init__.py index 4e6618b7..b57f4e18 100644 --- a/modules/cassandra/testcontainers/cassandra/__init__.py +++ b/modules/cassandra/testcontainers/cassandra/__init__.py @@ -10,6 +10,8 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. +from typing_extensions import override + from testcontainers.core.container import DockerContainer from testcontainers.core.waiting_utils import wait_for_logs @@ -47,14 +49,10 @@ def __init__(self, image: str = "cassandra:latest", **kwargs) -> None: self.with_env("CASSANDRA_ENDPOINT_SNITCH", "GossipingPropertyFileSnitch") self.with_env("CASSANDRA_DC", self.DEFAULT_LOCAL_DATACENTER) - def _connect(self): + @override + def _wait_until_ready(self): wait_for_logs(self, "Startup complete") - def start(self) -> "CassandraContainer": - super().start() - self._connect() - return self - def get_contact_points(self) -> list[tuple[str, int]]: return [(self.get_container_host_ip(), int(self.get_exposed_port(self.CQL_PORT)))] diff --git a/modules/cassandra/tests/test_cassandra.py b/modules/cassandra/tests/test_cassandra.py index 1aa5858b..b7b9d8dd 100644 --- a/modules/cassandra/tests/test_cassandra.py +++ b/modules/cassandra/tests/test_cassandra.py @@ -1,14 +1,17 @@ +import pytest + from cassandra.cluster import Cluster, DCAwareRoundRobinPolicy from testcontainers.cassandra import CassandraContainer -def test_docker_run_cassandra(): - with CassandraContainer("cassandra:4.1.4") as cassandra: +@pytest.mark.parametrize("version", ["4.1.4", "3.11.16"]) +def test_docker_run_cassandra(version: str): + with CassandraContainer(f"cassandra:{version}") as cassandra: cluster = Cluster( cassandra.get_contact_points(), load_balancing_policy=DCAwareRoundRobinPolicy(cassandra.get_local_datacenter()), ) session = cluster.connect() result = session.execute("SELECT release_version FROM system.local;") - assert result.one().release_version == "4.1.4" + assert result.one().release_version == version From 9437d893bb19ba50ec2667f50f7203db1c2b4e81 Mon Sep 17 00:00:00 2001 From: Vemund Santi Date: Tue, 2 Apr 2024 16:09:25 +0200 Subject: [PATCH 05/20] Add http waiting strategy for basic HTTP endpoint status codes --- core/testcontainers/core/waiting_utils.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/core/testcontainers/core/waiting_utils.py b/core/testcontainers/core/waiting_utils.py index ea52683d..2859b569 100644 --- a/core/testcontainers/core/waiting_utils.py +++ b/core/testcontainers/core/waiting_utils.py @@ -12,10 +12,13 @@ # under the License. +import contextlib import re import time import traceback +from http.client import HTTPException from typing import TYPE_CHECKING, Any, Callable, Optional, Union +from urllib.request import urlopen import wrapt @@ -77,6 +80,18 @@ def wait_for(condition: Callable[..., bool]) -> bool: return condition() +def wait_for_http(url: str, timeout: Optional[float] = config.TIMEOUT, interval: float = 1) -> float: + start = time.time() + while True: + duration = time.time() - start + with contextlib.suppress(HTTPException), urlopen(url) as r: + if r.status < 300: + return duration + if timeout and duration > timeout: + raise TimeoutError(f"Container did not respond to URL check {url} in {timeout:.3f} seconds") + time.sleep(interval) + + def wait_for_logs( container: "DockerContainer", predicate: Union[Callable, str], timeout: Optional[float] = None, interval: float = 1 ) -> float: From d4a06120122ffb6e7a4e2dfb42f178b20f5292e9 Mon Sep 17 00:00:00 2001 From: Vemund Santi Date: Tue, 2 Apr 2024 16:09:38 +0200 Subject: [PATCH 06/20] Use new _wait_until_ready hook in ChromaDB module --- .../chroma/testcontainers/chroma/__init__.py | 29 ++++--------------- modules/chroma/tests/test_chroma.py | 4 +-- 2 files changed, 8 insertions(+), 25 deletions(-) diff --git a/modules/chroma/testcontainers/chroma/__init__.py b/modules/chroma/testcontainers/chroma/__init__.py index 9e574409..09bf3b4c 100644 --- a/modules/chroma/testcontainers/chroma/__init__.py +++ b/modules/chroma/testcontainers/chroma/__init__.py @@ -1,13 +1,8 @@ -from typing import TYPE_CHECKING - -from requests import ConnectionError, get +from typing_extensions import override from testcontainers.core.container import DockerContainer from testcontainers.core.utils import raise_for_deprecated_parameter -from testcontainers.core.waiting_utils import wait_container_is_ready - -if TYPE_CHECKING: - from requests import Response +from testcontainers.core.waiting_utils import wait_for_http class ChromaContainer(DockerContainer): @@ -22,7 +17,7 @@ class ChromaContainer(DockerContainer): >>> import chromadb >>> from testcontainers.chroma import ChromaContainer - >>> with ChromaContainer() as chroma: + >>> with ChromaContainer("chromadb/chroma:0.4.24") as chroma: ... config = chroma.get_config() ... client = chromadb.HttpClient(host=config["host"], port=config["port"]) ... col = client.get_or_create_collection("test") @@ -48,7 +43,6 @@ def __init__( self.port = port self.with_exposed_ports(self.port) - # self.with_command(f"server /data --address :{self.port}") def get_config(self) -> dict: """This method returns the configuration of the Chroma container, @@ -65,17 +59,6 @@ def get_config(self) -> dict: "port": exposed_port, } - @wait_container_is_ready(ConnectionError) - def _healthcheck(self) -> None: - """This is an internal method used to check if the Chroma container - is healthy and ready to receive requests.""" - url = f"http://{self.get_config()['endpoint']}/api/v1/heartbeat" - response: Response = get(url) - response.raise_for_status() - - def start(self) -> "ChromaContainer": - """This method starts the Chroma container and runs the healthcheck - to verify that the container is ready to use.""" - super().start() - self._healthcheck() - return self + @override + def _wait_until_ready(self) -> None: + wait_for_http(f"http://{self.get_config()['endpoint']}/api/v1/heartbeat") diff --git a/modules/chroma/tests/test_chroma.py b/modules/chroma/tests/test_chroma.py index fee55b78..0d8c8785 100644 --- a/modules/chroma/tests/test_chroma.py +++ b/modules/chroma/tests/test_chroma.py @@ -5,5 +5,5 @@ def test_docker_run_chroma(): with ChromaContainer(image="chromadb/chroma:0.4.24") as chroma: client = chromadb.HttpClient(host=chroma.get_config()["host"], port=chroma.get_config()["port"]) - col = client.get_or_create_collection("test") - assert col.name == "test" + collection = client.get_or_create_collection("test") + assert collection.name == "test" From b7e52ba87e1b7c08cd2fe107a73e36f1e93a1ef2 Mon Sep 17 00:00:00 2001 From: Vemund Santi Date: Wed, 3 Apr 2024 10:18:01 +0200 Subject: [PATCH 07/20] Create utility for generating db client compatible connection strings --- core/testcontainers/core/utils.py | 35 +++++++++++++++++++ .../testcontainers/arangodb/__init__.py | 9 +++-- .../testcontainers/clickhouse/__init__.py | 8 ++--- modules/clickhouse/tests/test_clickhouse.py | 9 ++--- 4 files changed, 50 insertions(+), 11 deletions(-) diff --git a/core/testcontainers/core/utils.py b/core/testcontainers/core/utils.py index 5ca1c2f7..88637ddd 100644 --- a/core/testcontainers/core/utils.py +++ b/core/testcontainers/core/utils.py @@ -3,6 +3,7 @@ import platform import subprocess import sys +from typing import Optional, Union LINUX = "linux" MAC = "mac" @@ -53,6 +54,40 @@ def inside_container() -> bool: return os.path.exists("/.dockerenv") +def create_connection_string( + dialect: str, + host: str, + port: Union[str, int, None] = None, + username: Optional[str] = None, + password: Optional[str] = None, + driver: Optional[str] = None, + dbname: Optional[str] = None, +) -> str: + """ + Returns a connection URL following the RFC-1738 format. + Compatible with database clients such as SQLAlchemy and other popular database client libraries. + + Example: postgres+psycopg2://myuser:mypassword@localhost:5432/mytestdb + """ + dialect_driver = dialect + if driver: + dialect_driver += f"+{driver}" + + username_password = username if username else "" + if password: + username_password += f":{password}" + + host_port = host + if port: + host_port += f":{port}" + + connection_string = f"{dialect_driver}://{username_password}@{host_port}" + if dbname: + connection_string += f"/{dbname}" + + return connection_string + + def default_gateway_ip() -> str: """ Returns gateway IP address of the host that testcontainer process is diff --git a/modules/arangodb/testcontainers/arangodb/__init__.py b/modules/arangodb/testcontainers/arangodb/__init__.py index 896c3579..a0dc2a52 100644 --- a/modules/arangodb/testcontainers/arangodb/__init__.py +++ b/modules/arangodb/testcontainers/arangodb/__init__.py @@ -9,7 +9,7 @@ from testcontainers.core.config import TIMEOUT from testcontainers.core.container import DockerContainer -from testcontainers.core.utils import raise_for_deprecated_parameter +from testcontainers.core.utils import create_connection_string, raise_for_deprecated_parameter from testcontainers.core.waiting_utils import wait_for_logs @@ -93,5 +93,8 @@ def _wait_until_ready(self) -> None: wait_for_logs(self, predicate="is ready for business", timeout=TIMEOUT) def get_connection_url(self) -> str: - port = self.get_exposed_port(self.port) - return f"http://{self.get_container_host_ip()}:{port}" + return create_connection_string( + dialect="http", + host=self.get_container_host_ip(), + port=self.get_exposed_port(self.port), + ) diff --git a/modules/clickhouse/testcontainers/clickhouse/__init__.py b/modules/clickhouse/testcontainers/clickhouse/__init__.py index fbc8fab6..5090afdd 100644 --- a/modules/clickhouse/testcontainers/clickhouse/__init__.py +++ b/modules/clickhouse/testcontainers/clickhouse/__init__.py @@ -70,12 +70,12 @@ def _configure(self) -> None: self.with_env("CLICKHOUSE_PASSWORD", self.password) self.with_env("CLICKHOUSE_DB", self.dbname) - def get_connection_url(self, host: Optional[str] = None) -> str: - return self._create_connection_url( + def get_connection_url(self) -> str: + return create_connection_string( dialect="clickhouse", username=self.username, password=self.password, + host=self.get_container_host_ip(), + port=self.get_exposed_port(8123), dbname=self.dbname, - host=host, - port=self.port, ) diff --git a/modules/clickhouse/tests/test_clickhouse.py b/modules/clickhouse/tests/test_clickhouse.py index 23e5f468..f0dc2e97 100644 --- a/modules/clickhouse/tests/test_clickhouse.py +++ b/modules/clickhouse/tests/test_clickhouse.py @@ -1,12 +1,13 @@ -import clickhouse_driver +import pytest + +from clickhouse_driver import Client from testcontainers.clickhouse import ClickHouseContainer def test_docker_run_clickhouse(): - clickhouse_container = ClickHouseContainer() - with clickhouse_container as clickhouse: - client = clickhouse_driver.Client.from_url(clickhouse.get_connection_url()) + with ClickHouseContainer(f"clickhouse/clickhouse-server:24.3.1-alpine") as clickhouse: + client = Client.from_url(clickhouse.get_connection_url()) result = client.execute("select 'working'") assert result == [("working",)] From 6964d3e33329c6bbabb6585a5ffe5dc83331e6e8 Mon Sep 17 00:00:00 2001 From: Vemund Santi Date: Wed, 3 Apr 2024 10:19:33 +0200 Subject: [PATCH 08/20] Use new _wait_until_ready hook in clickhouse module --- .../testcontainers/clickhouse/__init__.py | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/modules/clickhouse/testcontainers/clickhouse/__init__.py b/modules/clickhouse/testcontainers/clickhouse/__init__.py index 5090afdd..620ec35d 100644 --- a/modules/clickhouse/testcontainers/clickhouse/__init__.py +++ b/modules/clickhouse/testcontainers/clickhouse/__init__.py @@ -12,15 +12,17 @@ # under the License. import os from typing import Optional -from urllib.error import HTTPError, URLError +from urllib.error import URLError from urllib.request import urlopen -from testcontainers.core.generic import DbContainer -from testcontainers.core.utils import raise_for_deprecated_parameter +from typing_extensions import override + +from testcontainers.core.container import DockerContainer +from testcontainers.core.utils import create_connection_string, raise_for_deprecated_parameter from testcontainers.core.waiting_utils import wait_container_is_ready -class ClickHouseContainer(DbContainer): +class ClickHouseContainer(DockerContainer): """ ClickHouse database container. @@ -31,11 +33,11 @@ class ClickHouseContainer(DbContainer): .. doctest:: - >>> import clickhouse_driver + >>> from clickhouse_driver import Client >>> from testcontainers.clickhouse import ClickHouseContainer - >>> with ClickHouseContainer("clickhouse/clickhouse-server:21.8") as clickhouse: - ... client = clickhouse_driver.Client.from_url(clickhouse.get_connection_url()) + >>> with ClickHouseContainer("clickhouse/clickhouse-server:24.3.1") as clickhouse: + ... client = Client.from_url(clickhouse.get_connection_url()) ... client.execute("select 'working'") [('working',)] """ @@ -58,13 +60,13 @@ def __init__( self.with_exposed_ports(self.port) self.with_exposed_ports(8123) - @wait_container_is_ready(HTTPError, URLError) - def _connect(self) -> None: - # noinspection HttpUrlsUsage - url = f"http://{self.get_container_host_ip()}:{self.get_exposed_port(8123)}" - with urlopen(url) as r: + @override + @wait_container_is_ready(URLError) + def _wait_until_ready(self) -> None: + with urlopen(f"http://{self.get_container_host_ip()}:{self.get_exposed_port(8123)}") as r: assert b"Ok" in r.read() + @override def _configure(self) -> None: self.with_env("CLICKHOUSE_USER", self.username) self.with_env("CLICKHOUSE_PASSWORD", self.password) From 9e43875cecd3b86a238bc698274a161d7f8de8dc Mon Sep 17 00:00:00 2001 From: Vemund Santi Date: Wed, 3 Apr 2024 10:24:15 +0200 Subject: [PATCH 09/20] Use new _wait_until_ready hook in elasticsearch module --- .../testcontainers/elasticsearch/__init__.py | 31 ++++++++----------- 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/modules/elasticsearch/testcontainers/elasticsearch/__init__.py b/modules/elasticsearch/testcontainers/elasticsearch/__init__.py index 1b943916..1b3d2d53 100644 --- a/modules/elasticsearch/testcontainers/elasticsearch/__init__.py +++ b/modules/elasticsearch/testcontainers/elasticsearch/__init__.py @@ -12,12 +12,12 @@ # under the License. import logging import re -import urllib -from urllib.error import URLError + +from typing_extensions import override from testcontainers.core.container import DockerContainer -from testcontainers.core.utils import raise_for_deprecated_parameter -from testcontainers.core.waiting_utils import wait_container_is_ready +from testcontainers.core.utils import create_connection_string, raise_for_deprecated_parameter +from testcontainers.core.waiting_utils import wait_for_http _FALLBACK_VERSION = 8 """This version is used when no version could be detected from the image name.""" @@ -68,7 +68,7 @@ class ElasticSearchContainer(DockerContainer): >>> import urllib >>> from testcontainers.elasticsearch import ElasticSearchContainer - >>> with ElasticSearchContainer(f'elasticsearch:8.3.3', mem_limit='3G') as es: + >>> with ElasticSearchContainer(f'elasticsearch:8.12.2', mem_limit='3G') as es: ... resp = urllib.request.urlopen(es.get_url()) ... json.loads(resp.read().decode())['version']['number'] '8.3.3' @@ -86,18 +86,13 @@ def __init__(self, image: str = "elasticsearch", port: int = 9200, **kwargs) -> for key, value in _environment_by_version(major_version).items(): self.with_env(key, value) - @wait_container_is_ready(URLError) - def _connect(self) -> None: - res = urllib.request.urlopen(self.get_url()) - if res.status != 200: - raise Exception() + @override + def _wait_until_ready(self) -> None: + wait_for_http(self.get_url()) def get_url(self) -> str: - host = self.get_container_host_ip() - port = self.get_exposed_port(self.port) - return f"http://{host}:{port}" - - def start(self) -> "ElasticSearchContainer": - super().start() - self._connect() - return self + return create_connection_string( + dialect="http", + host=self.get_container_host_ip(), + port=self.get_exposed_port(self.port), + ) From 1b3aebc6c89e370dfbe07b83971afa6ea1d848d5 Mon Sep 17 00:00:00 2001 From: Vemund Santi Date: Wed, 3 Apr 2024 10:35:41 +0200 Subject: [PATCH 10/20] Use new _wait_until_ready hook in google module --- modules/google/testcontainers/google/__init__.py | 6 ++++-- modules/google/testcontainers/google/datastore.py | 10 +++++++--- modules/google/testcontainers/google/pubsub.py | 3 +-- modules/google/tests/test_google.py | 9 ++++----- 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/modules/google/testcontainers/google/__init__.py b/modules/google/testcontainers/google/__init__.py index 92c782ef..0e7097a1 100644 --- a/modules/google/testcontainers/google/__init__.py +++ b/modules/google/testcontainers/google/__init__.py @@ -1,2 +1,4 @@ -from .datastore import DatastoreContainer # noqa: F401 -from .pubsub import PubSubContainer # noqa: F401 +from .datastore import DatastoreContainer +from .pubsub import PubSubContainer + +__all__ = ["DatastoreContainer", "PubSubContainer"] diff --git a/modules/google/testcontainers/google/datastore.py b/modules/google/testcontainers/google/datastore.py index 24edbdcd..6d312a50 100644 --- a/modules/google/testcontainers/google/datastore.py +++ b/modules/google/testcontainers/google/datastore.py @@ -13,6 +13,8 @@ import os from unittest.mock import patch +from typing_extensions import override + from google.cloud import datastore from testcontainers.core.container import DockerContainer from testcontainers.core.waiting_utils import wait_for_logs @@ -32,8 +34,7 @@ class DatastoreContainer(DockerContainer): >>> from testcontainers.google import DatastoreContainer - >>> config = DatastoreContainer() - >>> with config as datastore: + >>> with DatastoreContainer("google/cloud-sdk:471.0.0-emulators") as datastore: ... datastore_client = datastore.get_datastore_client() """ @@ -56,7 +57,6 @@ def get_datastore_emulator_host(self) -> str: return f"{self.get_container_host_ip()}:{self.get_exposed_port(self.port)}" def get_datastore_client(self, **kwargs) -> datastore.Client: - wait_for_logs(self, "Dev App Server is now running.", timeout=30.0) env_vars = { "DATASTORE_DATASET": self.project, "DATASTORE_EMULATOR_HOST": self.get_datastore_emulator_host(), @@ -66,3 +66,7 @@ def get_datastore_client(self, **kwargs) -> datastore.Client: } with patch.dict(os.environ, env_vars): return datastore.Client(**kwargs) + + @override + def _wait_until_ready(self) -> None: + wait_for_logs(self, "Dev App Server is now running.", timeout=30.0) diff --git a/modules/google/testcontainers/google/pubsub.py b/modules/google/testcontainers/google/pubsub.py index 70603059..6f3eaab3 100644 --- a/modules/google/testcontainers/google/pubsub.py +++ b/modules/google/testcontainers/google/pubsub.py @@ -32,8 +32,7 @@ class PubSubContainer(DockerContainer): >>> from testcontainers.google import PubSubContainer - >>> config = PubSubContainer() - >>> with config as pubsub: + >>> with PubSubContainer("google/cloud-sdk:471.0.0-emulators") as pubsub: ... publisher = pubsub.get_publisher_client() ... topic_path = publisher.topic_path(pubsub.project, "my-topic") ... topic = publisher.create_topic(name=topic_path) diff --git a/modules/google/tests/test_google.py b/modules/google/tests/test_google.py index 0c412d70..b7705762 100644 --- a/modules/google/tests/test_google.py +++ b/modules/google/tests/test_google.py @@ -6,8 +6,7 @@ def test_pubsub_container(): - pubsub: PubSubContainer - with PubSubContainer() as pubsub: + with PubSubContainer("google/cloud-sdk:471.0.0-emulators") as pubsub: wait_for_logs(pubsub, r"Server started, listening on \d+", timeout=60) # Create a new topic publisher = pubsub.get_publisher_client() @@ -32,7 +31,7 @@ def test_pubsub_container(): def test_datastore_container_creation(): # Initialize the Datastore emulator container - with DatastoreContainer() as datastore: + with DatastoreContainer("google/cloud-sdk:471.0.0-emulators") as datastore: # Obtain a datastore client configured to connect to the emulator client = datastore.get_datastore_client() @@ -54,7 +53,7 @@ def test_datastore_container_creation(): def test_datastore_container_isolation(): # Initialize the Datastore emulator container - with DatastoreContainer() as datastore: + with DatastoreContainer("google/cloud-sdk:471.0.0-emulators") as datastore: # Obtain a datastore client configured to connect to the emulator client = datastore.get_datastore_client() @@ -67,7 +66,7 @@ def test_datastore_container_isolation(): client.put(entity) # Create a second container and try to fetch the entity to makesure its a different container - with DatastoreContainer() as datastore2: + with DatastoreContainer("google/cloud-sdk:471.0.0-emulators") as datastore2: assert ( datastore.get_datastore_emulator_host() != datastore2.get_datastore_emulator_host() ), "Datastore containers use the same port." From abbbf7de3766c94bbcf26b934e42644513e68164 Mon Sep 17 00:00:00 2001 From: Vemund Santi Date: Wed, 3 Apr 2024 11:56:39 +0200 Subject: [PATCH 11/20] Fix case for connection_string without username or password --- core/testcontainers/core/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/testcontainers/core/utils.py b/core/testcontainers/core/utils.py index 88637ddd..715a67c3 100644 --- a/core/testcontainers/core/utils.py +++ b/core/testcontainers/core/utils.py @@ -77,11 +77,14 @@ def create_connection_string( if password: username_password += f":{password}" + if username_password: + username_password += "@" + host_port = host if port: host_port += f":{port}" - connection_string = f"{dialect_driver}://{username_password}@{host_port}" + connection_string = f"{dialect_driver}://{username_password}{host_port}" if dbname: connection_string += f"/{dbname}" From 09c10ba012a03a9b755bca9060a36b070084afc5 Mon Sep 17 00:00:00 2001 From: Vemund Santi Date: Wed, 3 Apr 2024 11:57:32 +0200 Subject: [PATCH 12/20] Use new _wait_until_ready hook in influxdb module --- modules/influxdb/testcontainers/influxdb.py | 48 ++++--------------- .../testcontainers/influxdb1/__init__.py | 8 +--- .../testcontainers/influxdb2/__init__.py | 6 --- modules/influxdb/tests/__init__.py | 0 modules/influxdb/tests/test_influxdb.py | 21 ++------ 5 files changed, 14 insertions(+), 69 deletions(-) delete mode 100644 modules/influxdb/tests/__init__.py diff --git a/modules/influxdb/testcontainers/influxdb.py b/modules/influxdb/testcontainers/influxdb.py index 4b9d9b90..017b7516 100644 --- a/modules/influxdb/testcontainers/influxdb.py +++ b/modules/influxdb/testcontainers/influxdb.py @@ -28,11 +28,11 @@ """ from typing import Optional -from requests import get -from requests.exceptions import ConnectionError, ReadTimeout +from typing_extensions import override from testcontainers.core.container import DockerContainer -from testcontainers.core.waiting_utils import wait_container_is_ready +from testcontainers.core.utils import create_connection_string +from testcontainers.core.waiting_utils import wait_for_http class InfluxDbContainer(DockerContainer): @@ -60,40 +60,10 @@ def __init__( self.with_bind_ports(self.container_port, self.host_port) def get_url(self) -> str: - """ - Returns the url to interact with the InfluxDB container (health check, REST API, etc.) - """ - host = self.get_container_host_ip() - port = self.get_exposed_port(self.container_port) + return create_connection_string( + dialect="http", host=self.get_container_host_ip(), port=self.get_exposed_port(self.container_port) + ) - return f"http://{host}:{port}" - - @wait_container_is_ready(ConnectionError, ReadTimeout) - def _health_check(self) -> dict: - """ - Performs a health check on the running InfluxDB container. - The call is retried until it works thanks to the @wait_container_is_ready decorator. - See its documentation for the max number of retries or the timeout. - """ - - url = self.get_url() - response = get(f"{url}/health", timeout=1) - response.raise_for_status() - - return response.json() - - def get_influxdb_version(self) -> str: - """ - Returns the version of the InfluxDB service, as returned by the healthcheck. - """ - - return self._health_check().get("version") - - def start(self) -> "InfluxDbContainer": - """ - Spawns a container of the InfluxDB Docker image, ready to be used. - """ - super().start() - self._health_check() - - return self + @override + def _wait_until_ready(self) -> None: + wait_for_http(f"{self.get_url()}/health") diff --git a/modules/influxdb/testcontainers/influxdb1/__init__.py b/modules/influxdb/testcontainers/influxdb1/__init__.py index 81f21c16..270f906a 100644 --- a/modules/influxdb/testcontainers/influxdb1/__init__.py +++ b/modules/influxdb/testcontainers/influxdb1/__init__.py @@ -29,7 +29,7 @@ class InfluxDb1Container(InfluxDbContainer): >>> from testcontainers.influxdb1 import InfluxDbContainer - >>> with InfluxDbContainer() as influxdb: + >>> with InfluxDbContainer(influxdb:1.8-alpine) as influxdb: ... version = influxdb.get_version() """ @@ -57,9 +57,3 @@ def get_client(self, **client_kwargs): """ return InfluxDBClient(self.get_container_host_ip(), self.get_exposed_port(self.container_port), **client_kwargs) - - def start(self) -> "InfluxDb1Container": - """ - Overridden for better typing reason - """ - return super().start() diff --git a/modules/influxdb/testcontainers/influxdb2/__init__.py b/modules/influxdb/testcontainers/influxdb2/__init__.py index bd33d7fc..af39efb8 100644 --- a/modules/influxdb/testcontainers/influxdb2/__init__.py +++ b/modules/influxdb/testcontainers/influxdb2/__init__.py @@ -70,12 +70,6 @@ def __init__( if env_value: self.with_env(env_key, env_value) - def start(self) -> "InfluxDb2Container": - """ - Overridden for better typing reason - """ - return super().start() - def get_client( self, token: Optional[str] = None, org_name: Optional[str] = None, **influxdb_client_kwargs ) -> tuple[InfluxDBClient, Organization]: diff --git a/modules/influxdb/tests/__init__.py b/modules/influxdb/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/modules/influxdb/tests/test_influxdb.py b/modules/influxdb/tests/test_influxdb.py index 62144a3c..2c967df1 100644 --- a/modules/influxdb/tests/test_influxdb.py +++ b/modules/influxdb/tests/test_influxdb.py @@ -18,39 +18,26 @@ from influxdb_client import Bucket from influxdb_client.client.write_api import SYNCHRONOUS from pytest import mark +from requests import get from testcontainers.influxdb import InfluxDbContainer from testcontainers.influxdb1 import InfluxDb1Container from testcontainers.influxdb2 import InfluxDb2Container -@mark.parametrize( - ["image", "influxdb_container_class", "exposed_port"], - [ - ("influxdb:2.7", InfluxDb1Container, 8086), - ("influxdb:1.8", InfluxDb2Container, 8086), - ], -) -def test_influxdbcontainer_get_url(image: str, influxdb_container_class: Type[InfluxDbContainer], exposed_port: int): - with influxdb_container_class(image, host_port=exposed_port) as influxdb_container: - connection_url = influxdb_container.get_url() - assert str(exposed_port) in connection_url - - @mark.parametrize( ["image", "influxdb_container_class", "expected_version"], [ - ("influxdb:1.8", InfluxDb1Container, "1.8.10"), ("influxdb:1.8.10", InfluxDb1Container, "1.8.10"), - ("influxdb:2.7.4", InfluxDb2Container, "v2.7.4"), - ("influxdb:2.7", InfluxDb2Container, "v2.7"), + ("influxdb:2.7.5", InfluxDb2Container, "v2.7.5"), ], ) def test_influxdbcontainer_get_influxdb_version( image: str, influxdb_container_class: Type[InfluxDbContainer], expected_version: str ): with influxdb_container_class(image) as influxdb_container: - assert influxdb_container.get_influxdb_version().startswith(expected_version) + version = get(f"{influxdb_container.get_url()}/health").json()["version"] + assert version == expected_version def test_influxdb1container_get_client(): From bea1372ab25e14de94359cb3d48ec45346fa77e3 Mon Sep 17 00:00:00 2001 From: Vemund Santi Date: Wed, 3 Apr 2024 12:02:14 +0200 Subject: [PATCH 13/20] Use new _wait_until_ready hook in k3s module --- modules/k3s/testcontainers/k3s/__init__.py | 16 +++++++--------- modules/k3s/tests/test_k3s.py | 2 +- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/modules/k3s/testcontainers/k3s/__init__.py b/modules/k3s/testcontainers/k3s/__init__.py index 045e2eb5..ece3c782 100644 --- a/modules/k3s/testcontainers/k3s/__init__.py +++ b/modules/k3s/testcontainers/k3s/__init__.py @@ -11,7 +11,9 @@ # License for the specific language governing permissions and limitations # under the License. -from testcontainers.core.config import MAX_TRIES +from typing_extensions import override + +from testcontainers.core.config import TIMEOUT from testcontainers.core.container import DockerContainer from testcontainers.core.waiting_utils import wait_for_logs @@ -28,7 +30,7 @@ class K3SContainer(DockerContainer): >>> from testcontainers.k3s import K3SContainer >>> from kubernetes import client, config - >>> with K3SContainer() as k3s: + >>> with K3SContainer("rancher/k3s:v1.29.3-k3s1") as k3s: ... config.load_kube_config_from_dict(yaml.safe_load(k3s.config_yaml())) ... pod = client.CoreV1Api().list_pod_for_all_namespaces(limit=1) ... assert len(pod.items) > 0, "Unable to get running nodes from k3s cluster" @@ -45,13 +47,9 @@ def __init__(self, image="rancher/k3s:latest", **kwargs) -> None: self.with_kwargs(privileged=True, tmpfs={"/run": "", "/var/run": ""}) self.with_volume_mapping("/sys/fs/cgroup", "/sys/fs/cgroup", "rw") - def _connect(self) -> None: - wait_for_logs(self, predicate="Node controller sync successful", timeout=MAX_TRIES) - - def start(self) -> "K3SContainer": - super().start() - self._connect() - return self + @override + def _wait_until_ready(self) -> None: + wait_for_logs(self, predicate="Node controller sync successful", timeout=TIMEOUT) def config_yaml(self) -> str: """This function returns the kubernetes config yaml which can be used diff --git a/modules/k3s/tests/test_k3s.py b/modules/k3s/tests/test_k3s.py index edff1c6d..3dba16ae 100644 --- a/modules/k3s/tests/test_k3s.py +++ b/modules/k3s/tests/test_k3s.py @@ -6,7 +6,7 @@ def test_docker_run_k3s(): - with K3SContainer() as k3s: + with K3SContainer("rancher/k3s:v1.29.3-k3s1") as k3s: config.load_kube_config_from_dict(yaml.safe_load(k3s.config_yaml())) pod = client.CoreV1Api().list_pod_for_all_namespaces(limit=1) assert len(pod.items) > 0, "Unable to get running nodes from k3s cluster" From d19f6c9dbba84b4742566bc89f1d32c4c348a2f2 Mon Sep 17 00:00:00 2001 From: Vemund Santi Date: Wed, 3 Apr 2024 12:22:13 +0200 Subject: [PATCH 14/20] Use new _wait_until_ready hook in kafka module --- modules/kafka/testcontainers/kafka/__init__.py | 12 ++++++++---- modules/kafka/testcontainers/kafka/_redpanda.py | 12 ++++++++---- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/modules/kafka/testcontainers/kafka/__init__.py b/modules/kafka/testcontainers/kafka/__init__.py index 648140d4..7f50f0e3 100644 --- a/modules/kafka/testcontainers/kafka/__init__.py +++ b/modules/kafka/testcontainers/kafka/__init__.py @@ -3,6 +3,8 @@ from io import BytesIO from textwrap import dedent +from typing_extensions import override + from testcontainers.core.container import DockerContainer from testcontainers.core.utils import raise_for_deprecated_parameter from testcontainers.core.waiting_utils import wait_for_logs @@ -75,14 +77,16 @@ def tc_start(self) -> None: ) self.create_file(data, KafkaContainer.TC_START_SCRIPT) - def start(self, timeout=30) -> "KafkaContainer": + @override + def _configure(self) -> None: script = KafkaContainer.TC_START_SCRIPT command = f'sh -c "while [ ! -f {script} ]; do sleep 0.1; done; sh {script}"' self.with_command(command) - super().start() + + @override + def _wait_until_ready(self) -> None: self.tc_start() - wait_for_logs(self, r".*\[KafkaServer id=\d+\] started.*", timeout=timeout) - return self + wait_for_logs(self, r".*\[KafkaServer id=\d+\] started.*", timeout=30.0) def create_file(self, content: bytes, path: str) -> None: with BytesIO() as archive, tarfile.TarFile(fileobj=archive, mode="w") as tar: diff --git a/modules/kafka/testcontainers/kafka/_redpanda.py b/modules/kafka/testcontainers/kafka/_redpanda.py index 90d02c4c..fc56b5ff 100644 --- a/modules/kafka/testcontainers/kafka/_redpanda.py +++ b/modules/kafka/testcontainers/kafka/_redpanda.py @@ -3,6 +3,8 @@ from io import BytesIO from textwrap import dedent +from typing_extensions import override + from testcontainers.core.container import DockerContainer from testcontainers.core.waiting_utils import wait_for_logs @@ -63,14 +65,16 @@ def tc_start(self) -> None: self.create_file(data, RedpandaContainer.TC_START_SCRIPT) - def start(self, timeout=10) -> "RedpandaContainer": + @override + def _configure(self) -> None: script = RedpandaContainer.TC_START_SCRIPT command = f'-c "while [ ! -f {script} ]; do sleep 0.1; done; sh {script}"' self.with_command(command) - super().start() + + @override + def _wait_until_ready(self) -> None: self.tc_start() - wait_for_logs(self, r".*Started Kafka API server.*", timeout=timeout) - return self + wait_for_logs(self, r".*Started Kafka API server.*", timeout=30.0) def create_file(self, content: bytes, path: str) -> None: with BytesIO() as archive, tarfile.TarFile(fileobj=archive, mode="w") as tar: From 124727c26dac6acadc5ffcfd89cfe38b4b346bc0 Mon Sep 17 00:00:00 2001 From: Vemund Santi Date: Wed, 3 Apr 2024 12:24:56 +0200 Subject: [PATCH 15/20] Use new _wait_until_ready hook in keycloak module --- modules/keycloak/testcontainers/keycloak/__init__.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/modules/keycloak/testcontainers/keycloak/__init__.py b/modules/keycloak/testcontainers/keycloak/__init__.py index ca570229..22aed09e 100644 --- a/modules/keycloak/testcontainers/keycloak/__init__.py +++ b/modules/keycloak/testcontainers/keycloak/__init__.py @@ -66,19 +66,13 @@ def get_url(self) -> str: return f"http://{host}:{port}" @wait_container_is_ready(requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout) - def _readiness_probe(self) -> None: + def _wait_until_ready(self) -> None: # Keycloak provides an REST API endpoints for health checks: https://www.keycloak.org/server/health response = requests.get(f"{self.get_url()}/health/ready", timeout=1) response.raise_for_status() if self._command == _DEFAULT_DEV_COMMAND: wait_for_logs(self, "Added user .* to realm .*") - def start(self) -> "KeycloakContainer": - self._configure() - super().start() - self._readiness_probe() - return self - def get_client(self, **kwargs) -> KeycloakAdmin: default_kwargs = { "server_url": self.get_url(), From 18313660370b9467a67237bdfcceb42a63ab4ecf Mon Sep 17 00:00:00 2001 From: Vemund Santi Date: Wed, 3 Apr 2024 12:28:00 +0200 Subject: [PATCH 16/20] Use new _wait_until_ready hook in localstack module --- .../localstack/testcontainers/localstack/__init__.py | 10 +++++----- modules/localstack/tests/test_localstack.py | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/modules/localstack/testcontainers/localstack/__init__.py b/modules/localstack/testcontainers/localstack/__init__.py index 15cabeab..89dbdd30 100644 --- a/modules/localstack/testcontainers/localstack/__init__.py +++ b/modules/localstack/testcontainers/localstack/__init__.py @@ -15,6 +15,7 @@ from typing import Any, Optional import boto3 +from typing_extensions import override from testcontainers.core.container import DockerContainer from testcontainers.core.waiting_utils import wait_for_logs @@ -30,7 +31,7 @@ class LocalStackContainer(DockerContainer): >>> from testcontainers.localstack import LocalStackContainer - >>> with LocalStackContainer(image="localstack/localstack:2.0.1") as localstack: + >>> with LocalStackContainer("localstack/localstack:2.0.1") as localstack: ... dynamo_client = localstack.get_client("dynamodb") ... tables = dynamo_client.list_tables() >>> tables @@ -85,7 +86,6 @@ def get_client(self, name, **kwargs) -> Any: kwargs_.update(kwargs) return boto3.client(name, **kwargs_) - def start(self, timeout: float = 60) -> "LocalStackContainer": - super().start() - wait_for_logs(self, r"Ready\.\n", timeout=timeout) - return self + @override + def _wait_until_ready(self) -> None: + wait_for_logs(self, r"Ready\.\n", timeout=60.0) diff --git a/modules/localstack/tests/test_localstack.py b/modules/localstack/tests/test_localstack.py index 6801aefd..58197aa0 100644 --- a/modules/localstack/tests/test_localstack.py +++ b/modules/localstack/tests/test_localstack.py @@ -5,7 +5,7 @@ def test_docker_run_localstack(): - with LocalStackContainer() as localstack: + with LocalStackContainer("localstack/localstack:2.0.1") as localstack: resp = urllib.request.urlopen(f"{localstack.get_url()}/health") services = json.loads(resp.read().decode())["services"] @@ -18,7 +18,7 @@ def test_docker_run_localstack(): def test_localstack_boto3(): from testcontainers.localstack import LocalStackContainer - with LocalStackContainer(image="localstack/localstack:2.0.1") as localstack: + with LocalStackContainer("localstack/localstack:2.0.1") as localstack: dynamo_client = localstack.get_client("dynamodb") tables = dynamo_client.list_tables() assert tables["TableNames"] == [] From 0688858ab36a120d7f1aa7634816f920bc998a14 Mon Sep 17 00:00:00 2001 From: Vemund Santi Date: Wed, 3 Apr 2024 12:33:17 +0200 Subject: [PATCH 17/20] Use new _wait_until_ready hook in minio module --- .../minio/testcontainers/minio/__init__.py | 30 +++++-------------- modules/minio/tests/test_minio.py | 4 ++- 2 files changed, 10 insertions(+), 24 deletions(-) diff --git a/modules/minio/testcontainers/minio/__init__.py b/modules/minio/testcontainers/minio/__init__.py index 51a7094e..409cd74b 100644 --- a/modules/minio/testcontainers/minio/__init__.py +++ b/modules/minio/testcontainers/minio/__init__.py @@ -1,14 +1,9 @@ -from typing import TYPE_CHECKING - -from requests import ConnectionError, get +from typing_extensions import override from minio import Minio from testcontainers.core.container import DockerContainer from testcontainers.core.utils import raise_for_deprecated_parameter -from testcontainers.core.waiting_utils import wait_container_is_ready - -if TYPE_CHECKING: - from requests import Response +from testcontainers.core.waiting_utils import wait_for_http class MinioContainer(DockerContainer): @@ -27,7 +22,7 @@ class MinioContainer(DockerContainer): >>> import io >>> from testcontainers.minio import MinioContainer - >>> with MinioContainer() as minio: + >>> with MinioContainer("minio/minio:RELEASE.2024-03-30T09-41-56Z") as minio: ... client = minio.get_client() ... client.make_bucket("test") ... test_content = b"Hello World" @@ -42,7 +37,7 @@ class MinioContainer(DockerContainer): def __init__( self, - image: str = "minio/minio:RELEASE.2022-12-02T19-19-22Z", + image: str = "minio/minio:RELEASE.2024-03-30T09-41-56Z", port: int = 9000, access_key: str = "minioadmin", secret_key: str = "minioadmin", @@ -98,17 +93,6 @@ def get_config(self) -> dict: "secret_key": self.secret_key, } - @wait_container_is_ready(ConnectionError) - def _healthcheck(self) -> None: - """This is an internal method used to check if the Minio container - is healthy and ready to receive requests.""" - url = f"http://{self.get_config()['endpoint']}/minio/health/live" - response: Response = get(url) - response.raise_for_status() - - def start(self) -> "MinioContainer": - """This method starts the Minio container and runs the healthcheck - to verify that the container is ready to use.""" - super().start() - self._healthcheck() - return self + @override + def _wait_until_ready(self) -> None: + wait_for_http(f"http://{self.get_config()['endpoint']}/minio/health/live") diff --git a/modules/minio/tests/test_minio.py b/modules/minio/tests/test_minio.py index 99eed438..79a5c1f1 100644 --- a/modules/minio/tests/test_minio.py +++ b/modules/minio/tests/test_minio.py @@ -4,7 +4,9 @@ def test_docker_run_minio(): - config = MinioContainer(access_key="test-access", secret_key="test-secret") + config = MinioContainer( + "minio/minio:RELEASE.2024-03-30T09-41-56Z", access_key="test-access", secret_key="test-secret" + ) with config as minio: client = minio.get_client() client.make_bucket("test") From e5afe8848539c3fef59427de0b73c4c8a4f943fa Mon Sep 17 00:00:00 2001 From: Vemund Santi Date: Wed, 3 Apr 2024 12:39:09 +0200 Subject: [PATCH 18/20] Use new _wait_until_ready hook in mongodb module --- .../mongodb/testcontainers/mongodb/__init__.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/modules/mongodb/testcontainers/mongodb/__init__.py b/modules/mongodb/testcontainers/mongodb/__init__.py index 32e1f748..57bec453 100644 --- a/modules/mongodb/testcontainers/mongodb/__init__.py +++ b/modules/mongodb/testcontainers/mongodb/__init__.py @@ -13,14 +13,16 @@ import os from typing import Optional +from typing_extensions import override + from pymongo import MongoClient -from testcontainers.core.generic import DbContainer -from testcontainers.core.utils import raise_for_deprecated_parameter +from testcontainers.core.container import DockerContainer +from testcontainers.core.utils import create_connection_string, raise_for_deprecated_parameter from testcontainers.core.waiting_utils import wait_for_logs -class MongoDbContainer(DbContainer): +class MongoDbContainer(DockerContainer): """ Mongo document-based database container. @@ -63,20 +65,23 @@ def __init__( self.port = port self.with_exposed_ports(self.port) + @override def _configure(self) -> None: self.with_env("MONGO_INITDB_ROOT_USERNAME", self.username) self.with_env("MONGO_INITDB_ROOT_PASSWORD", self.password) self.with_env("MONGO_DB", self.dbname) def get_connection_url(self) -> str: - return self._create_connection_url( + return create_connection_string( dialect="mongodb", username=self.username, password=self.password, - port=self.port, + host=self.get_container_host_ip(), + port=self.get_exposed_port(self.port), ) - def _connect(self) -> None: + @override + def _wait_until_ready(self) -> None: wait_for_logs(self, "Waiting for connections") def get_connection_client(self) -> MongoClient: From 20aead192362bfe4abeab3c99d47c87c9492a613 Mon Sep 17 00:00:00 2001 From: Vemund Santi Date: Wed, 3 Apr 2024 13:01:08 +0200 Subject: [PATCH 19/20] Reorder imports for linting --- modules/mongodb/testcontainers/mongodb/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/mongodb/testcontainers/mongodb/__init__.py b/modules/mongodb/testcontainers/mongodb/__init__.py index 57bec453..e557966a 100644 --- a/modules/mongodb/testcontainers/mongodb/__init__.py +++ b/modules/mongodb/testcontainers/mongodb/__init__.py @@ -13,9 +13,8 @@ import os from typing import Optional -from typing_extensions import override - from pymongo import MongoClient +from typing_extensions import override from testcontainers.core.container import DockerContainer from testcontainers.core.utils import create_connection_string, raise_for_deprecated_parameter From 00b2e7f39013a8e4541a518da00b824396046034 Mon Sep 17 00:00:00 2001 From: Vemund Santi Date: Wed, 3 Apr 2024 13:01:27 +0200 Subject: [PATCH 20/20] Use new _wait_until_ready hook in mssql module --- .../mssql/testcontainers/mssql/__init__.py | 39 ++++++++++++++++--- modules/mssql/tests/test_mssql.py | 2 +- 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/modules/mssql/testcontainers/mssql/__init__.py b/modules/mssql/testcontainers/mssql/__init__.py index 98b66826..9efd3f16 100644 --- a/modules/mssql/testcontainers/mssql/__init__.py +++ b/modules/mssql/testcontainers/mssql/__init__.py @@ -1,11 +1,22 @@ from os import environ from typing import Optional -from testcontainers.core.generic import DbContainer -from testcontainers.core.utils import raise_for_deprecated_parameter +from typing_extensions import override +from testcontainers.core.container import DockerContainer +from testcontainers.core.utils import create_connection_string, raise_for_deprecated_parameter +from testcontainers.core.waiting_utils import wait_container_is_ready -class SqlServerContainer(DbContainer): +ADDITIONAL_TRANSIENT_ERRORS = [] +try: + from sqlalchemy.exc import DBAPIError + + ADDITIONAL_TRANSIENT_ERRORS.append(DBAPIError) +except ImportError: + pass + + +class SqlServerContainer(DockerContainer): """ Microsoft SQL Server database container. @@ -16,7 +27,7 @@ class SqlServerContainer(DbContainer): >>> import sqlalchemy >>> from testcontainers.mssql import SqlServerContainer - >>> with SqlServerContainer() as mssql: + >>> with SqlServerContainer("mcr.microsoft.com/azure-sql-edge:1.0.7") as mssql: ... engine = sqlalchemy.create_engine(mssql.get_connection_url()) ... with engine.begin() as connection: ... result = connection.execute(sqlalchemy.text("select @@VERSION")) @@ -43,6 +54,17 @@ def __init__( self.dbname = dbname self.dialect = dialect + @wait_container_is_ready(*ADDITIONAL_TRANSIENT_ERRORS) + def _wait_until_ready(self): + import sqlalchemy + + engine = sqlalchemy.create_engine(self.get_connection_url()) + try: + engine.connect() + finally: + engine.dispose() + + @override def _configure(self) -> None: self.with_env("SA_PASSWORD", self.password) self.with_env("SQLSERVER_USER", self.username) @@ -50,6 +72,11 @@ def _configure(self) -> None: self.with_env("ACCEPT_EULA", "Y") def get_connection_url(self) -> str: - return super()._create_connection_url( - dialect=self.dialect, username=self.username, password=self.password, dbname=self.dbname, port=self.port + return create_connection_string( + dialect=self.dialect, + username=self.username, + password=self.password, + host=self.get_container_host_ip(), + port=self.get_exposed_port(self.port), + dbname=self.dbname, ) diff --git a/modules/mssql/tests/test_mssql.py b/modules/mssql/tests/test_mssql.py index 6f48f0a1..a4d02f7e 100644 --- a/modules/mssql/tests/test_mssql.py +++ b/modules/mssql/tests/test_mssql.py @@ -4,7 +4,7 @@ def test_docker_run_mssql(): - image = "mcr.microsoft.com/azure-sql-edge" + image = "mcr.microsoft.com/azure-sql-edge:1.0.7" dialect = "mssql+pymssql" with SqlServerContainer(image, dialect=dialect) as mssql: engine = sqlalchemy.create_engine(mssql.get_connection_url())