WIP test refactor setup

brewblox_history/app.py

@@ -4,16 +4,15 @@
 
 from fastapi import FastAPI
 
-from . import datastore_api, mqtt, redis, relays, timeseries_api, victoria
-from .models import ServiceConfig
+from . import (datastore_api, mqtt, redis, relays, timeseries_api, utils,
+               victoria)
 
 LOGGER = logging.getLogger(__name__)
 
 
-def init_logging():
-    config = ServiceConfig.cached()
-    level = logging.DEBUG if config.debug else logging.INFO
-    unimportant_level = logging.INFO if config.debug else logging.WARN
+def init_logging(debug: bool):
+    level = logging.DEBUG if debug else logging.INFO
+    unimportant_level = logging.INFO if debug else logging.WARN
     format = '%(asctime)s.%(msecs)03d [%(levelname).1s:%(name)s:%(lineno)d] %(message)s'
     datefmt = '%Y/%m/%d %H:%M:%S'
 
@@ -34,7 +33,7 @@ def setup():
 
 @asynccontextmanager
 async def lifespan(app: FastAPI):
-    LOGGER.info(ServiceConfig.cached())
+    LOGGER.info(utils.get_config())
     LOGGER.debug('ROUTES:\n' + pformat(app.routes))
     # LOGGER.debug('LOGGERS:\n' + pformat(logging.root.manager.loggerDict))
 
@@ -45,10 +44,11 @@ async def lifespan(app: FastAPI):
 
 
 def create_app():
-    init_logging()
+    config = utils.get_config()
+
+    init_logging(config.debug)
     setup()
 
-    config = ServiceConfig.cached()
     prefix = f'/{config.name}'
     app = FastAPI(lifespan=lifespan,
                   docs_url=f'{prefix}/api/doc',

brewblox_history/models.py

@@ -4,12 +4,12 @@
 
 import collections
 from datetime import datetime
-from functools import lru_cache
-from typing import Any, Literal, NamedTuple, Self
+from typing import Any, Literal, NamedTuple
 
 from pydantic import (BaseModel, ConfigDict, Field, field_validator,
                       model_validator)
-from pydantic_settings import BaseSettings, SettingsConfigDict
+from pydantic_settings import (BaseSettings, PydanticBaseSettingsSource,
+                               SettingsConfigDict)
 
 
 def flatten(d, parent_key=''):
@@ -36,29 +36,36 @@ class ServiceConfig(BaseSettings):
         env_file='.appenv',
         env_prefix='brewblox_',
         case_sensitive=False,
-        extra='ignore',
+        json_schema_extra='ignore',
     )
 
     name: str = 'history'
     debug: bool = False
+
+    mqtt_protocol: Literal['mqtt', 'mqtts'] = 'mqtt'
     mqtt_host: str = 'eventbus'
     mqtt_port: int = 1883
-    redis_url: str = 'redis://redis'
-    victoria_url: str = 'http://victoria:8428/victoria'
+
+    redis_host: str = 'redis'
+    redis_port: int = 6379
+
+    victoria_protocol: Literal['http', 'https'] = 'http'
+    victoria_host: str = 'victoria'
+    victoria_port: int = 8428
+    victoria_path: str = '/victoria'
+
     history_topic: str = 'brewcast/history'
     datastore_topic: str = 'brewcast/datastore'
+
     ranges_interval: float = 10
     metrics_interval: float = 10
     minimum_step: float = 10
 
-    @classmethod
-    @lru_cache
-    def cached(cls) -> Self:
-        return cls()
-
 
 class HistoryEvent(BaseModel):
-    model_config = ConfigDict(extra='ignore')
+    model_config = ConfigDict(
+        json_schema_extra='ignore',
+    )
 
     key: str
     data: dict[str, Any]  # converted to float later
@@ -68,14 +75,12 @@ class HistoryEvent(BaseModel):
     def flatten_data(cls, v):
         assert isinstance(v, dict)
         return flatten(v)
-        # assert 'key' in v
-        # assert 'data' in v
-        # data = flatten(v['data'])
-        # return {'key': v['key'], 'data': data}
 
 
 class DatastoreValue(BaseModel):
-    model_config = ConfigDict(extra='allow')
+    model_config = ConfigDict(
+        json_schema_extra='allow',
+    )
 
     namespace: str = Field(pattern=r'^[\w\-\.\:~_ \(\)]*$')
     id: str = Field(pattern=r'^[\w\-\.\:~_ \(\)]*$')

brewblox_history/mqtt.py

@@ -1,21 +1,34 @@
+import asyncio
 from contextlib import asynccontextmanager
 from contextvars import ContextVar
 
 from fastapi_mqtt.config import MQTTConfig
 from fastapi_mqtt.fastmqtt import FastMQTT
 
-from .models import ServiceConfig
+from . import utils
 
-CV: ContextVar[FastMQTT] = ContextVar('FastMQTT')
+CV: ContextVar[FastMQTT] = ContextVar('mqtt.client')
+CV_CONNECTED: ContextVar[asyncio.Event] = ContextVar('mqtt.connected')
 
 
 def setup():
-    config = ServiceConfig.cached()
+    config = utils.get_config()
     mqtt_config = MQTTConfig(host=config.mqtt_host,
                              port=config.mqtt_port,
+                             ssl=(config.mqtt_protocol == 'mqtts'),
                              reconnect_retries=-1)
     fast_mqtt = FastMQTT(config=mqtt_config)
+    evt = asyncio.Event()
     CV.set(fast_mqtt)
+    CV_CONNECTED.set(evt)
+
+    @fast_mqtt.on_connect()
+    def on_connect(client, flags, rc, properties):
+        CV_CONNECTED.get().set()
+
+    @fast_mqtt.on_disconnect()
+    def on_disconnect(client, packet, exc=None):
+        CV_CONNECTED.get().clear()
 
 
 @asynccontextmanager

brewblox_history/redis.py

@@ -7,12 +7,12 @@
 
 from redis import asyncio as aioredis
 
-from . import mqtt
-from .models import DatastoreValue, ServiceConfig
+from . import mqtt, utils
+from .models import DatastoreValue
 
 LOGGER = logging.getLogger(__name__)
 
-CV: ContextVar['RedisClient'] = ContextVar('RedisClient')
+CV: ContextVar['RedisClient'] = ContextVar('redis.client')
 
 
 def keycat(namespace: str, key: str) -> str:
@@ -36,8 +36,8 @@ async def wrapper(self: 'RedisClient', *args, **kwargs):
 class RedisClient:
 
     def __init__(self):
-        config = ServiceConfig.cached()
-        self.url = config.redis_url
+        config = utils.get_config()
+        self.url = f'redis://{config.redis_host}:{config.redis_port}'
         self.topic = config.datastore_topic
         # Lazy-loaded in autoconnect wrapper
         self._redis: aioredis.Redis = None

brewblox_history/relays.py

@@ -46,13 +46,13 @@
 from pydantic import ValidationError
 
 from . import mqtt, utils, victoria
-from .models import HistoryEvent, ServiceConfig
+from .models import HistoryEvent
 
 LOGGER = logging.getLogger(__name__)
 
 
 def setup():
-    config = ServiceConfig.cached()
+    config = utils.get_config()
     fast_mqtt = mqtt.CV.get()
 
     @fast_mqtt.subscribe(config.history_topic + '/#')

brewblox_history/timeseries_api.py

@@ -11,9 +11,8 @@
 from fastapi.responses import StreamingResponse
 
 from brewblox_history import utils, victoria
-from brewblox_history.models import (ServiceConfig, TimeSeriesCsvQuery,
-                                     TimeSeriesFieldsQuery, TimeSeriesMetric,
-                                     TimeSeriesMetricsQuery,
+from brewblox_history.models import (TimeSeriesCsvQuery, TimeSeriesFieldsQuery,
+                                     TimeSeriesMetric, TimeSeriesMetricsQuery,
                                      TimeSeriesMetricStreamData,
                                      TimeSeriesRange, TimeSeriesRangesQuery,
                                      TimeSeriesRangeStreamData,
@@ -111,7 +110,7 @@ async def generate():
 
 
 async def _stream_ranges(ws: WebSocket, id: str, query: TimeSeriesRangesQuery):
-    interval = ServiceConfig.cached().ranges_interval
+    interval = utils.get_config().ranges_interval
     open_ended = utils.is_open_ended(start=query.start,
                                      duration=query.duration,
                                      end=query.end)
@@ -139,7 +138,7 @@ async def _stream_ranges(ws: WebSocket, id: str, query: TimeSeriesRangesQuery):
 
 
 async def _stream_metrics(ws: WebSocket, id: str, query: TimeSeriesMetricsQuery):
-    interval = ServiceConfig.cached().metrics_interval
+    interval = utils.get_config().metrics_interval
 
     while True:
         async with protected('metrics push'):

brewblox_history/utils.py

@@ -1,10 +1,13 @@
 import logging
 import traceback
 from datetime import datetime, timedelta, timezone
+from functools import lru_cache
 
 import ciso8601
 from pytimeparse.timeparse import timeparse
 
+from .models import ServiceConfig
+
 FLAT_SEPARATOR = '/'
 DESIRED_POINTS = 1000
 DEFAULT_DURATION = timedelta(days=1)
@@ -29,6 +32,11 @@ def filter(self, record):
         return False
 
 
+@lru_cache
+def get_config() -> ServiceConfig:
+    return ServiceConfig()
+
+
 def strex(ex: Exception, tb=False):
     """
     Generic formatter for exceptions.

brewblox_history/victoria.py

@@ -9,23 +9,24 @@
 from sortedcontainers import SortedDict
 
 from brewblox_history import utils
-from brewblox_history.models import (HistoryEvent, ServiceConfig,
-                                     TimeSeriesCsvQuery, TimeSeriesFieldsQuery,
-                                     TimeSeriesMetric, TimeSeriesMetricsQuery,
-                                     TimeSeriesRange, TimeSeriesRangesQuery)
+from brewblox_history.models import (HistoryEvent, TimeSeriesCsvQuery,
+                                     TimeSeriesFieldsQuery, TimeSeriesMetric,
+                                     TimeSeriesMetricsQuery, TimeSeriesRange,
+                                     TimeSeriesRangesQuery)
 
 LOGGER = logging.getLogger(__name__)
 LOGGER.addFilter(utils.DuplicateFilter())
 
-CV: ContextVar['VictoriaClient'] = ContextVar('VictoriaClient')
+CV: ContextVar['VictoriaClient'] = ContextVar('victoria.client')
 
 
 class VictoriaClient:
 
     def __init__(self):
-        config = ServiceConfig.cached()
+        config = utils.get_config()
 
-        self._url = config.victoria_url
+        self._url = f'{config.victoria_protocol}://{config.victoria_host}:{config.victoria_port}'
+        self._url += config.victoria_path
         self._minimum_step = timedelta(seconds=config.minimum_step)
         self._query_headers = {
             'Content-Type': 'application/x-www-form-urlencoded',

pyproject.toml

@@ -35,7 +35,6 @@ requires = ["poetry-core>=1.0.0"]
 build-backend = "poetry.core.masonry.api"
 
 [tool.pytest.ini_options]
-asyncio_mode = "auto"
 addopts = """
     --ignore=app/
     --ignore=victoria/

test/conftest.py

@@ -5,18 +5,39 @@
 
 
 import logging
-import os
 import socket
 from contextlib import contextmanager
 from subprocess import run
 
 import pytest
+from pydantic_settings import BaseSettings, PydanticBaseSettingsSource
 
-from brewblox_history import app
+from brewblox_history import app, models, utils
 
 LOGGER = logging.getLogger(__name__)
 
 
+class TestConfig(models.ServiceConfig):
+    """
+    An override for ServiceConfig that only uses
+    settings provided to __init__()
+
+    This makes tests independent from env values
+    and the content of .appenv
+    """
+
+    @classmethod
+    def settings_customise_sources(
+        cls,
+        settings_cls: type[BaseSettings],
+        init_settings: PydanticBaseSettingsSource,
+        env_settings: PydanticBaseSettingsSource,
+        dotenv_settings: PydanticBaseSettingsSource,
+        file_secret_settings: PydanticBaseSettingsSource,
+    ) -> tuple[PydanticBaseSettingsSource, ...]:
+        return (init_settings,)
+
+
 def find_free_port():
     s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
     s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
@@ -65,20 +86,15 @@ def docker_container(name: str, ports: dict[str, int], args: list[str]):
 
 
 @pytest.fixture(scope='session', autouse=True)
-def env_settings():
-    os.environ['BREWBLOX_DEBUG'] = 'True'
-
-
-@pytest.fixture(scope='session', autouse=True)
-def log_enabled(env_settings):
-    app.init_logging()
+def log_enabled():
+    app.init_logging(True)
 
 
 @pytest.fixture(scope='session')
 def mqtt_container():
     with docker_container(
         name='mqtt-test-container',
-        ports={'mqtt': 1883, 'ws': 15675},
+        ports={'mqtt': 1883},
         args=['ghcr.io/brewblox/mosquitto:develop'],
     ) as ports:
         yield ports
@@ -92,3 +108,16 @@ def redis_container():
         args=['redis:6.0'],
     ) as ports:
         yield ports
+
+
+@pytest.fixture(autouse=True)
+def config(monkeypatch: pytest.MonkeyPatch, mqtt_container, redis_container):
+    cfg = TestConfig(
+        debug=True,
+        mqtt_host='localhost',
+        mqtt_port=mqtt_container['mqtt'],
+        redis_host='localhost',
+        redis_port=redis_container['redis'],
+    )
+    monkeypatch.setattr(utils, 'get_config', lambda: cfg)
+    yield cfg