diff --git a/.github/workflows/_system_test.yml b/.github/workflows/_system_test.yml index dfd6ef8fee..e314f68ea7 100644 --- a/.github/workflows/_system_test.yml +++ b/.github/workflows/_system_test.yml @@ -40,7 +40,7 @@ jobs: run: uv run --locked playwright install --with-deps chromium - name: Run tests - run: uv run --locked tox -e system-test + run: uv run --locked tox -e client-system-test,system-test - name: Compose logs if: failure() diff --git a/Dockerfile b/Dockerfile index e328687ec3..e456696ad9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,7 +29,7 @@ ENV UV_PYTHON_INSTALL_DIR=/python # Sync the project without its dev dependencies RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync --locked --no-editable --no-dev --managed-python + uv sync --locked --no-editable --no-dev --extra server --managed-python RUN uv pip install debugpy diff --git a/README.md b/README.md index 5ba2e7e5ad..32adc22ffa 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ Lightweight bluesky-as-a-service wrapper application. Also usable as a library. What | Where :---: | :---: Source | -PyPI | `pip install blueapi` +PyPI | `pip install blueapi[server]` (server) / `pip install blueapi` (client only) Docker | `docker run ghcr.io/diamondlightsource/blueapi:latest` Documentation | Releases | diff --git a/docs/tutorials/1. installation.md b/docs/tutorials/1. installation.md index 4b49b9b477..5313a3111c 100644 --- a/docs/tutorials/1. installation.md +++ b/docs/tutorials/1. installation.md @@ -25,15 +25,18 @@ source /path/to/venv/bin/activate You can now use `pip` to install the library and its dependencies: ``` -python3 -m pip install blueapi +python3 -m pip install blueapi[server] ``` If you require a feature that is not currently released you can also install from GitHub: ``` -python3 -m pip install git+https://github.com/DiamondLightSource/blueapi.git +python3 -m pip install "blueapi[server] @ git+https://github.com/DiamondLightSource/blueapi.git" ``` +The `server` extra is only needed to run the worker (`blueapi serve`). For the client +and `blueapi controller` commands, `pip install blueapi` is enough. + The library should now be installed and the commandline interface on your path. You can check the version that has been installed by typing: ``` diff --git a/docs/tutorials/2. quickstart.md b/docs/tutorials/2. quickstart.md index 670d0c9774..d2d4d6b0b3 100644 --- a/docs/tutorials/2. quickstart.md +++ b/docs/tutorials/2. quickstart.md @@ -7,7 +7,7 @@ You should be in the blueapi directory that you have cloned. From here, if you o If you not, you can still run the following from the venv created in page 1 but you may need to run the command below to have the dependencies required for this tutorial. ``` -pip install .[demo] +pip install .[demo,server] ``` ## Start Worker diff --git a/pyproject.toml b/pyproject.toml index b4d2c542a9..fe2f73838f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,28 +14,23 @@ classifiers = [ ] description = "Lightweight bluesky-as-a-service wrapper application. Also usable as a library." dependencies = [ - "tiled[client]==0.2.18", "bluesky[plotting]>=1.14.0", # plotting includes matplotlib, required for BestEffortCallback in run plans "ophyd-async>=0.13.5", - "aioca", "pydantic>=2.0", "pydantic-settings", "stomp-py", "PyYAML>=6.0.2", "click>=8.2.0", - "fastapi>=0.112.0", - "uvicorn>=0.52.1", "requests", - "GitPython>=3.1.58", #security base min https://github.com/advisories/GHSA-4gmw-gg2m-w46p + "GitPython>=3.1.58", #security base min https://github.com/advisories/GHSA-4gmw-gg2m-w46p - used by the server's /python_environment endpoint and the CLI's scratch setup "event-model==1.24.0", # https://github.com/DiamondLightSource/blueapi/issues/684 "bluesky-stomp>=0.1.6", "opentelemetry-distro>=0.48b0", - "opentelemetry-instrumentation-fastapi>=0.48b0", "observability-utils>=0.1.4", "pyjwt[crypto]", - "graypy>=2.1.0", - "httpx>=0.28.1", - "aiohttp>=3.13.5", + "websockets", + "tqdm", + "packaging", ] dynamic = ["version"] license.file = "LICENSE" @@ -43,10 +38,23 @@ readme = "README.md" requires-python = ">=3.11" [project.optional-dependencies] + demo = ["dls-dodal>=3.0.2", "ophyd-async[sim]"] +# Only needed to run `blueapi serve` (the FastAPI worker service). +server = [ + "fastapi>=0.112.0", + "uvicorn>=0.52.1", + "aiohttp>=3.13.5", + "opentelemetry-instrumentation-fastapi>=0.48b0", + "graypy>=2.1.0", + "aioca", + "tiled[client]==0.2.18", + "httpx>=0.28.1", # used by service/numtracker.py and service/authentication.py's TiledAuth +] [dependency-groups] dev = [ + "blueapi[server]", "ophyd_async[sim]", "copier", "dls-dodal>=3.0.2", @@ -79,6 +87,14 @@ dev = [ "httpx2>=2.5.0", "playwright>=1.63.0", ] +# Runs tests/system_tests/test_blueapi_client.py against blueapi without the server extra. +client-system-test = [ + "pytest", + "pytest-asyncio", + "playwright>=1.63.0", + "responses", # imported by tests/conftest.py + "jwcrypto", # imported by tests/conftest.py +] [project.scripts] blueapi = "blueapi.cli:main" @@ -173,12 +189,31 @@ commands = [ ], ] +[tool.tox.env.client-system-test] +description = "Run client system tests with blueapi only, no server dependencies" +commands = [ + [ + "uv", + "run", + "--isolated", # uv run leaves extra packages in place, so don't reuse the dev venv + "--locked", + "--no-default-groups", + "--group", + "client-system-test", + "pytest", + "tests/system_tests/test_blueapi_client.py", + { replace = "posargs", default = [ + ], extend = true }, + ], +] + [tool.tox.env.system-test] description = "Run system tests with coverage" commands = [ [ "pytest", "tests/system_tests", + "--ignore=tests/system_tests/test_blueapi_client.py", "--cov=blueapi", "--cov-report", "term", diff --git a/src/blueapi/cli/cli.py b/src/blueapi/cli/cli.py index 8424c3c4ec..d78a88d012 100644 --- a/src/blueapi/cli/cli.py +++ b/src/blueapi/cli/cli.py @@ -31,13 +31,13 @@ UnauthorisedAccessError, UnknownPlanError, ) +from blueapi.client.session import SessionCacheManager, SessionManager from blueapi.config import ( ApplicationConfig, ConfigLoader, ) from blueapi.core import OTLP_EXPORT_ENABLED, DataEvent from blueapi.log import set_up_logging -from blueapi.service.authentication import SessionCacheManager, SessionManager from blueapi.service.model import DeviceResponse, PlanResponse, SourceInfo, TaskRequest from blueapi.worker import ProgressEvent, WorkerEvent from blueapi.worker.event import TaskError, TaskResult @@ -164,6 +164,13 @@ def main( print("Please invoke subcommand!") +def _missing_server_extra(e: ImportError) -> ClickException: + return ClickException( + f"{e.name} is not installed, it is needed to run the server. " + "Install it with: pip install blueapi[server]" + ) + + @main.command(name="schema") @click.option("-o", "--output", type=Path, help="Path to file to save the schema") @click.option( @@ -177,12 +184,15 @@ def schema(output: Path | None = None, update: bool = False) -> None: """Only import the service functions when starting the service or generating the schema, not the controller as a new FastAPI app will be started each time. """ - from blueapi.service.openapi import ( - DOCS_SCHEMA_LOCATION, - generate_schema, - print_schema_as_yaml, - write_schema_as_yaml, - ) + try: + from blueapi.service.openapi import ( + DOCS_SCHEMA_LOCATION, + generate_schema, + print_schema_as_yaml, + write_schema_as_yaml, + ) + except ImportError as e: + raise _missing_server_extra(e) from e """Generate the schema for the REST API""" schema = generate_schema() @@ -229,7 +239,10 @@ def start_application(obj: dict): """Only import the service functions when starting the service or generating the schema, not the controller as a new FastAPI app will be started each time. """ - from blueapi.service.main import start + try: + from blueapi.service.main import start + except ImportError as e: + raise _missing_server_extra(e) from e """ Set up basic automated instrumentation for the FastAPI app, creating the diff --git a/src/blueapi/cli/scratch.py b/src/blueapi/cli/scratch.py index b9d752e663..f67dd9bd5b 100644 --- a/src/blueapi/cli/scratch.py +++ b/src/blueapi/cli/scratch.py @@ -1,9 +1,7 @@ -import importlib.metadata import logging import os import stat import textwrap -import tomllib from concurrent.futures import ThreadPoolExecutor, as_completed from pathlib import Path from subprocess import Popen @@ -11,8 +9,7 @@ from git import Repo from blueapi.config import FORBIDDEN_OWN_REMOTE_URL, ScratchConfig -from blueapi.service.model import PackageInfo, PythonEnvironmentResponse, SourceInfo -from blueapi.utils import get_owner_gid, is_sgid_set +from blueapi.utils import get_owner_gid, is_sgid_set, validate_directory _DEFAULT_INSTALL_TIMEOUT: float = 300.0 @@ -131,7 +128,7 @@ def scratch_install(*paths: Path, timeout: float = _DEFAULT_INSTALL_TIMEOUT) -> return LOGGER.info("Installing packages") for path in paths: - _validate_directory(path) + validate_directory(path) args = ["uv", "pip", "install", "-e", str(path)] process = Popen(args) process.wait(timeout=timeout) @@ -142,7 +139,7 @@ def scratch_install(*paths: Path, timeout: float = _DEFAULT_INSTALL_TIMEOUT) -> def _validate_root_directory(root_path: Path, required_gid: int | None) -> None: - _validate_directory(root_path) + validate_directory(root_path) if required_gid is not None: if not is_sgid_set(root_path): @@ -171,98 +168,3 @@ def _validate_root_directory(root_path: Path, required_gid: int | None) -> None: chgrp command. """) ) - - -def _validate_directory(path: Path) -> None: - if not path.exists(): - raise KeyError(f"{path}: No such file or directory") - elif path.is_file(): - raise KeyError(f"{path}: Is a file, not a directory") - - -def _get_project_name_from_pyproject(path: Path) -> str: - pyproject_path = path / "pyproject.toml" - if pyproject_path.exists(): - with pyproject_path.open("rb") as file: - toml_data = tomllib.load(file) - return toml_data.get("project", {}).get("name", "") - return "" - - -def _fetch_installed_packages_details() -> list[PackageInfo]: - installed_packages = importlib.metadata.distributions() - return [ - PackageInfo( - name=dist.metadata["Name"], - version=dist.version, - location=str(dist.locate_file("")), - is_dirty=False, - ) - for dist in installed_packages - ] - - -def get_python_environment( - config: ScratchConfig | None, - name: str | None = None, - source: SourceInfo | None = None, -) -> PythonEnvironmentResponse: - """ - Get the Python environment. This includes all installed packages and - the scratch packages. - """ - scratch_packages = {} - packages = [] - - if config is None: - python_env_response = PythonEnvironmentResponse(scratch_enabled=False) - else: - python_env_response = PythonEnvironmentResponse(scratch_enabled=True) - _validate_directory(config.root) - for repo in config.repositories: - local_directory = config.root / repo.name - repo = Repo(local_directory) - try: - branch = repo.active_branch.name - except TypeError: - branch = repo.head.commit.hexsha - - is_dirty = repo.is_dirty() - - version = ( - f"{repo.remotes[0].url} @{branch}" - if repo.remotes - else f"UNKNOWN REMOTE @{branch}" - ) - package_name = _get_project_name_from_pyproject(local_directory) - package_location = "" - - packages.append( - PackageInfo( - name=package_name, - version=version, - location=package_location, - source=SourceInfo.SCRATCH, - is_dirty=is_dirty, - ) - ) - scratch_packages = {p.name: p for p in packages} - - for pkg in _fetch_installed_packages_details(): - if pkg.name not in scratch_packages: - packages.append(pkg) - else: - scratch_packages[pkg.name].location += f"{pkg.location} &&" - - python_env_response.installed_packages = sorted( - packages, key=lambda pkg: pkg.name.lower() - ) - if name: - python_env_response.installed_packages = [ - p for p in python_env_response.installed_packages if p.name == name - ] - if source: - python_env_response.installed_packages = [ - p for p in python_env_response.installed_packages if p.source == source - ] - return python_env_response diff --git a/src/blueapi/client/client.py b/src/blueapi/client/client.py index b0258141f4..2105ac64e3 100644 --- a/src/blueapi/client/client.py +++ b/src/blueapi/client/client.py @@ -13,6 +13,7 @@ from observability_utils.tracing import get_tracer, start_as_current_span from pydantic import HttpUrl +from blueapi.client.session import SessionCacheManager, SessionManager from blueapi.config import ( ApplicationConfig, ConfigLoader, @@ -20,7 +21,6 @@ RestConfig, ) from blueapi.core.bluesky_types import DataEvent -from blueapi.service.authentication import SessionCacheManager, SessionManager from blueapi.service.model import ( DeviceModel, DeviceResponse, diff --git a/src/blueapi/client/rest.py b/src/blueapi/client/rest.py index f07f6b659c..7bd242a011 100644 --- a/src/blueapi/client/rest.py +++ b/src/blueapi/client/rest.py @@ -1,10 +1,10 @@ import json import logging from collections.abc import Callable, Iterable, Mapping +from http import HTTPStatus from typing import Any, Literal, TypeVar import requests -from fastapi import status from observability_utils.tracing import ( get_context_propagator, get_tracer, @@ -17,9 +17,9 @@ from blueapi import __version__ from blueapi.client.protocols import ClientObjectRef +from blueapi.client.session import JWTAuth, SessionManager from blueapi.config import RestConfig from blueapi.core.bluesky_types import DataEvent -from blueapi.service.authentication import JWTAuth, SessionManager from blueapi.service.model import ( DeviceModel, DeviceResponse, @@ -340,7 +340,7 @@ def _request_and_deserialize( exception = get_exception(response) if exception is not None: raise exception - if response.status_code == status.HTTP_204_NO_CONTENT: + if response.status_code == HTTPStatus.NO_CONTENT: raise NoContentError(target_type) if (server_version := response.headers.get("x-blueapi-version")) is not None: from packaging.version import Version diff --git a/src/blueapi/client/session.py b/src/blueapi/client/session.py new file mode 100644 index 0000000000..07ca6a7ae5 --- /dev/null +++ b/src/blueapi/client/session.py @@ -0,0 +1,247 @@ +from __future__ import annotations + +import base64 +import os +import time +import webbrowser +from abc import ABC, abstractmethod +from functools import cached_property +from http import HTTPStatus +from pathlib import Path +from typing import Any, cast + +import jwt +import requests +from pydantic import TypeAdapter +from requests.auth import AuthBase + +from blueapi.config import OIDCConfig +from blueapi.service.model import Cache + +DEFAULT_CACHE_DIR = "~/.cache/" +SCOPES = "openid offline_access" + + +class CacheManager(ABC): + @abstractmethod + def can_access_cache(self) -> bool: ... + @abstractmethod + def save_cache(self, cache: Cache) -> None: ... + @abstractmethod + def load_cache(self) -> Cache: ... + @abstractmethod + def delete_cache(self) -> None: ... + + +class SessionCacheManager(CacheManager): + def __init__(self, token_path: Path | None) -> None: + self._token_path: Path = ( + token_path if token_path else self._default_token_cache_path() + ) + + @cached_property + def _file_path(self) -> str: + return os.path.expanduser(self._token_path) + + def save_cache(self, cache: Cache) -> None: + self.delete_cache() + self._create_parent_folder_if_necessary() + with open(self._file_path, "xb") as token_file: + token_file.write(base64.b64encode(cache.model_dump_json().encode("utf-8"))) + os.chmod(self._file_path, 0o600) + + def load_cache(self) -> Cache: + self._create_parent_folder_if_necessary() + with open(self._file_path, "rb") as cache_file: + return TypeAdapter(Cache).validate_json( + base64.b64decode(cache_file.read()).decode("utf-8") + ) + + def delete_cache(self) -> None: + Path(self._file_path).unlink(missing_ok=True) + + @staticmethod + def _default_token_cache_path() -> Path: + """ + Return the default cache file path. + """ + cache_path = os.environ.get("XDG_CACHE_HOME", DEFAULT_CACHE_DIR) + return Path(cache_path).expanduser() / "blueapi_cache" + + def can_access_cache(self) -> bool: + assert self._token_path + try: + self._create_parent_folder_if_necessary() + self._token_path.write_text("") + except IsADirectoryError: + print("Invalid path: a directory path was provided instead of a file path") + return False + except PermissionError: + print(f"Permission denied: Cannot write to {self._token_path.absolute()}") + return False + return True + + def _create_parent_folder_if_necessary(self): + self._token_path.parent.mkdir(parents=True, exist_ok=True) + + +class SessionManager: + def __init__(self, server_config: OIDCConfig, cache_manager: CacheManager) -> None: + self._server_config = server_config + self._cache_manager: CacheManager = cache_manager + + @classmethod + def from_cache(cls, auth_token_path: Path | None) -> SessionManager: + cache_manager = SessionCacheManager(auth_token_path) + cache = cache_manager.load_cache() + return SessionManager( + server_config=cache.oidc_config, cache_manager=cache_manager + ) + + def delete_cache(self) -> None: + self._cache_manager.delete_cache() + + def get_valid_access_token(self) -> str: + """ + Retrieves a valid access token. + + Returns: + str: A valid access token if successful. + "": If the operation fails (no valid token could be fetched or refreshed) + """ + try: + cache = self._cache_manager.load_cache() + self.decode_jwt(cache.access_token) + return cache.access_token + except jwt.ExpiredSignatureError: + cache = self._cache_manager.load_cache() + return self._refresh_auth_token(cache.refresh_token) + except Exception: + self.delete_cache() + return "" + + @cached_property + def client(self): + return jwt.PyJWKClient(self._server_config.jwks_uri) + + def decode_jwt(self, json_web_token: str): + signing_key = self.client.get_signing_key_from_jwt(json_web_token) + return jwt.decode( + json_web_token, + signing_key.key, + algorithms=self._server_config.id_token_signing_alg_values_supported, + verify=True, + audience=self._server_config.client_audience, + issuer=self._server_config.issuer, + ) + + def logout(self) -> None: + cache = self._cache_manager.load_cache() + self.delete_cache() + try: + response = requests.get( + self._server_config.end_session_endpoint, + params={ + "id_token_hint": cache.id_token, + "client_id": self._server_config.client_id, + }, + ) + response.raise_for_status() + print("Logged out") + except Exception as e: + print( + "An unexpected error occurred while attempting " + f"to log out from the server.{e}" + ) + + def _refresh_auth_token(self, refresh_token: str) -> str: + response = requests.post( + self._server_config.token_endpoint, + data={ + "client_id": self._server_config.client_id, + "grant_type": "refresh_token", + "refresh_token": refresh_token, + }, + headers={"Content-Type": "application/x-www-form-urlencoded"}, + ) + if response.status_code == HTTPStatus.OK: + token = response.json() + self._cache_manager.save_cache( + Cache( + oidc_config=self._server_config, + refresh_token=token["refresh_token"], + id_token=token["id_token"], + access_token=token["access_token"], + ), + ) + return token["access_token"] + else: + self.delete_cache() + return "" + + def poll_for_token( + self, device_code: str, polling_interval: float, expires_in: float + ) -> dict[str, Any]: + expiry_time: float = time.time() + expires_in + while time.time() < expiry_time: + response = requests.post( + self._server_config.token_endpoint, + data={ + "grant_type": "urn:ietf:params:oauth:grant-type:device_code", + "device_code": device_code, + "client_id": self._server_config.client_id, + }, + headers={"Content-Type": "application/x-www-form-urlencoded"}, + ) + if response.status_code == HTTPStatus.OK: + return response.json() + time.sleep(polling_interval) + + raise TimeoutError("Polling timed out") + + def start_device_flow(self): + assert self._cache_manager.can_access_cache() + print("Logging in") + response: requests.Response = requests.post( + self._server_config.device_authorization_endpoint, + data={ + "client_id": self._server_config.client_id, + "scope": SCOPES, + }, + headers={"Content-Type": "application/x-www-form-urlencoded"}, + ) + + response.raise_for_status() + + response_json: dict[str, Any] = response.json() + device_code = cast(str, response_json.get("device_code")) + interval = cast(float, response_json.get("interval")) + expires_in = cast(float, response_json.get("expires_in")) + webbrowser.open_new_tab(response_json["verification_uri_complete"]) + print( + f"Please login from this URL:- {response_json['verification_uri_complete']}" + ) + auth_token_json: dict[str, Any] = self.poll_for_token( + device_code, interval, expires_in + ) + self._cache_manager.save_cache( + Cache( + oidc_config=self._server_config, + refresh_token=auth_token_json["refresh_token"], + id_token=auth_token_json["id_token"], + access_token=auth_token_json["access_token"], + ) + ) + print("Logged in and cached new token") + + +class JWTAuth(AuthBase): + def __init__(self, session_manager: SessionManager | None): + self.token: str = ( + session_manager.get_valid_access_token() if session_manager else "" + ) + + def __call__(self, request): + if self.token: + request.headers["Authorization"] = f"Bearer {self.token}" + return request diff --git a/src/blueapi/core/context.py b/src/blueapi/core/context.py index c5383c94c7..fbcdb1d175 100644 --- a/src/blueapi/core/context.py +++ b/src/blueapi/core/context.py @@ -5,7 +5,14 @@ from importlib import import_module, metadata from inspect import Parameter, isclass, signature from types import ModuleType, NoneType, UnionType -from typing import Any, TypeVar, Union, get_args, get_origin, get_type_hints +from typing import ( + Any, + TypeVar, + Union, + get_args, + get_origin, + get_type_hints, +) from bluesky.protocols import HasName from bluesky.run_engine import RunEngine @@ -32,7 +39,6 @@ from blueapi.core.protocols import DeviceManager from blueapi.utils import ( BlueapiPlanModelConfig, - NumtrackerClient, is_function_sourced_from_module, load_module_all, ) @@ -125,7 +131,6 @@ class BlueskyContext: default_factory=lambda: RunEngine(context_managers=[], call_returns_result=True) ) tiled_conf: TiledConfig | None = field(default=None, init=False, repr=False) - numtracker: NumtrackerClient | None = field(default=None, init=False, repr=False) path_provider: PathProvider | None = None plans: dict[str, Plan] = field(default_factory=dict) devices: dict[str, Device] = field(default_factory=dict) @@ -137,10 +142,10 @@ def __post_init__(self, configuration: ApplicationConfig | None): if not configuration: return - if (nt_conf := configuration.numtracker) is not None: - if configuration.env.metadata is not None: - self.numtracker = NumtrackerClient(url=nt_conf.url) - else: + if configuration.numtracker is not None: + # The numtracker client itself is wired up by the service, see + # blueapi.service.interface.context + if configuration.env.metadata is None: raise InvalidConfigError( "Numtracker url has been configured, but there is no instrument or" " instrument_session in the environment metadata" @@ -152,20 +157,6 @@ def __post_init__(self, configuration: ApplicationConfig | None): self.run_engine.subscribe(path_provider.run_stop, "stop") self.path_provider = path_provider - # local reference so it's available in _update_scan_num - numtracker = self.numtracker - - async def _update_scan_num(md: dict[str, Any]) -> int: - scan = await numtracker.create_scan( - md["instrument_session"], md["instrument"] - ) - md["data_session_directory"] = str(scan.scan.directory.path) - md["detector_file_template"] = nt_conf.detector_file_template - md["scan_file"] = scan.scan.scan_file - return scan.scan.scan_number - - self.run_engine.scan_id_source = _update_scan_num - self.with_config(configuration.env) if (tiled_conf := configuration.tiled) is not None and tiled_conf.enabled: diff --git a/src/blueapi/log.py b/src/blueapi/log.py index 1f28ad70bb..7c1406f060 100644 --- a/src/blueapi/log.py +++ b/src/blueapi/log.py @@ -5,7 +5,6 @@ from copy import copy import click -from graypy import GELFTCPHandler from blueapi.config import LoggingConfig @@ -107,13 +106,15 @@ def set_up_stream_handler( def set_up_graylog_handler( logger: logging.Logger, logging_config: LoggingConfig, filters: list[logging.Filter] -) -> GELFTCPHandler: +) -> logging.Handler: """Creates and configures GELFTCPHandler, then attaches to logger. Args: logger: Logger to attach handler to logging_config: LoggingConfig """ + from graypy import GELFTCPHandler + assert logging_config.graylog.url.host is not None, "Graylog URL missing host" assert logging_config.graylog.url.port is not None, "Graylog URL missing port" graylog_handler = GELFTCPHandler( diff --git a/src/blueapi/service/__init__.py b/src/blueapi/service/__init__.py index ad8ccb1f62..a95592c635 100644 --- a/src/blueapi/service/__init__.py +++ b/src/blueapi/service/__init__.py @@ -1,4 +1,22 @@ -from .authentication import SessionManager +import logging + from .model import DeviceModel, PlanModel +LOGGER = logging.getLogger(__name__) + __all__ = ["PlanModel", "DeviceModel", "SessionManager"] + + +def __getattr__(name: str): + # Backwards compatible alias: SessionManager now lives in blueapi.client.session. + # Imported lazily as blueapi.client.session itself imports blueapi.service.model. + if name == "SessionManager": + from blueapi.client.session import SessionManager + + LOGGER.warning( + "Importing SessionManager from blueapi.service is deprecated, import it " + "from blueapi.client.session instead. The old import will be removed in " + "2 releases of blueapi or 6 months (May 2027)." + ) + return SessionManager + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/src/blueapi/service/authentication.py b/src/blueapi/service/authentication.py index 9177ad47fe..57e357c6fc 100644 --- a/src/blueapi/service/authentication.py +++ b/src/blueapi/service/authentication.py @@ -1,257 +1,24 @@ from __future__ import annotations -import base64 -import os import threading -import time -import webbrowser -from abc import ABC, abstractmethod from collections.abc import Mapping -from functools import cached_property -from http import HTTPStatus -from pathlib import Path -from typing import Annotated, Any, cast +from typing import Annotated, Any import httpx import jwt -import requests from fastapi import Depends, HTTPException from fastapi.requests import HTTPConnection from fastapi.security.utils import get_authorization_scheme_param -from pydantic import TypeAdapter -from requests.auth import AuthBase from starlette.status import HTTP_401_UNAUTHORIZED +# Re-exported for backwards compatibility, these now live in blueapi.client.session +from blueapi.client.session import ( # noqa: F401 + CacheManager, + JWTAuth, + SessionCacheManager, + SessionManager, +) from blueapi.config import OIDCConfig, ServiceAccount -from blueapi.service.model import Cache - -DEFAULT_CACHE_DIR = "~/.cache/" -SCOPES = "openid offline_access" - - -class CacheManager(ABC): - @abstractmethod - def can_access_cache(self) -> bool: ... - @abstractmethod - def save_cache(self, cache: Cache) -> None: ... - @abstractmethod - def load_cache(self) -> Cache: ... - @abstractmethod - def delete_cache(self) -> None: ... - - -class SessionCacheManager(CacheManager): - def __init__(self, token_path: Path | None) -> None: - self._token_path: Path = ( - token_path if token_path else self._default_token_cache_path() - ) - - @cached_property - def _file_path(self) -> str: - return os.path.expanduser(self._token_path) - - def save_cache(self, cache: Cache) -> None: - self.delete_cache() - self._create_parent_folder_if_necessary() - with open(self._file_path, "xb") as token_file: - token_file.write(base64.b64encode(cache.model_dump_json().encode("utf-8"))) - os.chmod(self._file_path, 0o600) - - def load_cache(self) -> Cache: - self._create_parent_folder_if_necessary() - with open(self._file_path, "rb") as cache_file: - return TypeAdapter(Cache).validate_json( - base64.b64decode(cache_file.read()).decode("utf-8") - ) - - def delete_cache(self) -> None: - Path(self._file_path).unlink(missing_ok=True) - - @staticmethod - def _default_token_cache_path() -> Path: - """ - Return the default cache file path. - """ - cache_path = os.environ.get("XDG_CACHE_HOME", DEFAULT_CACHE_DIR) - return Path(cache_path).expanduser() / "blueapi_cache" - - def can_access_cache(self) -> bool: - assert self._token_path - try: - self._create_parent_folder_if_necessary() - self._token_path.write_text("") - except IsADirectoryError: - print("Invalid path: a directory path was provided instead of a file path") - return False - except PermissionError: - print(f"Permission denied: Cannot write to {self._token_path.absolute()}") - return False - return True - - def _create_parent_folder_if_necessary(self): - self._token_path.parent.mkdir(parents=True, exist_ok=True) - - -class SessionManager: - def __init__(self, server_config: OIDCConfig, cache_manager: CacheManager) -> None: - self._server_config = server_config - self._cache_manager: CacheManager = cache_manager - - @classmethod - def from_cache(cls, auth_token_path: Path | None) -> SessionManager: - cache_manager = SessionCacheManager(auth_token_path) - cache = cache_manager.load_cache() - return SessionManager( - server_config=cache.oidc_config, cache_manager=cache_manager - ) - - def delete_cache(self) -> None: - self._cache_manager.delete_cache() - - def get_valid_access_token(self) -> str: - """ - Retrieves a valid access token. - - Returns: - str: A valid access token if successful. - "": If the operation fails (no valid token could be fetched or refreshed) - """ - try: - cache = self._cache_manager.load_cache() - self.decode_jwt(cache.access_token) - return cache.access_token - except jwt.ExpiredSignatureError: - cache = self._cache_manager.load_cache() - return self._refresh_auth_token(cache.refresh_token) - except Exception: - self.delete_cache() - return "" - - @cached_property - def client(self): - return jwt.PyJWKClient(self._server_config.jwks_uri) - - def decode_jwt(self, json_web_token: str): - signing_key = self.client.get_signing_key_from_jwt(json_web_token) - return jwt.decode( - json_web_token, - signing_key.key, - algorithms=self._server_config.id_token_signing_alg_values_supported, - verify=True, - audience=self._server_config.client_audience, - issuer=self._server_config.issuer, - ) - - def logout(self) -> None: - cache = self._cache_manager.load_cache() - self.delete_cache() - try: - response = requests.get( - self._server_config.end_session_endpoint, - params={ - "id_token_hint": cache.id_token, - "client_id": self._server_config.client_id, - }, - ) - response.raise_for_status() - print("Logged out") - except Exception as e: - print( - "An unexpected error occurred while attempting " - f"to log out from the server.{e}" - ) - - def _refresh_auth_token(self, refresh_token: str) -> str: - response = requests.post( - self._server_config.token_endpoint, - data={ - "client_id": self._server_config.client_id, - "grant_type": "refresh_token", - "refresh_token": refresh_token, - }, - headers={"Content-Type": "application/x-www-form-urlencoded"}, - ) - if response.status_code == HTTPStatus.OK: - token = response.json() - self._cache_manager.save_cache( - Cache( - oidc_config=self._server_config, - refresh_token=token["refresh_token"], - id_token=token["id_token"], - access_token=token["access_token"], - ), - ) - return token["access_token"] - else: - self.delete_cache() - return "" - - def poll_for_token( - self, device_code: str, polling_interval: float, expires_in: float - ) -> dict[str, Any]: - expiry_time: float = time.time() + expires_in - while time.time() < expiry_time: - response = requests.post( - self._server_config.token_endpoint, - data={ - "grant_type": "urn:ietf:params:oauth:grant-type:device_code", - "device_code": device_code, - "client_id": self._server_config.client_id, - }, - headers={"Content-Type": "application/x-www-form-urlencoded"}, - ) - if response.status_code == HTTPStatus.OK: - return response.json() - time.sleep(polling_interval) - - raise TimeoutError("Polling timed out") - - def start_device_flow(self): - assert self._cache_manager.can_access_cache() - print("Logging in") - response: requests.Response = requests.post( - self._server_config.device_authorization_endpoint, - data={ - "client_id": self._server_config.client_id, - "scope": SCOPES, - }, - headers={"Content-Type": "application/x-www-form-urlencoded"}, - ) - - response.raise_for_status() - - response_json: dict[str, Any] = response.json() - device_code = cast(str, response_json.get("device_code")) - interval = cast(float, response_json.get("interval")) - expires_in = cast(float, response_json.get("expires_in")) - webbrowser.open_new_tab(response_json["verification_uri_complete"]) - print( - f"Please login from this URL:- {response_json['verification_uri_complete']}" - ) - auth_token_json: dict[str, Any] = self.poll_for_token( - device_code, interval, expires_in - ) - self._cache_manager.save_cache( - Cache( - oidc_config=self._server_config, - refresh_token=auth_token_json["refresh_token"], - id_token=auth_token_json["id_token"], - access_token=auth_token_json["access_token"], - ) - ) - print("Logged in and cached new token") - - -class JWTAuth(AuthBase): - def __init__(self, session_manager: SessionManager | None): - self.token: str = ( - session_manager.get_valid_access_token() if session_manager else "" - ) - - def __call__(self, request): - if self.token: - request.headers["Authorization"] = f"Bearer {self.token}" - return request class TiledAuth(httpx.Auth): diff --git a/src/blueapi/service/environment.py b/src/blueapi/service/environment.py new file mode 100644 index 0000000000..1fca3ad583 --- /dev/null +++ b/src/blueapi/service/environment.py @@ -0,0 +1,97 @@ +import importlib.metadata +import tomllib +from pathlib import Path + +from git import Repo + +from blueapi.config import ScratchConfig +from blueapi.service.model import PackageInfo, PythonEnvironmentResponse, SourceInfo +from blueapi.utils import validate_directory + + +def _get_project_name_from_pyproject(path: Path) -> str: + pyproject_path = path / "pyproject.toml" + if pyproject_path.exists(): + with pyproject_path.open("rb") as file: + toml_data = tomllib.load(file) + return toml_data.get("project", {}).get("name", "") + return "" + + +def _fetch_installed_packages_details() -> list[PackageInfo]: + installed_packages = importlib.metadata.distributions() + return [ + PackageInfo( + name=dist.metadata["Name"], + version=dist.version, + location=str(dist.locate_file("")), + is_dirty=False, + ) + for dist in installed_packages + ] + + +def get_python_environment( + config: ScratchConfig | None, + name: str | None = None, + source: SourceInfo | None = None, +) -> PythonEnvironmentResponse: + """ + Get the Python environment. This includes all installed packages and + the scratch packages. + """ + scratch_packages = {} + packages = [] + + if config is None: + python_env_response = PythonEnvironmentResponse(scratch_enabled=False) + else: + python_env_response = PythonEnvironmentResponse(scratch_enabled=True) + validate_directory(config.root) + for repo in config.repositories: + local_directory = config.root / repo.name + repo = Repo(local_directory) + try: + branch = repo.active_branch.name + except TypeError: + branch = repo.head.commit.hexsha + + is_dirty = repo.is_dirty() + + version = ( + f"{repo.remotes[0].url} @{branch}" + if repo.remotes + else f"UNKNOWN REMOTE @{branch}" + ) + package_name = _get_project_name_from_pyproject(local_directory) + package_location = "" + + packages.append( + PackageInfo( + name=package_name, + version=version, + location=package_location, + source=SourceInfo.SCRATCH, + is_dirty=is_dirty, + ) + ) + scratch_packages = {p.name: p for p in packages} + + for pkg in _fetch_installed_packages_details(): + if pkg.name not in scratch_packages: + packages.append(pkg) + else: + scratch_packages[pkg.name].location += f"{pkg.location} &&" + + python_env_response.installed_packages = sorted( + packages, key=lambda pkg: pkg.name.lower() + ) + if name: + python_env_response.installed_packages = [ + p for p in python_env_response.installed_packages if p.name == name + ] + if source: + python_env_response.installed_packages = [ + p for p in python_env_response.installed_packages if p.source == source + ] + return python_env_response diff --git a/src/blueapi/service/interface.py b/src/blueapi/service/interface.py index 7df4745570..946dec8658 100644 --- a/src/blueapi/service/interface.py +++ b/src/blueapi/service/interface.py @@ -13,13 +13,13 @@ from tiled.client import from_uri from tiled.client.utils import ClientError -from blueapi.cli.scratch import get_python_environment from blueapi.config import ApplicationConfig, OIDCConfig, ServiceAccount, StompConfig from blueapi.core.bluesky_types import DataEvent from blueapi.core.context import BlueskyContext from blueapi.core.event import EventStream from blueapi.log import set_up_logging from blueapi.service.authentication import TiledAuth +from blueapi.service.environment import get_python_environment from blueapi.service.model import ( DeviceModel, PlanModel, @@ -28,6 +28,7 @@ TaskRequest, WorkerTask, ) +from blueapi.service.numtracker import NumtrackerClient from blueapi.utils.serialization import access_blob from blueapi.worker.event import ProgressEvent, TaskStatusEnum, WorkerEvent, WorkerState from blueapi.worker.task import Task @@ -50,9 +51,26 @@ def set_config(new_config: ApplicationConfig): _CONFIG = new_config +@cache +def numtracker() -> NumtrackerClient | None: + if (nt_conf := config().numtracker) is None: + return None + return NumtrackerClient(url=nt_conf.url) + + @cache def context() -> BlueskyContext: ctx = BlueskyContext(config()) + if (nt_conf := config().numtracker) is not None and (nt := numtracker()): + + async def _update_scan_num(md: dict[str, Any]) -> int: + scan = await nt.create_scan(md["instrument_session"], md["instrument"]) + md["data_session_directory"] = str(scan.scan.directory.path) + md["detector_file_template"] = nt_conf.detector_file_template + md["scan_file"] = scan.scan.scan_file + return scan.scan.scan_number + + ctx.run_engine.scan_id_source = _update_scan_num return ctx @@ -157,6 +175,7 @@ def teardown() -> None: if (stomp_client_ref := stomp_client()) is not None: stomp_client_ref.disconnect() context.cache_clear() + numtracker.cache_clear() worker.cache_clear() stomp_client.cache_clear() service_account_tiled_client.cache_clear() @@ -239,7 +258,7 @@ def begin_task( active_worker = worker() active_context = context() - if nt := active_context.numtracker: + if nt := numtracker(): nt.set_headers(pass_through_headers or {}) if task.task_id is None: diff --git a/src/blueapi/utils/numtracker.py b/src/blueapi/service/numtracker.py similarity index 100% rename from src/blueapi/utils/numtracker.py rename to src/blueapi/service/numtracker.py diff --git a/src/blueapi/utils/__init__.py b/src/blueapi/utils/__init__.py index 1c0c660f4f..1cbe561e85 100644 --- a/src/blueapi/utils/__init__.py +++ b/src/blueapi/utils/__init__.py @@ -5,10 +5,9 @@ from typing import Any, ParamSpec, TypeVar from .base_model import BlueapiBaseModel, BlueapiModelConfig, BlueapiPlanModelConfig -from .file_permissions import get_owner_gid, is_sgid_set +from .file_permissions import get_owner_gid, is_sgid_set, validate_directory from .invalid_config_error import InvalidConfigError from .modules import is_function_sourced_from_module, load_module_all -from .numtracker import NumtrackerClient from .serialization import serialize from .thread_exception import handle_all_exceptions @@ -20,10 +19,10 @@ "BlueapiModelConfig", "BlueapiPlanModelConfig", "InvalidConfigError", - "NumtrackerClient", "report_successful_devices", "is_sgid_set", "get_owner_gid", + "validate_directory", "is_function_sourced_from_module", "deprecated", ] diff --git a/src/blueapi/utils/file_permissions.py b/src/blueapi/utils/file_permissions.py index 0e1921f6d6..ae10988651 100644 --- a/src/blueapi/utils/file_permissions.py +++ b/src/blueapi/utils/file_permissions.py @@ -30,3 +30,10 @@ def get_owner_gid(path: Path) -> int: """ return path.stat().st_gid + + +def validate_directory(path: Path) -> None: + if not path.exists(): + raise KeyError(f"{path}: No such file or directory") + elif path.is_file(): + raise KeyError(f"{path}: Is a file, not a directory") diff --git a/tests/conftest.py b/tests/conftest.py index 6f517b3bf9..52c99ee139 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,7 +13,6 @@ import yaml from bluesky._vendor.super_state_machine.errors import TransitionError # type: ignore from bluesky.run_engine import RunEngine -from fastapi import status from jwcrypto.jwk import JWK from observability_utils.tracing import JsonObjectSpanExporter, setup_tracing from opentelemetry.sdk.trace import TracerProvider @@ -330,9 +329,7 @@ def mock_authn_server( @pytest.fixture def mock_unauthenticated_server(): requests_mock = responses.RequestsMock(assert_all_requests_are_fired=False) - requests_mock.get( - "http://localhost:8000/config/oidc", status=status.HTTP_204_NO_CONTENT - ) + requests_mock.get("http://localhost:8000/config/oidc", status=204) with requests_mock: yield requests_mock diff --git a/tests/system_tests/test_blueapi_client.py b/tests/system_tests/test_blueapi_client.py index f16e267cf4..32bbe484ac 100644 --- a/tests/system_tests/test_blueapi_client.py +++ b/tests/system_tests/test_blueapi_client.py @@ -51,7 +51,7 @@ def login(token_path: Path, user: User) -> BlueapiClient: config_file.write_text(yaml.dump(config)) with patch( - "blueapi.service.authentication.webbrowser.open_new_tab", + "blueapi.client.session.webbrowser.open_new_tab", side_effect=lambda url: browser_login(url, user), ): BlueapiClient.from_config_file(str(config_file)).login(token_path) diff --git a/tests/system_tests/test_blueapi_system.py b/tests/system_tests/test_blueapi_system.py index 9feda470c6..f393c7ab8c 100644 --- a/tests/system_tests/test_blueapi_system.py +++ b/tests/system_tests/test_blueapi_system.py @@ -159,7 +159,7 @@ def get_access_token(user: ValidUser) -> str: @pytest.fixture(scope="module") def client_without_auth() -> Generator[BlueapiClient]: with patch( - "blueapi.service.authentication.SessionManager.from_cache", + "blueapi.client.session.SessionManager.from_cache", return_value=None, ): yield BlueapiClient.from_config(config=ApplicationConfig()) @@ -169,7 +169,7 @@ def patch_session(user: ValidUser): mock_session_manager = MagicMock() mock_session_manager.get_valid_access_token.return_value = get_access_token(user) return patch( - "blueapi.service.authentication.SessionManager.from_cache", + "blueapi.client.session.SessionManager.from_cache", return_value=mock_session_manager, ) diff --git a/tests/unit_tests/cli/test_scratch.py b/tests/unit_tests/cli/test_scratch.py index 22f3ad3d8f..a7df496186 100644 --- a/tests/unit_tests/cli/test_scratch.py +++ b/tests/unit_tests/cli/test_scratch.py @@ -3,21 +3,13 @@ import uuid from collections.abc import Generator from pathlib import Path -from unittest.mock import ANY, MagicMock, Mock, PropertyMock, call, patch +from unittest.mock import ANY, MagicMock, Mock, call, patch import pytest from git import Repo -from blueapi.cli.scratch import ( - _fetch_installed_packages_details, - _get_project_name_from_pyproject, - ensure_repo, - get_python_environment, - scratch_install, - setup_scratch, -) +from blueapi.cli.scratch import ensure_repo, scratch_install, setup_scratch from blueapi.config import ScratchConfig, ScratchRepository -from blueapi.service.model import PackageInfo, SourceInfo from blueapi.utils import get_owner_gid @@ -383,258 +375,3 @@ def test_setup_scratch_continues_after_failure( RuntimeError, match="Failed to clone", check=lambda e: str(e.__cause__) == "bar" ): setup_scratch(config) - - -@pytest.fixture -def config(directory_path_with_sgid: Path) -> ScratchConfig: - return ScratchConfig( - root=directory_path_with_sgid, - repositories=[ - ScratchRepository( - name="foo", - remote_url="http://example.com/foo.git", - ), - ScratchRepository( - name="bar", - remote_url="http://example.com/bar.git", - ), - ], - ) - - -@patch("blueapi.cli.scratch.Repo") -@patch("blueapi.cli.scratch._fetch_installed_packages_details") -@patch("blueapi.cli.scratch._get_project_name_from_pyproject") -def test_get_python_env_returns_correct_packages( - mock_get_project_name: Mock, - mock_fetch_installed_packages: Mock, - mock_repo: Mock, - directory_path_with_sgid: Path, - config: ScratchConfig, -): - repo_path = directory_path_with_sgid / "foo" - repo_path.mkdir() - mock_repo_1 = Mock() - mock_repo_1.active_branch.name = "main" - mock_repo_1.is_dirty.return_value = False - mock_repo_1.remotes = [Mock(url="http://example.com/foo.git")] - - repo_path = directory_path_with_sgid / "bar" - repo_path.mkdir() - mock_repo_2 = Mock() - type(mock_repo_2.active_branch).name = PropertyMock(side_effect=TypeError) - mock_repo_2.head.commit.hexsha = "adsad23123" - mock_repo_2.is_dirty.return_value = True - mock_repo_2.remotes = [Mock(url="http://example.com/bar.git")] - - mock_repo.side_effect = [mock_repo_1, mock_repo_2] - - mock_get_project_name.side_effect = ["foo-package", "bar-package"] - mock_fetch_installed_packages.return_value = [ - PackageInfo( - name="package-01", - version="1.0.1", - location="/some/location", - is_dirty=False, - ) - ] - - response = get_python_environment(config) - - assert response.installed_packages == [ - PackageInfo( - name="bar-package", - version="http://example.com/bar.git @adsad23123", - location="", - is_dirty=True, - source=SourceInfo.SCRATCH, - ), - PackageInfo( - name="foo-package", - version="http://example.com/foo.git @main", - location="", - is_dirty=False, - source=SourceInfo.SCRATCH, - ), - PackageInfo( - name="package-01", - version="1.0.1", - location="/some/location", - is_dirty=False, - source=SourceInfo.PYPI, - ), - ] - - -@patch("blueapi.cli.scratch.Repo") -@patch("blueapi.cli.scratch._fetch_installed_packages_details") -@patch("blueapi.cli.scratch._get_project_name_from_pyproject") -def test_fetch_python_env_with_identical_packages( - mock_get_project_name: Mock, - mock_fetch_installed_packages: Mock, - mock_repo: Mock, - directory_path_with_sgid: Path, -): - repo_path = directory_path_with_sgid / "foo" - repo_path.mkdir() - mock_repo_instance = Mock() - mock_repo_instance.active_branch.name = "main" - mock_repo_instance.is_dirty.return_value = False - mock_repo_instance.remotes = [Mock(url="http://example.com/foo.git")] - - mock_repo.return_value = mock_repo_instance - - mock_get_project_name.return_value = "foo-package" - mock_fetch_installed_packages.return_value = [ - PackageInfo( - name="foo-package", - version="http://example.com/foo.git @main", - location="/some/location", - is_dirty=False, - source=SourceInfo.SCRATCH, - ) - ] - config = ScratchConfig( - root=directory_path_with_sgid, - repositories=[ - ScratchRepository( - name="foo", - remote_url="http://example.com/foo.git", - ), - ], - ) - response = get_python_environment(config) - - assert response.installed_packages == [ - PackageInfo( - name="foo-package", - version="http://example.com/foo.git @main", - location="/some/location &&", - is_dirty=False, - source=SourceInfo.SCRATCH, - ), - ] - - -@patch("blueapi.cli.scratch.importlib.metadata.distributions") -def test_fetch_installed_packages_details_returns_correct_packages(mock_distributions): - mock_distribution = Mock() - mock_distribution.metadata = {"Name": "example-package"} - mock_distribution.version = "1.0.0" - mock_distribution.locate_file.return_value = Path("/example/location") - mock_distributions.return_value = [mock_distribution] - - packages = _fetch_installed_packages_details() - - assert len(packages) == 1 - assert packages == [ - PackageInfo( - name="example-package", - version="1.0.0", - location="/example/location", - is_dirty=False, - ) - ] - - -@patch("blueapi.cli.scratch.Repo") -@patch("blueapi.cli.scratch._fetch_installed_packages_details") -@patch("blueapi.cli.scratch._get_project_name_from_pyproject") -def test_get_python_env_filters_by_name_and_source( - mock_get_project_name: Mock, - mock_fetch_installed_packages: Mock, - mock_repo: Mock, - directory_path_with_sgid: Path, -): - # Setup for scratch source filtering - repo_path = directory_path_with_sgid / "foo" - repo_path.mkdir() - mock_repo_instance = Mock() - mock_repo_instance.active_branch.name = "main" - mock_repo_instance.is_dirty.return_value = False - mock_repo_instance.remotes = [Mock(url="http://example.com/foo.git")] - mock_repo.return_value = mock_repo_instance - - mock_get_project_name.return_value = "foo-package" - mock_fetch_installed_packages.return_value = [ - PackageInfo( - name="bar-package", - version="1.0.0", - location="/some/location", - is_dirty=False, - source=SourceInfo.PYPI, - ) - ] - config = ScratchConfig( - root=directory_path_with_sgid, - repositories=[ - ScratchRepository( - name="foo", - remote_url="http://example.com/foo.git", - ), - ], - ) - # Test filtering by name - response_by_name = get_python_environment(config, name="foo-package") - assert response_by_name.installed_packages == [ - PackageInfo( - name="foo-package", - version="http://example.com/foo.git @main", - location="", - is_dirty=False, - source=SourceInfo.SCRATCH, - ) - ] - - # Test filtering by source - response_by_source = get_python_environment(config, source=SourceInfo.SCRATCH) - assert response_by_source.installed_packages == [ - PackageInfo( - name="foo-package", - version="http://example.com/foo.git @main", - location="", - is_dirty=False, - source=SourceInfo.SCRATCH, - ) - ] - - -@pytest.fixture -def pyproject_file(tmp_path: Path) -> Generator[Path]: - pyproject_path = tmp_path / "pyproject.toml" - with pyproject_path.open("w") as f: - f.write( - """ - [project] - name = "example-project" - """ - ) - yield pyproject_path - os.remove(pyproject_path) - - -def test_get_project_name_from_pyproject_returns_name(pyproject_file: Path): - project_name = _get_project_name_from_pyproject(pyproject_file.parent) - assert project_name == "example-project" - - -def test_get_project_name_from_pyproject_returns_empty_if_no_pyproject( - tmp_path: Path, -): - project_name = _get_project_name_from_pyproject(tmp_path) - assert project_name == "" - - -def test_get_project_name_from_pyproject_returns_empty_if_no_name_key( - tmp_path: Path, -): - pyproject_path = tmp_path / "pyproject.toml" - with pyproject_path.open("w") as f: - f.write( - """ - [project] - version = "1.0.0" - """ - ) - project_name = _get_project_name_from_pyproject(tmp_path) - assert project_name == "" diff --git a/tests/unit_tests/client/test_rest.py b/tests/unit_tests/client/test_rest.py index b756b0b75f..a36ee71a3f 100644 --- a/tests/unit_tests/client/test_rest.py +++ b/tests/unit_tests/client/test_rest.py @@ -28,8 +28,8 @@ _create_task_exceptions, _exception, ) +from blueapi.client.session import SessionCacheManager, SessionManager from blueapi.config import OIDCConfig -from blueapi.service.authentication import SessionCacheManager, SessionManager from blueapi.service.model import ( DeviceModel, EnvironmentResponse, diff --git a/tests/unit_tests/client/test_session.py b/tests/unit_tests/client/test_session.py new file mode 100644 index 0000000000..f54334fdc4 --- /dev/null +++ b/tests/unit_tests/client/test_session.py @@ -0,0 +1,120 @@ +import os +from pathlib import Path +from typing import Any +from unittest.mock import patch + +import pytest +import responses +from starlette.status import HTTP_403_FORBIDDEN + +from blueapi.client.session import SessionCacheManager, SessionManager +from blueapi.config import OIDCConfig + + +@pytest.fixture +def auth_token_path(tmp_path) -> Path: + return tmp_path / "blueapi_cache" + + +@pytest.fixture +def session_manager( + oidc_config: OIDCConfig, + auth_token_path, + mock_authn_server: responses.RequestsMock, +) -> SessionManager: + return SessionManager( + server_config=oidc_config, cache_manager=SessionCacheManager(auth_token_path) + ) + + +def test_logout( + session_manager: SessionManager, + oidc_config: OIDCConfig, + cached_valid_token: Path, + auth_token_path: Path, +): + assert os.path.exists(auth_token_path) + session_manager.logout() + assert not os.path.exists(auth_token_path) + + +def test_refresh_auth_token( + mock_authn_server: responses.RequestsMock, + session_manager: SessionManager, + cached_valid_refresh: Path, +): + token = session_manager.get_valid_access_token() + assert token == "new_token" + + +def test_get_empty_token_if_no_cache(session_manager: SessionManager): + token = session_manager.get_valid_access_token() + assert token == "" + + +def test_get_empty_token_if_refresh_fails( + mock_authn_server: responses.RequestsMock, + session_manager: SessionManager, + cached_expired_refresh: Path, +): + assert cached_expired_refresh.exists() + token = session_manager.get_valid_access_token() + assert token == "" + assert not cached_expired_refresh.exists() + + +def test_get_empty_token_if_invalid_cache( + mock_authn_server: responses.RequestsMock, + session_manager: SessionManager, + cache_with_invalid_audience: Path, +): + token = session_manager.get_valid_access_token() + assert token == "" + + +def test_get_empty_token_if_exception_in_decode( + mock_authn_server: responses.RequestsMock, + session_manager: SessionManager, + cached_expired_refresh: Path, +): + assert cached_expired_refresh.exists() + token = session_manager.get_valid_access_token() + assert token == "" + assert not cached_expired_refresh.exists() + + +def test_poll_for_token( + mock_authn_server: responses.RequestsMock, + session_manager: SessionManager, + valid_token: dict[str, Any], + device_code: str, +): + token = session_manager.poll_for_token(device_code, 1, 2) + assert token == valid_token + + +@patch("blueapi.client.session.time.sleep", return_value=None) +def test_poll_for_token_timeout( + mock_sleep, + oidc_well_known: dict[str, Any], + mock_authn_server: responses.RequestsMock, + session_manager: SessionManager, + device_code: str, +): + mock_authn_server.stop() + mock_authn_server.remove(responses.POST, oidc_well_known["token_endpoint"]) + mock_authn_server.post( + url=oidc_well_known["token_endpoint"], + json={"error": "authorization_pending"}, + status=HTTP_403_FORBIDDEN, + ) + with pytest.raises(TimeoutError), mock_authn_server: + session_manager.poll_for_token(device_code, 0.01, 0.01) + + +def test_session_cache_manager_returns_writable_file_path(tmp_path): + os.environ["XDG_CACHE_HOME"] = str(tmp_path) + cache = SessionCacheManager(token_path=None) + Path(cache._file_path).touch() + assert os.path.isfile(cache._file_path) + assert cache._file_path == f"{tmp_path}/blueapi_cache" diff --git a/tests/unit_tests/service/test_authentication.py b/tests/unit_tests/service/test_authentication.py index a763758122..9e117b92fe 100644 --- a/tests/unit_tests/service/test_authentication.py +++ b/tests/unit_tests/service/test_authentication.py @@ -1,6 +1,3 @@ -import os -from pathlib import Path -from typing import Any from unittest.mock import Mock, patch import httpx @@ -10,13 +7,13 @@ import respx from fastapi import HTTPException from pydantic import SecretStr -from starlette.status import HTTP_200_OK, HTTP_403_FORBIDDEN +from starlette.status import HTTP_200_OK +from blueapi import service +from blueapi.client.session import SessionManager from blueapi.config import OIDCConfig, ServiceAccount from blueapi.service import authentication from blueapi.service.authentication import ( - SessionCacheManager, - SessionManager, TiledAuth, access_token, build_access_token_check, @@ -24,105 +21,10 @@ ) -@pytest.fixture -def auth_token_path(tmp_path) -> Path: - return tmp_path / "blueapi_cache" - - -@pytest.fixture -def session_manager( - oidc_config: OIDCConfig, - auth_token_path, - mock_authn_server: responses.RequestsMock, -) -> SessionManager: - return SessionManager( - server_config=oidc_config, cache_manager=SessionCacheManager(auth_token_path) - ) - - -def test_logout( - session_manager: SessionManager, - oidc_config: OIDCConfig, - cached_valid_token: Path, - auth_token_path: Path, -): - assert os.path.exists(auth_token_path) - session_manager.logout() - assert not os.path.exists(auth_token_path) - - -def test_refresh_auth_token( - mock_authn_server: responses.RequestsMock, - session_manager: SessionManager, - cached_valid_refresh: Path, -): - token = session_manager.get_valid_access_token() - assert token == "new_token" - - -def test_get_empty_token_if_no_cache(session_manager: SessionManager): - token = session_manager.get_valid_access_token() - assert token == "" - - -def test_get_empty_token_if_refresh_fails( - mock_authn_server: responses.RequestsMock, - session_manager: SessionManager, - cached_expired_refresh: Path, -): - assert cached_expired_refresh.exists() - token = session_manager.get_valid_access_token() - assert token == "" - assert not cached_expired_refresh.exists() - - -def test_get_empty_token_if_invalid_cache( - mock_authn_server: responses.RequestsMock, - session_manager: SessionManager, - cache_with_invalid_audience: Path, -): - token = session_manager.get_valid_access_token() - assert token == "" - - -def test_get_empty_token_if_exception_in_decode( - mock_authn_server: responses.RequestsMock, - session_manager: SessionManager, - cached_expired_refresh: Path, -): - assert cached_expired_refresh.exists() - token = session_manager.get_valid_access_token() - assert token == "" - assert not cached_expired_refresh.exists() - - -def test_poll_for_token( - mock_authn_server: responses.RequestsMock, - session_manager: SessionManager, - valid_token: dict[str, Any], - device_code: str, -): - token = session_manager.poll_for_token(device_code, 1, 2) - assert token == valid_token - - -@patch("blueapi.service.authentication.time.sleep", return_value=None) -def test_poll_for_token_timeout( - mock_sleep, - oidc_well_known: dict[str, Any], - mock_authn_server: responses.RequestsMock, - session_manager: SessionManager, - device_code: str, -): - mock_authn_server.stop() - mock_authn_server.remove(responses.POST, oidc_well_known["token_endpoint"]) - mock_authn_server.post( - url=oidc_well_known["token_endpoint"], - json={"error": "authorization_pending"}, - status=HTTP_403_FORBIDDEN, - ) - with pytest.raises(TimeoutError), mock_authn_server: - session_manager.poll_for_token(device_code, 0.01, 0.01) +def test_session_manager_old_import_logs_deprecation(caplog: pytest.LogCaptureFixture): + assert service.SessionManager is SessionManager + assert "import it from blueapi.client.session" in caplog.text + assert "2 releases of blueapi or 6 months" in caplog.text def test_server_raises_exception_for_invalid_token( @@ -142,52 +44,6 @@ def test_processes_valid_token( inner(Mock(), token=valid_token_with_jwt["access_token"]) -def test_session_cache_manager_returns_writable_file_path(tmp_path): - os.environ["XDG_CACHE_HOME"] = str(tmp_path) - cache = SessionCacheManager(token_path=None) - Path(cache._file_path).touch() - assert os.path.isfile(cache._file_path) - assert cache._file_path == f"{tmp_path}/blueapi_cache" - - -def test_tiled_auth_raises_exception(): - with pytest.raises( - RuntimeError, match="Token URL is not set please check oidc config" - ): - auth = ServiceAccount() - TiledAuth(tiled_auth=auth) - - -@respx.mock -def test_tiled_auth_sync_auth_flow(): - client_id = "client" - client_secret = SecretStr("secret") - token_url = "http://keycloak.com/token" - access_token = "access_token" - - respx.post(token_url).mock( - return_value=httpx.Response( - status_code=HTTP_200_OK, json={"access_token": access_token} - ) - ) - - tiled_auth = TiledAuth( - tiled_auth=ServiceAccount( - client_id=client_id, - client_secret=client_secret, - token_url=token_url, - ) - ) - - request = Mock() - request.headers = {} - - flow = tiled_auth.sync_auth_flow(request) - result = next(flow) - - assert result.headers["Authorization"] == f"Bearer {access_token}" - - @pytest.mark.parametrize( "header,cookie,token", [ @@ -236,3 +92,41 @@ def test_build_access_token(mock_jwt: Mock): validate_fn(req, token=None) mock_jwt.decode.assert_not_called() + + +def test_tiled_auth_raises_exception(): + with pytest.raises( + RuntimeError, match="Token URL is not set please check oidc config" + ): + auth = ServiceAccount() + TiledAuth(tiled_auth=auth) + + +@respx.mock +def test_tiled_auth_sync_auth_flow(): + client_id = "client" + client_secret = SecretStr("secret") + token_url = "http://keycloak.com/token" + access_token = "access_token" + + respx.post(token_url).mock( + return_value=httpx.Response( + status_code=HTTP_200_OK, json={"access_token": access_token} + ) + ) + + tiled_auth = TiledAuth( + tiled_auth=ServiceAccount( + client_id=client_id, + client_secret=client_secret, + token_url=token_url, + ) + ) + + request = Mock() + request.headers = {} + + flow = tiled_auth.sync_auth_flow(request) + result = next(flow) + + assert result.headers["Authorization"] == f"Bearer {access_token}" diff --git a/tests/unit_tests/service/test_environment.py b/tests/unit_tests/service/test_environment.py new file mode 100644 index 0000000000..5ce15b283f --- /dev/null +++ b/tests/unit_tests/service/test_environment.py @@ -0,0 +1,274 @@ +import os +from collections.abc import Generator +from pathlib import Path +from unittest.mock import Mock, PropertyMock, patch + +import pytest + +from blueapi.config import ScratchConfig, ScratchRepository +from blueapi.service.environment import ( + _fetch_installed_packages_details, + _get_project_name_from_pyproject, + get_python_environment, +) +from blueapi.service.model import PackageInfo, SourceInfo + + +@pytest.fixture +def directory_path_with_sgid(tmp_path: Path) -> Path: + return tmp_path + + +@pytest.fixture +def config(directory_path_with_sgid: Path) -> ScratchConfig: + return ScratchConfig( + root=directory_path_with_sgid, + repositories=[ + ScratchRepository( + name="foo", + remote_url="http://example.com/foo.git", + ), + ScratchRepository( + name="bar", + remote_url="http://example.com/bar.git", + ), + ], + ) + + +@patch("blueapi.service.environment.Repo") +@patch("blueapi.service.environment._fetch_installed_packages_details") +@patch("blueapi.service.environment._get_project_name_from_pyproject") +def test_get_python_env_returns_correct_packages( + mock_get_project_name: Mock, + mock_fetch_installed_packages: Mock, + mock_repo: Mock, + directory_path_with_sgid: Path, + config: ScratchConfig, +): + repo_path = directory_path_with_sgid / "foo" + repo_path.mkdir() + mock_repo_1 = Mock() + mock_repo_1.active_branch.name = "main" + mock_repo_1.is_dirty.return_value = False + mock_repo_1.remotes = [Mock(url="http://example.com/foo.git")] + + repo_path = directory_path_with_sgid / "bar" + repo_path.mkdir() + mock_repo_2 = Mock() + type(mock_repo_2.active_branch).name = PropertyMock(side_effect=TypeError) + mock_repo_2.head.commit.hexsha = "adsad23123" + mock_repo_2.is_dirty.return_value = True + mock_repo_2.remotes = [Mock(url="http://example.com/bar.git")] + + mock_repo.side_effect = [mock_repo_1, mock_repo_2] + + mock_get_project_name.side_effect = ["foo-package", "bar-package"] + mock_fetch_installed_packages.return_value = [ + PackageInfo( + name="package-01", + version="1.0.1", + location="/some/location", + is_dirty=False, + ) + ] + + response = get_python_environment(config) + + assert response.installed_packages == [ + PackageInfo( + name="bar-package", + version="http://example.com/bar.git @adsad23123", + location="", + is_dirty=True, + source=SourceInfo.SCRATCH, + ), + PackageInfo( + name="foo-package", + version="http://example.com/foo.git @main", + location="", + is_dirty=False, + source=SourceInfo.SCRATCH, + ), + PackageInfo( + name="package-01", + version="1.0.1", + location="/some/location", + is_dirty=False, + source=SourceInfo.PYPI, + ), + ] + + +@patch("blueapi.service.environment.Repo") +@patch("blueapi.service.environment._fetch_installed_packages_details") +@patch("blueapi.service.environment._get_project_name_from_pyproject") +def test_fetch_python_env_with_identical_packages( + mock_get_project_name: Mock, + mock_fetch_installed_packages: Mock, + mock_repo: Mock, + directory_path_with_sgid: Path, +): + repo_path = directory_path_with_sgid / "foo" + repo_path.mkdir() + mock_repo_instance = Mock() + mock_repo_instance.active_branch.name = "main" + mock_repo_instance.is_dirty.return_value = False + mock_repo_instance.remotes = [Mock(url="http://example.com/foo.git")] + + mock_repo.return_value = mock_repo_instance + + mock_get_project_name.return_value = "foo-package" + mock_fetch_installed_packages.return_value = [ + PackageInfo( + name="foo-package", + version="http://example.com/foo.git @main", + location="/some/location", + is_dirty=False, + source=SourceInfo.SCRATCH, + ) + ] + config = ScratchConfig( + root=directory_path_with_sgid, + repositories=[ + ScratchRepository( + name="foo", + remote_url="http://example.com/foo.git", + ), + ], + ) + response = get_python_environment(config) + + assert response.installed_packages == [ + PackageInfo( + name="foo-package", + version="http://example.com/foo.git @main", + location="/some/location &&", + is_dirty=False, + source=SourceInfo.SCRATCH, + ), + ] + + +@patch("blueapi.service.environment.importlib.metadata.distributions") +def test_fetch_installed_packages_details_returns_correct_packages(mock_distributions): + mock_distribution = Mock() + mock_distribution.metadata = {"Name": "example-package"} + mock_distribution.version = "1.0.0" + mock_distribution.locate_file.return_value = Path("/example/location") + mock_distributions.return_value = [mock_distribution] + + packages = _fetch_installed_packages_details() + + assert len(packages) == 1 + assert packages == [ + PackageInfo( + name="example-package", + version="1.0.0", + location="/example/location", + is_dirty=False, + ) + ] + + +@patch("blueapi.service.environment.Repo") +@patch("blueapi.service.environment._fetch_installed_packages_details") +@patch("blueapi.service.environment._get_project_name_from_pyproject") +def test_get_python_env_filters_by_name_and_source( + mock_get_project_name: Mock, + mock_fetch_installed_packages: Mock, + mock_repo: Mock, + directory_path_with_sgid: Path, +): + # Setup for scratch source filtering + repo_path = directory_path_with_sgid / "foo" + repo_path.mkdir() + mock_repo_instance = Mock() + mock_repo_instance.active_branch.name = "main" + mock_repo_instance.is_dirty.return_value = False + mock_repo_instance.remotes = [Mock(url="http://example.com/foo.git")] + mock_repo.return_value = mock_repo_instance + + mock_get_project_name.return_value = "foo-package" + mock_fetch_installed_packages.return_value = [ + PackageInfo( + name="bar-package", + version="1.0.0", + location="/some/location", + is_dirty=False, + source=SourceInfo.PYPI, + ) + ] + config = ScratchConfig( + root=directory_path_with_sgid, + repositories=[ + ScratchRepository( + name="foo", + remote_url="http://example.com/foo.git", + ), + ], + ) + # Test filtering by name + response_by_name = get_python_environment(config, name="foo-package") + assert response_by_name.installed_packages == [ + PackageInfo( + name="foo-package", + version="http://example.com/foo.git @main", + location="", + is_dirty=False, + source=SourceInfo.SCRATCH, + ) + ] + + # Test filtering by source + response_by_source = get_python_environment(config, source=SourceInfo.SCRATCH) + assert response_by_source.installed_packages == [ + PackageInfo( + name="foo-package", + version="http://example.com/foo.git @main", + location="", + is_dirty=False, + source=SourceInfo.SCRATCH, + ) + ] + + +@pytest.fixture +def pyproject_file(tmp_path: Path) -> Generator[Path]: + pyproject_path = tmp_path / "pyproject.toml" + with pyproject_path.open("w") as f: + f.write( + """ + [project] + name = "example-project" + """ + ) + yield pyproject_path + os.remove(pyproject_path) + + +def test_get_project_name_from_pyproject_returns_name(pyproject_file: Path): + project_name = _get_project_name_from_pyproject(pyproject_file.parent) + assert project_name == "example-project" + + +def test_get_project_name_from_pyproject_returns_empty_if_no_pyproject( + tmp_path: Path, +): + project_name = _get_project_name_from_pyproject(tmp_path) + assert project_name == "" + + +def test_get_project_name_from_pyproject_returns_empty_if_no_name_key( + tmp_path: Path, +): + pyproject_path = tmp_path / "pyproject.toml" + with pyproject_path.open("w") as f: + f.write( + """ + [project] + version = "1.0.0" + """ + ) + project_name = _get_project_name_from_pyproject(tmp_path) + assert project_name == "" diff --git a/tests/unit_tests/service/test_interface.py b/tests/unit_tests/service/test_interface.py index e0c18f1a7b..e56e98c2e5 100644 --- a/tests/unit_tests/service/test_interface.py +++ b/tests/unit_tests/service/test_interface.py @@ -41,8 +41,8 @@ TaskRequest, WorkerTask, ) +from blueapi.service.numtracker import NumtrackerClient from blueapi.utils.invalid_config_error import InvalidConfigError -from blueapi.utils.numtracker import NumtrackerClient from blueapi.worker.event import ( TaskResult, TaskStatus, @@ -307,7 +307,7 @@ def mock_tasks( assert interface.get_tasks() == pending_task + running_task -@patch("blueapi.service.interface.BlueskyContext.numtracker") +@patch("blueapi.service.interface.numtracker") @patch("blueapi.service.interface.TaskWorker.begin_task") def test_begin_task_with_headers(worker_mock: MagicMock, mock_numtracker: MagicMock): uuid_value = "350043fd-597e-41a7-9a92-5d5478232cf7" @@ -315,7 +315,7 @@ def test_begin_task_with_headers(worker_mock: MagicMock, mock_numtracker: MagicM headers = {"a": "b"} returned_task = interface.begin_task(task, headers) - mock_numtracker.set_headers.assert_called_once_with(headers) + mock_numtracker().set_headers.assert_called_once_with(headers) assert task == returned_task worker_mock.assert_called_once_with(uuid_value) @@ -442,7 +442,6 @@ def test_submit_task_inserts_metadata(context_mock: MagicMock): @patch("blueapi.service.interface.worker") def test_remove_tiled_subscriber(worker, context, from_uri, writer): task = WorkerTask(task_id="foo_bar") - context().numtracker = None context().tiled_conf = TiledConfig() context().run_engine.subscribe.return_value = 17 worker().worker_events.subscribe.return_value = 42 @@ -510,7 +509,6 @@ def _missing_session_path(key: str): @patch("blueapi.service.interface.context") @patch("blueapi.service.interface.worker") def test_begin_task_creates_tiled_containers(worker, context, from_uri, writer): - context().numtracker = None context().tiled_conf = TiledConfig() context().run_engine.md = {"instrument": "p46"} worker().get_task_by_id.return_value = _existing_task() @@ -545,7 +543,6 @@ def test_begin_task_creates_tiled_containers(worker, context, from_uri, writer): def test_begin_task_finds_existing_session_in_one_lookup( worker, context, from_uri, writer ): - context().numtracker = None context().tiled_conf = TiledConfig() context().run_engine.md = {"instrument": "p46"} worker().get_task_by_id.return_value = _existing_task() @@ -562,7 +559,6 @@ def test_begin_task_finds_existing_session_in_one_lookup( @patch("blueapi.service.interface.context") @patch("blueapi.service.interface.worker") def test_begin_task_raises_for_invalid_instrument_session(worker, context, from_uri): - context().numtracker = None context().tiled_conf = TiledConfig() context().run_engine.md = {"instrument": "p46"} worker().get_task_by_id.return_value = _existing_task( @@ -584,7 +580,6 @@ def test_begin_task_raises_for_invalid_instrument_session(worker, context, from_ def test_begin_task_handles_tiled_container_create_errors( worker, context, from_uri, writer, status_code, expect_raises ): - context().numtracker = None context().tiled_conf = TiledConfig() context().run_engine.md = {"instrument": "p46"} worker().get_task_by_id.return_value = _existing_task() @@ -608,7 +603,6 @@ def test_begin_task_handles_tiled_container_create_errors( def test_begin_task_uses_service_account_auth_for_tiled( worker, context, from_uri, writer ): - context().numtracker = None context().tiled_conf = TiledConfig( authentication=ServiceAccount( client_id="tiled_writer", @@ -630,7 +624,6 @@ def test_begin_task_uses_service_account_auth_for_tiled( def test_service_account_tiled_client_reused_across_tasks( worker, context, from_uri, writer ): - context().numtracker = None context().tiled_conf = TiledConfig( authentication=ServiceAccount( client_id="tiled_writer", @@ -670,7 +663,7 @@ def test_stomp_config_makes_no_client_when_disabled(mock_stomp_client: StompClie assert interface.stomp_client() is None -@patch("blueapi.cli.scratch._fetch_installed_packages_details") +@patch("blueapi.service.environment._fetch_installed_packages_details") def test_get_scratch_no_config(mock_fetch_installed_packages: Mock): interface.set_config(ApplicationConfig(scratch=None)) mock_fetch_installed_packages.return_value = [] @@ -708,7 +701,7 @@ def test_configure_numtracker(): ) interface.set_config(conf) headers = {"a": "b"} - nt = interface.context().numtracker + nt = interface.numtracker() assert isinstance(nt, NumtrackerClient) nt.set_headers(headers) @@ -716,7 +709,7 @@ def test_configure_numtracker(): assert nt._url.unicode_string() == "https://numtracker-example.com/graphql" -@patch("blueapi.utils.numtracker.httpx.AsyncClient.post") +@patch("blueapi.service.numtracker.httpx.AsyncClient.post") async def test_headers_are_cleared(mock_post): mock_response = Mock() mock_post.return_value = mock_response @@ -779,7 +772,7 @@ def test_numtracker_requires_instrument_metadata(): interface.set_config(ApplicationConfig()) -@patch("blueapi.utils.numtracker.NumtrackerClient.create_scan") +@patch("blueapi.service.numtracker.NumtrackerClient.create_scan") async def test_numtracker_create_scan_called_with_arguments_from_metadata( mock_create_scan, ): @@ -794,10 +787,10 @@ async def test_numtracker_create_scan_called_with_arguments_from_metadata( headers = {"a": "b"} - assert ctx.numtracker is not None + assert (nt := interface.numtracker()) is not None assert ctx.run_engine.scan_id_source is not None - ctx.numtracker.set_headers(headers) + nt.set_headers(headers) ctx.run_engine.md["instrument_session"] = "ab123" scan_id = ctx.run_engine.scan_id_source(ctx.run_engine.md) assert isawaitable(scan_id) and await scan_id diff --git a/tests/unit_tests/utils/test_numtracker.py b/tests/unit_tests/service/test_numtracker.py similarity index 99% rename from tests/unit_tests/utils/test_numtracker.py rename to tests/unit_tests/service/test_numtracker.py index 1aa6df9e88..ee3546e242 100644 --- a/tests/unit_tests/utils/test_numtracker.py +++ b/tests/unit_tests/service/test_numtracker.py @@ -5,7 +5,7 @@ from pydantic import HttpUrl from pytest_httpx import HTTPXMock -from blueapi.utils.numtracker import ( +from blueapi.service.numtracker import ( DirectoryPath, NumtrackerClient, NumtrackerScanMutationResponse, diff --git a/tests/unit_tests/test_log.py b/tests/unit_tests/test_log.py index ba3325b151..f09eede82e 100644 --- a/tests/unit_tests/test_log.py +++ b/tests/unit_tests/test_log.py @@ -60,13 +60,13 @@ def mock_stream_handler_emit() -> Generator[Mock]: @pytest.fixture def mock_graylog_emit() -> Generator[Mock]: - with patch("blueapi.log.GELFTCPHandler.emit") as graylog_emit: + with patch("graypy.GELFTCPHandler.emit") as graylog_emit: yield graylog_emit MOCK_HANDLER_EMIT_STRINGS = [ "blueapi.log.logging.StreamHandler.emit", - "blueapi.log.GELFTCPHandler.emit", + "graypy.GELFTCPHandler.emit", ] diff --git a/uv.lock b/uv.lock index db9deff7b4..6ddebe63f6 100644 --- a/uv.lock +++ b/uv.lock @@ -3,16 +3,16 @@ revision = 5 requires-python = ">=3.11" resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version == '3.13.*' and sys_platform == 'emscripten'", - "python_full_version == '3.12.*' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and sys_platform != 'emscripten'", + "python_full_version == '3.13.*' and sys_platform == 'emscripten'", "python_full_version == '3.13.*' and sys_platform != 'emscripten'", + "python_full_version == '3.12.*' and sys_platform == 'emscripten'", "python_full_version == '3.12.*' and sys_platform != 'emscripten'", "python_full_version >= '3.11.5' and python_full_version < '3.12' and sys_platform == 'win32'", - "python_full_version < '3.11.5' and sys_platform == 'win32'", "python_full_version >= '3.11.5' and python_full_version < '3.12' and sys_platform == 'emscripten'", - "python_full_version < '3.11.5' and sys_platform == 'emscripten'", "python_full_version >= '3.11.5' and python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version < '3.11.5' and sys_platform == 'win32'", + "python_full_version < '3.11.5' and sys_platform == 'emscripten'", "python_full_version < '3.11.5' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] @@ -481,28 +481,23 @@ wheels = [ name = "blueapi" source = { editable = "." } dependencies = [ - { name = "aioca" }, - { name = "aiohttp" }, { name = "bluesky", extra = ["plotting"] }, { name = "bluesky-stomp" }, { name = "click" }, { name = "event-model" }, - { name = "fastapi" }, { name = "gitpython" }, - { name = "graypy" }, - { name = "httpx" }, { name = "observability-utils" }, { name = "opentelemetry-distro" }, - { name = "opentelemetry-instrumentation-fastapi" }, { name = "ophyd-async" }, + { name = "packaging" }, { name = "pydantic" }, { name = "pydantic-settings" }, { name = "pyjwt", extra = ["crypto"] }, { name = "pyyaml" }, { name = "requests" }, { name = "stomp-py" }, - { name = "tiled", extra = ["client"] }, - { name = "uvicorn" }, + { name = "tqdm" }, + { name = "websockets" }, ] [package.optional-dependencies] @@ -510,9 +505,27 @@ demo = [ { name = "dls-dodal" }, { name = "ophyd-async", extra = ["sim"] }, ] +server = [ + { name = "aioca" }, + { name = "aiohttp" }, + { name = "fastapi" }, + { name = "graypy" }, + { name = "httpx" }, + { name = "opentelemetry-instrumentation-fastapi" }, + { name = "tiled", extra = ["client"] }, + { name = "uvicorn" }, +] [package.dev-dependencies] +client-system-test = [ + { name = "jwcrypto" }, + { name = "playwright" }, + { name = "pytest" }, + { name = "pytest-asyncio" }, + { name = "responses" }, +] dev = [ + { name = "blueapi", extra = ["server"] }, { name = "copier" }, { name = "deepdiff" }, { name = "dls-dodal" }, @@ -548,35 +561,46 @@ dev = [ [package.metadata] requires-dist = [ - { name = "aioca" }, - { name = "aiohttp", specifier = ">=3.13.5" }, + { name = "aioca", marker = "extra == 'server'" }, + { name = "aiohttp", marker = "extra == 'server'", specifier = ">=3.13.5" }, { name = "bluesky", extras = ["plotting"], specifier = ">=1.14.0" }, { name = "bluesky-stomp", specifier = ">=0.1.6" }, { name = "click", specifier = ">=8.2.0" }, { name = "dls-dodal", marker = "extra == 'demo'", specifier = ">=3.0.2" }, { name = "event-model", specifier = "==1.24.0" }, - { name = "fastapi", specifier = ">=0.112.0" }, + { name = "fastapi", marker = "extra == 'server'", specifier = ">=0.112.0" }, { name = "gitpython", specifier = ">=3.1.58" }, - { name = "graypy", specifier = ">=2.1.0" }, - { name = "httpx", specifier = ">=0.28.1" }, + { name = "graypy", marker = "extra == 'server'", specifier = ">=2.1.0" }, + { name = "httpx", marker = "extra == 'server'", specifier = ">=0.28.1" }, { name = "observability-utils", specifier = ">=0.1.4" }, { name = "opentelemetry-distro", specifier = ">=0.48b0" }, - { name = "opentelemetry-instrumentation-fastapi", specifier = ">=0.48b0" }, + { name = "opentelemetry-instrumentation-fastapi", marker = "extra == 'server'", specifier = ">=0.48b0" }, { name = "ophyd-async", specifier = ">=0.13.5" }, { name = "ophyd-async", extras = ["sim"], marker = "extra == 'demo'" }, + { name = "packaging" }, { name = "pydantic", specifier = ">=2.0" }, { name = "pydantic-settings" }, { name = "pyjwt", extras = ["crypto"] }, { name = "pyyaml", specifier = ">=6.0.2" }, { name = "requests" }, { name = "stomp-py" }, - { name = "tiled", extras = ["client"], specifier = "==0.2.18" }, - { name = "uvicorn", specifier = ">=0.52.1" }, + { name = "tiled", extras = ["client"], marker = "extra == 'server'", specifier = "==0.2.18" }, + { name = "tqdm" }, + { name = "uvicorn", marker = "extra == 'server'", specifier = ">=0.52.1" }, + { name = "websockets" }, ] -provides-extras = ["demo"] +provides-extras = ["demo", "server"] [package.metadata.requires-dev] +client-system-test = [ + { name = "jwcrypto" }, + { name = "playwright", specifier = ">=1.63.0" }, + { name = "pytest" }, + { name = "pytest-asyncio" }, + { name = "responses" }, +] dev = [ + { name = "blueapi", extras = ["server"] }, { name = "copier" }, { name = "deepdiff" }, { name = "dls-dodal", specifier = ">=3.0.2" }, @@ -1080,10 +1104,10 @@ version = "1.3.3" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.11.5' and python_full_version < '3.12' and sys_platform == 'win32'", - "python_full_version < '3.11.5' and sys_platform == 'win32'", "python_full_version >= '3.11.5' and python_full_version < '3.12' and sys_platform == 'emscripten'", - "python_full_version < '3.11.5' and sys_platform == 'emscripten'", "python_full_version >= '3.11.5' and python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version < '3.11.5' and sys_platform == 'win32'", + "python_full_version < '3.11.5' and sys_platform == 'emscripten'", "python_full_version < '3.11.5' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ @@ -1170,10 +1194,10 @@ version = "1.4.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version == '3.13.*' and sys_platform == 'emscripten'", - "python_full_version == '3.12.*' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and sys_platform != 'emscripten'", + "python_full_version == '3.13.*' and sys_platform == 'emscripten'", "python_full_version == '3.13.*' and sys_platform != 'emscripten'", + "python_full_version == '3.12.*' and sys_platform == 'emscripten'", "python_full_version == '3.12.*' and sys_platform != 'emscripten'", ] dependencies = [ @@ -3711,10 +3735,10 @@ version = "3.6.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.11.5' and python_full_version < '3.12' and sys_platform == 'win32'", - "python_full_version < '3.11.5' and sys_platform == 'win32'", "python_full_version >= '3.11.5' and python_full_version < '3.12' and sys_platform == 'emscripten'", - "python_full_version < '3.11.5' and sys_platform == 'emscripten'", "python_full_version >= '3.11.5' and python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version < '3.11.5' and sys_platform == 'win32'", + "python_full_version < '3.11.5' and sys_platform == 'emscripten'", "python_full_version < '3.11.5' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] sdist = { url = "https://files.pythonhosted.org/packages/6a/51/63fe664f3908c97be9d2e4f1158eb633317598cfa6e1fc14af5383f17512/networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509", size = 2517025, upload-time = "2025-12-08T17:02:39.908Z" } @@ -3728,10 +3752,10 @@ version = "3.7" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version == '3.13.*' and sys_platform == 'emscripten'", - "python_full_version == '3.12.*' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and sys_platform != 'emscripten'", + "python_full_version == '3.13.*' and sys_platform == 'emscripten'", "python_full_version == '3.13.*' and sys_platform != 'emscripten'", + "python_full_version == '3.12.*' and sys_platform == 'emscripten'", "python_full_version == '3.12.*' and sys_platform != 'emscripten'", ] sdist = { url = "https://files.pythonhosted.org/packages/dc/76/3af777226b63a5e64a6b36b1ec5855c14e2b94a37096d4760e595fc43511/networkx-3.7.tar.gz", hash = "sha256:fd77a511bd90f39f3d016351345b52cf5319b813bdca01de3f755d3cca62e96a", size = 1866482, upload-time = "2026-09-21T16:45:16.974Z" } @@ -3860,10 +3884,10 @@ version = "2.4.6" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.11.5' and python_full_version < '3.12' and sys_platform == 'win32'", - "python_full_version < '3.11.5' and sys_platform == 'win32'", "python_full_version >= '3.11.5' and python_full_version < '3.12' and sys_platform == 'emscripten'", - "python_full_version < '3.11.5' and sys_platform == 'emscripten'", "python_full_version >= '3.11.5' and python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version < '3.11.5' and sys_platform == 'win32'", + "python_full_version < '3.11.5' and sys_platform == 'emscripten'", "python_full_version < '3.11.5' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] sdist = { url = "https://files.pythonhosted.org/packages/d0/ad/fed0499ce6a338d2a03ebae59cd15093910c8875328855781952abf6c2fe/numpy-2.4.6.tar.gz", hash = "sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda", size = 20735807, upload-time = "2026-05-18T23:37:14.07Z" } @@ -3947,10 +3971,10 @@ version = "2.5.4" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version == '3.13.*' and sys_platform == 'emscripten'", - "python_full_version == '3.12.*' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and sys_platform != 'emscripten'", + "python_full_version == '3.13.*' and sys_platform == 'emscripten'", "python_full_version == '3.13.*' and sys_platform != 'emscripten'", + "python_full_version == '3.12.*' and sys_platform == 'emscripten'", "python_full_version == '3.12.*' and sys_platform != 'emscripten'", ] sdist = { url = "https://files.pythonhosted.org/packages/95/b0/c7453d0b6e2073c3264468b106ee1563750cecc910965e67357e3698c83e/numpy-2.5.4.tar.gz", hash = "sha256:9a94cf751c9ad8ebaa835bcd3d40dacf8534ad086b88c38029b65123c7999d2a", size = 20866315, upload-time = "2026-10-10T20:05:31.422Z" } @@ -4640,10 +4664,10 @@ version = "0.25.3" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.11.5' and python_full_version < '3.12' and sys_platform == 'win32'", - "python_full_version < '3.11.5' and sys_platform == 'win32'", "python_full_version >= '3.11.5' and python_full_version < '3.12' and sys_platform == 'emscripten'", - "python_full_version < '3.11.5' and sys_platform == 'emscripten'", "python_full_version >= '3.11.5' and python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version < '3.11.5' and sys_platform == 'win32'", + "python_full_version < '3.11.5' and sys_platform == 'emscripten'", "python_full_version < '3.11.5' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ @@ -4663,10 +4687,10 @@ version = "0.26.2" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version == '3.13.*' and sys_platform == 'emscripten'", - "python_full_version == '3.12.*' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and sys_platform != 'emscripten'", + "python_full_version == '3.13.*' and sys_platform == 'emscripten'", "python_full_version == '3.13.*' and sys_platform != 'emscripten'", + "python_full_version == '3.12.*' and sys_platform == 'emscripten'", "python_full_version == '3.12.*' and sys_platform != 'emscripten'", ] dependencies = [ @@ -5257,10 +5281,10 @@ version = "13.0.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.11.5' and python_full_version < '3.12' and sys_platform == 'win32'", - "python_full_version < '3.11.5' and sys_platform == 'win32'", "python_full_version >= '3.11.5' and python_full_version < '3.12' and sys_platform == 'emscripten'", - "python_full_version < '3.11.5' and sys_platform == 'emscripten'", "python_full_version >= '3.11.5' and python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version < '3.11.5' and sys_platform == 'win32'", + "python_full_version < '3.11.5' and sys_platform == 'emscripten'", "python_full_version < '3.11.5' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ @@ -5277,10 +5301,10 @@ version = "14.0.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version == '3.13.*' and sys_platform == 'emscripten'", - "python_full_version == '3.12.*' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and sys_platform != 'emscripten'", + "python_full_version == '3.13.*' and sys_platform == 'emscripten'", "python_full_version == '3.13.*' and sys_platform != 'emscripten'", + "python_full_version == '3.12.*' and sys_platform == 'emscripten'", "python_full_version == '3.12.*' and sys_platform != 'emscripten'", ] dependencies = [ @@ -6188,10 +6212,10 @@ version = "3.0.4" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version == '3.13.*' and sys_platform == 'emscripten'", - "python_full_version == '3.12.*' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and sys_platform != 'emscripten'", + "python_full_version == '3.13.*' and sys_platform == 'emscripten'", "python_full_version == '3.13.*' and sys_platform != 'emscripten'", + "python_full_version == '3.12.*' and sys_platform == 'emscripten'", "python_full_version == '3.12.*' and sys_platform != 'emscripten'", "python_full_version >= '3.11.5' and python_full_version < '3.12' and sys_platform == 'win32'", "python_full_version >= '3.11.5' and python_full_version < '3.12' and sys_platform == 'emscripten'", @@ -6222,10 +6246,10 @@ version = "9.0.4" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.11.5' and python_full_version < '3.12' and sys_platform == 'win32'", - "python_full_version < '3.11.5' and sys_platform == 'win32'", "python_full_version >= '3.11.5' and python_full_version < '3.12' and sys_platform == 'emscripten'", - "python_full_version < '3.11.5' and sys_platform == 'emscripten'", "python_full_version >= '3.11.5' and python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version < '3.11.5' and sys_platform == 'win32'", + "python_full_version < '3.11.5' and sys_platform == 'emscripten'", "python_full_version < '3.11.5' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ @@ -6290,8 +6314,8 @@ version = "9.2.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version == '3.13.*' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and sys_platform != 'emscripten'", + "python_full_version == '3.13.*' and sys_platform == 'emscripten'", "python_full_version == '3.13.*' and sys_platform != 'emscripten'", ] dependencies = [ @@ -7149,10 +7173,10 @@ version = "3.3" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.11.5' and python_full_version < '3.12' and sys_platform == 'win32'", - "python_full_version < '3.11.5' and sys_platform == 'win32'", "python_full_version >= '3.11.5' and python_full_version < '3.12' and sys_platform == 'emscripten'", - "python_full_version < '3.11.5' and sys_platform == 'emscripten'", "python_full_version >= '3.11.5' and python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version < '3.11.5' and sys_platform == 'win32'", + "python_full_version < '3.11.5' and sys_platform == 'emscripten'", "python_full_version < '3.11.5' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ @@ -7175,10 +7199,10 @@ version = "3.7" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version == '3.13.*' and sys_platform == 'emscripten'", - "python_full_version == '3.12.*' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and sys_platform != 'emscripten'", + "python_full_version == '3.13.*' and sys_platform == 'emscripten'", "python_full_version == '3.13.*' and sys_platform != 'emscripten'", + "python_full_version == '3.12.*' and sys_platform == 'emscripten'", "python_full_version == '3.12.*' and sys_platform != 'emscripten'", ] dependencies = [