refactor: make client almost client again

.github/workflows/_system_test.yml

@@ -40,7 +40,7 @@ jobs:
         run: uv run --locked playwright install --with-deps chromium
 
       - name: Run tests
-        run: uv run --locked tox -e system-test
+        run: uv run --locked tox -e client-system-test,system-test
 
       - name: Compose logs
         if: failure()

Dockerfile

@@ -29,7 +29,7 @@ ENV UV_PYTHON_INSTALL_DIR=/python
 
 # Sync the project without its dev dependencies
 RUN --mount=type=cache,target=/root/.cache/uv \
-    uv sync --locked --no-editable --no-dev --managed-python
+    uv sync --locked --no-editable --no-dev --extra server --managed-python
 
 RUN uv pip install debugpy
 

README.md

@@ -13,7 +13,7 @@ Lightweight bluesky-as-a-service wrapper application. Also usable as a library.
 What            | Where
 :---:           | :---:
 Source          | <https://github.com/DiamondLightSource/blueapi>
-PyPI            | `pip install blueapi`
+PyPI            | `pip install blueapi[server]` (server) / `pip install blueapi` (client only)
 Docker          | `docker run ghcr.io/diamondlightsource/blueapi:latest`
 Documentation   | <https://diamondlightsource.github.io/blueapi>
 Releases        | <https://github.com/DiamondLightSource/blueapi/releases>

docs/tutorials/1. installation.md

@@ -25,15 +25,18 @@ source /path/to/venv/bin/activate
 
 You can now use `pip` to install the library and its dependencies:
 ``` 
-python3 -m pip install blueapi
+python3 -m pip install blueapi[server]
 ```
 
 If you require a feature that is not currently released you can also install
 from GitHub:
 ``` 
-python3 -m pip install git+https://github.com/DiamondLightSource/blueapi.git
+python3 -m pip install "blueapi[server] @ git+https://github.com/DiamondLightSource/blueapi.git"
 ```
 
+The `server` extra is only needed to run the worker (`blueapi serve`). For the client
+and `blueapi controller` commands, `pip install blueapi` is enough.
+
 The library should now be installed and the commandline interface on your path.
 You can check the version that has been installed by typing:
 ``` 

docs/tutorials/2. quickstart.md

@@ -7,7 +7,7 @@ You should be in the blueapi directory that you have cloned. From here, if you o
 If you not, you can still run the following from the venv created in page 1 but you may need to run the command below to have the dependencies required for this tutorial.
 
 ```
-pip install .[demo]
+pip install .[demo,server]
 ```
 
 ## Start Worker

pyproject.toml

@@ -14,39 +14,47 @@ classifiers = [
 ]
 description = "Lightweight bluesky-as-a-service wrapper application. Also usable as a library."
 dependencies = [
-    "tiled[client]==0.2.18",
     "bluesky[plotting]>=1.14.0",                     # plotting includes matplotlib, required for BestEffortCallback in run plans
     "ophyd-async>=0.13.5",
-    "aioca",
     "pydantic>=2.0",
     "pydantic-settings",
     "stomp-py",
     "PyYAML>=6.0.2",
     "click>=8.2.0",
-    "fastapi>=0.112.0",
-    "uvicorn>=0.52.1",
     "requests",
-    "GitPython>=3.1.58",                             #security base min https://github.com/advisories/GHSA-4gmw-gg2m-w46p
+    "GitPython>=3.1.58",                             #security base min https://github.com/advisories/GHSA-4gmw-gg2m-w46p - used by the server's /python_environment endpoint and the CLI's scratch setup
     "event-model==1.24.0",                           # https://github.com/DiamondLightSource/blueapi/issues/684
     "bluesky-stomp>=0.1.6",
     "opentelemetry-distro>=0.48b0",
-    "opentelemetry-instrumentation-fastapi>=0.48b0",
     "observability-utils>=0.1.4",
     "pyjwt[crypto]",
-    "graypy>=2.1.0",
-    "httpx>=0.28.1",
-    "aiohttp>=3.13.5",
+    "websockets",
+    "tqdm",
+    "packaging",
 ]
 dynamic = ["version"]
 license.file = "LICENSE"
 readme = "README.md"
 requires-python = ">=3.11"
 
 [project.optional-dependencies]
+
 demo = ["dls-dodal>=3.0.2", "ophyd-async[sim]"]
+# Only needed to run `blueapi serve` (the FastAPI worker service).
+server = [
+    "fastapi>=0.112.0",
+    "uvicorn>=0.52.1",
+    "aiohttp>=3.13.5",
+    "opentelemetry-instrumentation-fastapi>=0.48b0",
+    "graypy>=2.1.0",
+    "aioca",
+    "tiled[client]==0.2.18",
+    "httpx>=0.28.1",                                 # used by service/numtracker.py and service/authentication.py's TiledAuth
+]
 
 [dependency-groups]
 dev = [
+    "blueapi[server]",
     "ophyd_async[sim]",
     "copier",
     "dls-dodal>=3.0.2",
@@ -79,6 +87,14 @@ dev = [
     "httpx2>=2.5.0",
     "playwright>=1.63.0",
 ]
+# Runs tests/system_tests/test_blueapi_client.py against blueapi without the server extra.
+client-system-test = [
+    "pytest",
+    "pytest-asyncio",
+    "playwright>=1.63.0",
+    "responses",          # imported by tests/conftest.py
+    "jwcrypto",           # imported by tests/conftest.py
+]
 
 [project.scripts]
 blueapi = "blueapi.cli:main"
@@ -173,12 +189,31 @@ commands = [
     ],
 ]
 
+[tool.tox.env.client-system-test]
+description = "Run client system tests with blueapi only, no server dependencies"
+commands = [
+    [
+        "uv",
+        "run",
+        "--isolated", # uv run leaves extra packages in place, so don't reuse the dev venv
+        "--locked",
+        "--no-default-groups",
+        "--group",
+        "client-system-test",
+        "pytest",
+        "tests/system_tests/test_blueapi_client.py",
+        { replace = "posargs", default = [
+        ], extend = true },
+    ],
+]
+
 [tool.tox.env.system-test]
 description = "Run system tests with coverage"
 commands = [
     [
         "pytest",
         "tests/system_tests",
+        "--ignore=tests/system_tests/test_blueapi_client.py",
         "--cov=blueapi",
         "--cov-report",
         "term",

src/blueapi/cli/cli.py

@@ -31,13 +31,13 @@
     UnauthorisedAccessError,
     UnknownPlanError,
 )
+from blueapi.client.session import SessionCacheManager, SessionManager
 from blueapi.config import (
     ApplicationConfig,
     ConfigLoader,
 )
 from blueapi.core import OTLP_EXPORT_ENABLED, DataEvent
 from blueapi.log import set_up_logging
-from blueapi.service.authentication import SessionCacheManager, SessionManager
 from blueapi.service.model import DeviceResponse, PlanResponse, SourceInfo, TaskRequest
 from blueapi.worker import ProgressEvent, WorkerEvent
 from blueapi.worker.event import TaskError, TaskResult
@@ -164,6 +164,13 @@ def main(
         print("Please invoke subcommand!")
 
 
+def _missing_server_extra(e: ImportError) -> ClickException:
+    return ClickException(
+        f"{e.name} is not installed, it is needed to run the server. "
+        "Install it with: pip install blueapi[server]"
+    )
+
+
 @main.command(name="schema")
 @click.option("-o", "--output", type=Path, help="Path to file to save the schema")
 @click.option(
@@ -177,12 +184,15 @@ def schema(output: Path | None = None, update: bool = False) -> None:
     """Only import the service functions when starting the service or generating
     the schema, not the controller as a new FastAPI app will be started each time.
     """
-    from blueapi.service.openapi import (
-        DOCS_SCHEMA_LOCATION,
-        generate_schema,
-        print_schema_as_yaml,
-        write_schema_as_yaml,
-    )
+    try:
+        from blueapi.service.openapi import (
+            DOCS_SCHEMA_LOCATION,
+            generate_schema,
+            print_schema_as_yaml,
+            write_schema_as_yaml,
+        )
+    except ImportError as e:
+        raise _missing_server_extra(e) from e
 
     """Generate the schema for the REST API"""
     schema = generate_schema()
@@ -229,7 +239,10 @@ def start_application(obj: dict):
     """Only import the service functions when starting the service or generating
     the schema, not the controller as a new FastAPI app will be started each time.
     """
-    from blueapi.service.main import start
+    try:
+        from blueapi.service.main import start
+    except ImportError as e:
+        raise _missing_server_extra(e) from e
 
     """
     Set up basic automated instrumentation for the FastAPI app, creating the

src/blueapi/cli/scratch.py

@@ -1,18 +1,15 @@
-import importlib.metadata
 import logging
 import os
 import stat
 import textwrap
-import tomllib
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from pathlib import Path
 from subprocess import Popen
 
 from git import Repo
 
 from blueapi.config import FORBIDDEN_OWN_REMOTE_URL, ScratchConfig
-from blueapi.service.model import PackageInfo, PythonEnvironmentResponse, SourceInfo
-from blueapi.utils import get_owner_gid, is_sgid_set
+from blueapi.utils import get_owner_gid, is_sgid_set, validate_directory
 
 _DEFAULT_INSTALL_TIMEOUT: float = 300.0
 
@@ -131,7 +128,7 @@ def scratch_install(*paths: Path, timeout: float = _DEFAULT_INSTALL_TIMEOUT) ->
         return
     LOGGER.info("Installing packages")
     for path in paths:
-        _validate_directory(path)
+        validate_directory(path)
         args = ["uv", "pip", "install", "-e", str(path)]
         process = Popen(args)
         process.wait(timeout=timeout)
@@ -142,7 +139,7 @@ def scratch_install(*paths: Path, timeout: float = _DEFAULT_INSTALL_TIMEOUT) ->
 
 
 def _validate_root_directory(root_path: Path, required_gid: int | None) -> None:
-    _validate_directory(root_path)
+    validate_directory(root_path)
 
     if required_gid is not None:
         if not is_sgid_set(root_path):
@@ -171,98 +168,3 @@ def _validate_root_directory(root_path: Path, required_gid: int | None) -> None:
             chgrp command.
             """)
             )
-
-
-def _validate_directory(path: Path) -> None:
-    if not path.exists():
-        raise KeyError(f"{path}: No such file or directory")
-    elif path.is_file():
-        raise KeyError(f"{path}: Is a file, not a directory")
-
-
-def _get_project_name_from_pyproject(path: Path) -> str:
-    pyproject_path = path / "pyproject.toml"
-    if pyproject_path.exists():
-        with pyproject_path.open("rb") as file:
-            toml_data = tomllib.load(file)
-        return toml_data.get("project", {}).get("name", "")
-    return ""
-
-
-def _fetch_installed_packages_details() -> list[PackageInfo]:
-    installed_packages = importlib.metadata.distributions()
-    return [
-        PackageInfo(
-            name=dist.metadata["Name"],
-            version=dist.version,
-            location=str(dist.locate_file("")),
-            is_dirty=False,
-        )
-        for dist in installed_packages
-    ]
-
-
-def get_python_environment(
-    config: ScratchConfig | None,
-    name: str | None = None,
-    source: SourceInfo | None = None,
-) -> PythonEnvironmentResponse:
-    """
-    Get the Python environment. This includes all installed packages and
-    the scratch packages.
-    """
-    scratch_packages = {}
-    packages = []
-
-    if config is None:
-        python_env_response = PythonEnvironmentResponse(scratch_enabled=False)
-    else:
-        python_env_response = PythonEnvironmentResponse(scratch_enabled=True)
-        _validate_directory(config.root)
-        for repo in config.repositories:
-            local_directory = config.root / repo.name
-            repo = Repo(local_directory)
-            try:
-                branch = repo.active_branch.name
-            except TypeError:
-                branch = repo.head.commit.hexsha
-
-            is_dirty = repo.is_dirty()
-
-            version = (
-                f"{repo.remotes[0].url} @{branch}"
-                if repo.remotes
-                else f"UNKNOWN REMOTE @{branch}"
-            )
-            package_name = _get_project_name_from_pyproject(local_directory)
-            package_location = ""
-
-            packages.append(
-                PackageInfo(
-                    name=package_name,
-                    version=version,
-                    location=package_location,
-                    source=SourceInfo.SCRATCH,
-                    is_dirty=is_dirty,
-                )
-            )
-        scratch_packages = {p.name: p for p in packages}
-
-    for pkg in _fetch_installed_packages_details():
-        if pkg.name not in scratch_packages:
-            packages.append(pkg)
-        else:
-            scratch_packages[pkg.name].location += f"{pkg.location} &&"
-
-    python_env_response.installed_packages = sorted(
-        packages, key=lambda pkg: pkg.name.lower()
-    )
-    if name:
-        python_env_response.installed_packages = [
-            p for p in python_env_response.installed_packages if p.name == name
-        ]
-    if source:
-        python_env_response.installed_packages = [
-            p for p in python_env_response.installed_packages if p.source == source
-        ]
-    return python_env_response

src/blueapi/client/client.py

@@ -13,14 +13,14 @@
 from observability_utils.tracing import get_tracer, start_as_current_span
 from pydantic import HttpUrl
 
+from blueapi.client.session import SessionCacheManager, SessionManager
 from blueapi.config import (
     ApplicationConfig,
     ConfigLoader,
     MissingStompConfigurationError,
     RestConfig,
 )
 from blueapi.core.bluesky_types import DataEvent
-from blueapi.service.authentication import SessionCacheManager, SessionManager
 from blueapi.service.model import (
     DeviceModel,
     DeviceResponse,