diff --git a/README.md b/README.md index c0f4296..e58eb99 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,31 @@ To install: ```pip install s3dol``` [Documentation](https://i2mint.github.io/s3dol/) +## Heads-up: v1 is coming — run `s3dol.diagnose()` now + +A major, behaviour-clarifying v1 is in progress +(design: [`misc/docs/architecture.md`](misc/docs/architecture.md) and the ADRs in +[`misc/docs/decisions/`](misc/docs/decisions/)). Today's `S3Store` will keep +working through a compatibility shim, but a few v0 behaviours were bugs +(e.g. an explicit `endpoint_url=` being silently dropped whenever +`AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY` are exported) and v1 fixes them — +which can *move where your data goes*. + +Before upgrading to v1, run the diagnosis with the same arguments you pass +`S3Store` today, in the environment you deploy in: + +```python +import s3dol + +s3dol.diagnose(bucket_name='my-bucket', endpoint_url='...', path='...') +``` + +It prints what resolves (endpoint, region, signing, credential *source* — never +a secret), where each value came from, and a **v0-vs-v1 divergence table** +telling you whether the upgrade changes anything for your call. It never +raises; a failing resolution is part of the report. + + ## Set up credentials Recommended prerequisite to make getting started easier but not required. diff --git a/conftest.py b/conftest.py index 65254ed..1b661d5 100644 --- a/conftest.py +++ b/conftest.py @@ -30,7 +30,16 @@ #: module not listed here is deselected when no endpoint is reachable, so a new #: endpoint-using test can never silently fail the local gate. Widen only after #: verifying a module's tests are genuinely endpoint-free. -_S3_FREE_TEST_STEMS = frozenset({"test_url_for"}) +_S3_FREE_TEST_STEMS = frozenset( + { + "test_url_for", + # Layer A (v1) suites: tier-1 pure + moto-mocked tier-2 — no endpoint. + "test_errors", + "test_presets", + "test_connection", + "test_diagnose", + } +) def _probe_host_port(): diff --git a/s3dol/__init__.py b/s3dol/__init__.py index cce333d..edae93b 100644 --- a/s3dol/__init__.py +++ b/s3dol/__init__.py @@ -4,3 +4,41 @@ from s3dol.base import S3Dol from s3dol.store import S3Store + +# New (v1) layer-A names, loaded lazily (PEP 562) so `import s3dol` does not +# pay for what a script does not use. `s3dol.diagnose(...)` is the ADR-0007 §5 +# step-0 migration safety mechanism — run it with your current S3Store kwargs +# to see what resolves, from where, and whether the upcoming v1 moves it. +_LAZY_ATTRIBUTES = { + "diagnose": ("s3dol._diagnose", "diagnose"), + "S3Connection": "s3dol.connection", + "resolve": "s3dol.connection", + "Resolution": "s3dol.connection", + "Preset": "s3dol.presets", + "Capabilities": "s3dol.presets", + "register_preset": ("s3dol.presets", "register"), +} + + +def __getattr__(name): + try: + target = _LAZY_ATTRIBUTES[name] + except KeyError: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") from None + import importlib + + if isinstance(target, tuple): + module_path, attribute = target + else: + module_path, attribute = target, name + value = getattr(importlib.import_module(module_path), attribute) + # Cache the resolved attribute. This ALSO deliberately overrides the + # submodule binding that `import s3dol.diagnose` just created on this + # package — without it, `s3dol.diagnose` would resolve to the module on + # every access after the first, shadowing the function. + globals()[name] = value + return value + + +def __dir__(): + return sorted(set(globals()) | set(_LAZY_ATTRIBUTES)) diff --git a/s3dol/_diagnose.py b/s3dol/_diagnose.py new file mode 100644 index 0000000..32f335e --- /dev/null +++ b/s3dol/_diagnose.py @@ -0,0 +1,317 @@ +"""``s3dol.diagnose()`` — print what would resolve, from where, and whether v1 +moves it. The step-0 migration safety mechanism (ADR-0007 §5, ADR-0012 D6/D7). + +Run it with the same arguments you pass ``s3dol.store.S3Store`` today:: + + import s3dol + s3dol.diagnose(bucket_name='my-bucket', endpoint_url='...', path='logs') + +Non-negotiables (ADR-0012 D6): + +- it **never raises** — a failed resolution is a *row*; this function exists to + report exactly the misconfigurations that raise; +- it holds **no credential value**, only a provenance label; it never reads + ``.access_key`` (which can trigger a network refresh); credential identity is + discussed as a predicate, never a snapshot. + +The v0-vs-v1 divergence table is a **differential, not a frozen simulation** +(ADR-0012 D7): one ~20-line argument-mangler (:func:`_v0_mangle`, mapping 1:1 +onto v0's ``get_client``/``_find_default_credentials``) is run through v1's own +resolver twice, so resolver imprecision cancels. It dies in the same commit as +the compat shim. +""" + +from __future__ import annotations + +import io +import re +import sys +from typing import Mapping, Optional + +from s3dol.connection import S3Connection, Resolution, load_aws_config, resolve +from s3dol.errors import redact + +#: v0 ``S3Store`` parameter names (the shim signature, ADR-0007 §3 — including +#: ``is_supabase_endpoint``, which v0 accepts and an early shim draft forgot). +_V0_ARG_NAMES = frozenset( + { + "bucket_name", + "make_bucket", + "path", + "aws_access_key_id", + "aws_secret_access_key", + "aws_session_token", + "endpoint_url", + "region_name", + "profile_name", + "skip_bucket_check", + "is_supabase_endpoint", + } +) + +_SECRET_ARG_NAMES = frozenset({"aws_secret_access_key", "aws_session_token"}) + +#: Reasons a v0 branch is excluded from the differential (ADR-0012 D7): these +#: two branches build clients through the module-level ``boto3.client(...)``, +#: which resolves through the process-global cached DEFAULT_SESSION — they are +#: not functions of their arguments. +_EXCLUDED_BRANCHES = { + "supabase": ( + "v0's Supabase branch builds its client through boto3's process-global " + "session, so its behaviour is not a function of these arguments. Note: " + "a long-lived v0 process that builds one Supabase store pins its " + "credentials for the life of the interpreter." + ), + "skip-bucket-check": ( + "v0's skip_bucket_check branch builds its client through boto3's " + "process-global session, so its behaviour is not a function of these " + "arguments." + ), +} + + +def _v0_mangle(args: Mapping, environ: Mapping) -> tuple[str, Optional[dict]]: + """What v0's ``S3Store`` *effectively* does with these arguments. + + ~20 frozen lines mapping 1:1 onto v0 (``store.py`` + ``base.py:get_client`` + / ``_find_default_credentials``). Returns ``(branch, effective_args)``; + ``effective_args=None`` means the branch is excluded (see + :data:`_EXCLUDED_BRANCHES`). + """ + endpoint = args.get("endpoint_url") + is_supabase = args.get("is_supabase_endpoint") + if is_supabase is None: + is_supabase = bool(endpoint and ".supabase." in endpoint) + if is_supabase: + return "supabase", None + if args.get("skip_bucket_check"): + return "skip-bucket-check", None + effective = dict(args) + profile = args.get("profile_name") + env_creds = bool( + environ.get("AWS_ACCESS_KEY_ID") and environ.get("AWS_SECRET_ACCESS_KEY") + ) + if profile is None and env_creds: + # get_client(None) -> _find_default_credentials -> the env branch: + # env creds OVERRIDE explicit kwargs ({**kwargs, **env}), env's (maybe + # unset) token/region NULL explicit ones, and endpoint_url is DROPPED + # (never forwarded to get_client('environment variables', ...)). + effective.update( + aws_access_key_id=environ.get("AWS_ACCESS_KEY_ID"), + aws_secret_access_key=environ.get("AWS_SECRET_ACCESS_KEY"), + aws_session_token=environ.get("AWS_SESSION_TOKEN"), + region_name=environ.get("AWS_DEFAULT_REGION"), + endpoint_url=None, + ) + return "env-credentials (endpoint dropped)", effective + if profile == "environment variables": + effective.update( + aws_access_key_id=environ.get("AWS_ACCESS_KEY_ID"), + aws_secret_access_key=environ.get("AWS_SECRET_ACCESS_KEY"), + aws_session_token=environ.get("AWS_SESSION_TOKEN"), + region_name=environ.get("AWS_DEFAULT_REGION"), + ) + return "env-profile", effective + if profile is not None and any( + args.get(k) + for k in ("aws_access_key_id", "aws_secret_access_key", "aws_session_token") + ): + # boto3: profile + any explicit credential kwarg silently discards the + # profile's credentials (measured, ADR-0012 D5). + return "profile+explicit-creds (profile credentials discarded)", effective + return ("profile" if profile else "explicit"), effective + + +def _spec_from_v0_args(args: Mapping) -> S3Connection: + """The v1 spec equivalent of a v0 ``S3Store`` call (connection axes only).""" + access = args.get("aws_access_key_id") + secret = args.get("aws_secret_access_key") + token = args.get("aws_session_token") + credentials = None + if access and secret: + credentials = (access, secret, token) if token else (access, secret) + profile = args.get("profile_name") + if profile == "environment variables": + profile = None # env credentials are the top of botocore's own chain + return S3Connection( + endpoint_url=args.get("endpoint_url") or None, + region_name=args.get("region_name") or None, + profile=profile, + credentials=credentials, + ) + + +def _redact_url(url: Optional[str]) -> Optional[str]: + """Redact query secrets and any userinfo from a URL for display.""" + if url is None: + return None + return re.sub(r"://[^@/]+@", "://REDACTED@", redact(url)) + + +def _field_rows(resolution: Resolution): + for name in ( + "endpoint_url", + "region_name", + "signature_version", + "addressing_style", + "checksum", + "payload_signing_enabled", + "verify", + ): + sourced = getattr(resolution, name) + value = sourced.value + if name == "endpoint_url": + value = _redact_url(value) + handed = "passed" if sourced.passed else "omitted (channel open)" + yield name, value, sourced.source, handed + + +def _divergence_rows(args: Mapping, environ: Mapping, aws_config) -> list[str]: + branch, effective = _v0_mangle(args, environ) + if effective is None: + return [ + f"v0 branch: {branch} — EXCLUDED from the differential.", + _EXCLUDED_BRANCHES[branch], + ] + rows = [f"v0 branch: {branch}"] + v1_res = resolve(_spec_from_v0_args(args), environ, aws_config) + v0_res = resolve(_spec_from_v0_args(effective), environ, aws_config) + moved = False + for axis in ("endpoint_url", "region_name"): + v0_value = getattr(v0_res, axis).value + v1_value = getattr(v1_res, axis).value + if v0_value != v1_value: + moved = True + rows.append( + f" {axis}: v0 -> {_redact_url(v0_value)!r}, " + f"v1 -> {_redact_url(v1_value)!r} ** CHANGES ON UPGRADE **" + ) + if v0_res.credential_provenance != v1_res.credential_provenance: + moved = True + rows.append( + f" credentials: v0 -> {v0_res.credential_provenance}, " + f"v1 -> {v1_res.credential_provenance} ** CHANGES ON UPGRADE **" + ) + if not moved: + rows.append(" no divergence: v1 resolves this call exactly as v0 does.") + return rows + + +def diagnose( + connection: Optional[S3Connection] = None, + *, + environ: Optional[Mapping[str, str]] = None, + file=None, + **kwargs, +) -> str: + """Print (and return) the resolution report. Never raises. + + Call it either with a ready :class:`~s3dol.connection.S3Connection`, with + ``S3Connection`` kwargs, or — the migration case — with exactly the kwargs + you pass ``s3dol.store.S3Store`` today (``bucket_name=``, ``path=``, + ``profile_name=``, ...), which additionally prints the v0-vs-v1 + divergence table. + + >>> report = diagnose(endpoint_url='http://localhost:9000', + ... environ={}, file=io.StringIO()) + >>> 'endpoint_url' in report and 'resolution' in report.lower() + True + """ + out = io.StringIO() + + def emit(line=""): + print(line, file=out) + + try: + import importlib.metadata + + try: + own_version = importlib.metadata.version("s3dol") + except Exception: + own_version = "unknown" + try: + import botocore + + botocore_version = botocore.__version__ + except Exception: + botocore_version = "NOT IMPORTABLE" + emit("s3dol diagnose") + emit("=" * 60) + emit( + f"s3dol {own_version} | botocore {botocore_version} | " + f"python {sys.version.split()[0]}" + ) + + if environ is None: + import os + + environ = dict(os.environ) + aws_config: object + try: + aws_config = load_aws_config() + except Exception as error: + aws_config = {} + emit(f"note: aws config unreadable ({type(error).__name__}: {error})") + + v0_args = {k: v for k, v in kwargs.items() if k in _V0_ARG_NAMES} + v1_kwargs = {k: v for k, v in kwargs.items() if k not in _V0_ARG_NAMES} + v0_mode = bool(v0_args) and not v1_kwargs and connection is None + + if kwargs: + emit() + emit("arguments:") + for key in sorted(kwargs): + shown = "" if key in _SECRET_ARG_NAMES else repr(kwargs[key]) + emit(f" {key} = {shown}") + + emit() + emit("resolution:") + try: + if connection is None: + if v0_mode: + spec = _spec_from_v0_args(v0_args) + else: + spec = S3Connection(**{**v0_args, **v1_kwargs}) + else: + if kwargs: + emit("note: connection= given; extra kwargs ignored.") + spec = connection + resolution = resolve(spec, environ, aws_config) + except Exception as error: + emit(f" RESOLUTION FAILS: {type(error).__name__}: {error}") + emit() + emit("(That failure is the diagnosis: the same arguments would") + emit(" raise at first use. Fix the row above and re-run.)") + return _finish(out, file) + emit( + f" preset: {resolution.preset.name!r} ({resolution.preset_source})" + + ("" if resolution.preset.verified else " [doc-sourced row]") + ) + for name, value, source, handed in _field_rows(resolution): + emit(f" {name:<24} = {value!r:<28} [{source}; {handed}]") + emit(f" credentials: {resolution.credential_provenance}") + emit(f" can_presign: {not spec.anon}") + emit() + emit("environment consulted (non-secret keys only):") + for key, value in resolution.environ_consulted: + emit(f" {key} = {_redact_url(value)!r}") + for warning_class, message in resolution.notes: + emit(f"warning [{warning_class.__name__}]: {message}") + + if v0_mode: + emit() + emit("v0 (S3Store) vs v1 divergence:") + for row in _divergence_rows(v0_args, environ, aws_config): + emit(f" {row}") + except Exception as error: # never raises — D6 + emit( + f"diagnose itself hit an unexpected error: {type(error).__name__}: {error}" + ) + emit("Please report this at https://github.com/i2mint/s3dol/issues") + return _finish(out, file) + + +def _finish(out: io.StringIO, file) -> str: + text = out.getvalue() + print(text, file=file if file is not None else sys.stdout, end="") + return text diff --git a/s3dol/connection.py b/s3dol/connection.py new file mode 100644 index 0000000..799ff35 --- /dev/null +++ b/s3dol/connection.py @@ -0,0 +1,872 @@ +"""``S3Connection`` — the credential + endpoint SSOT, and the pure resolver. + +Design record: `misc/docs/decisions/0012-credential-and-endpoint-resolution.md` +(all of it) on top of `0002-boto3-as-engine.md` (lazy boto3) and +`0003-provider-presets-and-capabilities.md` (presets, SigV4 mandate). + +The two sentences the module follows from (ADR-0012): + +1. **A preset is the caller's explicit endpoint, spelled by name.** It sits + exactly where ``endpoint_url=`` sits — above every environment and + config-file channel. s3dol owns two rungs of the endpoint ladder, not six. +2. **A ``Config`` value s3dol does not set is a channel it leaves open to the + user.** Every field is a ladder with a documented bottom, and "the row makes + no claim" is spelled by *omitting the key*, never by passing ``None``. + +Structure: + +- :class:`S3Connection` — a frozen, picklable spec. No live objects, no + escape hatch for one. Construction performs no I/O and imports no boto3. +- :func:`resolve` — a **pure** function ``(spec, environ, aws_config) -> + Resolution`` where every field carries its provenance. Tier-1 tests cover + the whole ladder with a fake environ dict and zero I/O. +- :func:`load_aws_config` — the separate, *named impure* loader for the + aws-config rungs; pass it (or its result) to :func:`resolve`. +- Credential shapes (ADR-0012 D5) — normalised in ``__post_init__`` to small + frozen, picklable providers; live botocore objects are rejected with + shape-specific messages. + +>>> spec = S3Connection(endpoint_url='http://localhost:9000', +... credentials=('AKIAEXAMPLE', 'sEkR3tVALUE')) +>>> res = resolve(spec, {}) +>>> res.endpoint_url.value, res.endpoint_url.source +('http://localhost:9000', 'endpoint_url=') +>>> res.signature_version.value # never left to botocore (ADR-0003 §4) +'s3v4' +>>> 'sEkR3tVALUE' in repr(spec) # secrets never in repr (ADR-0012 D1) +False +""" + +from __future__ import annotations + +import pickle +import re +import threading +from dataclasses import dataclass, fields +from typing import ( + Any, + Callable, + Literal, + Mapping, + NamedTuple, + Optional, + Union, +) + +from s3dol.errors import ( + AmbiguousResolution, + ConfigurationError, + MissingEndpoint, + PresetConflict, + PresetHostMismatch, +) +from s3dol.presets import ( + Capabilities, + Preset, + detect_preset, + get_preset, + validate_endpoint_hygiene, +) + +# --------------------------------------------------------------------------- # +# Credential shapes (ADR-0012 D5) +# --------------------------------------------------------------------------- # + + +class CredentialProvider: + """Base of s3dol's picklable credential specs. + + Subclasses are small frozen dataclasses that *describe* how to obtain + credentials; nothing live is held (nothing live pickles — measured, ADR-0012 + D5). At client-build time each is adapted into a botocore provider and + **replaces** botocore's resolver via + ``session.register_component('credential_provider', ...)`` — never + prepended (silent fallthrough) and never ``insert_before`` (raises when a + profile is set). + """ + + def provenance(self) -> str: + """A label for diagnose() — never a credential value.""" + raise NotImplementedError + + def _load_botocore_credentials(self): + raise NotImplementedError + + def botocore_provider(self): + from botocore.credentials import CredentialProvider as _BotocoreBase + + spec = self + + class _Adapter(_BotocoreBase): + METHOD = f"s3dol-{type(spec).__name__}" + CANONICAL_NAME = METHOD + + def load(self): + return spec._load_botocore_credentials() + + return _Adapter() + + +@dataclass(frozen=True) +class StaticCredentials(CredentialProvider): + """An explicit key pair (optionally a session token). + + Note: a *static* token expires and cannot refresh — prefer ``profile=`` or + a callable provider for STS/SSO. Pickling this object carries the secret + (that is what "explicit credentials" means); its repr does not. + """ + + access_key: str + secret_key: str + token: Optional[str] = None + + def provenance(self) -> str: + return f"explicit static (access key {self.access_key[:4]}…)" + + def __repr__(self): # secrets never in repr (ADR-0012 D1 §4) + token = ", token=***" if self.token else "" + return ( + f"{type(self).__name__}(access_key={self.access_key[:4]!r}…, " + f"secret_key=***{token})" + ) + + def _load_botocore_credentials(self): + from botocore.credentials import Credentials + + return Credentials( + self.access_key, self.secret_key, self.token, method="s3dol-static" + ) + + +@dataclass(frozen=True) +class ProfileCredentials(CredentialProvider): + """Credentials from a named profile — *credentials only*; the profile's + other config is not selected by this (that is what ``profile=`` on the + connection is for). Rebuilds live, still-refreshable credentials after + unpickling (a frozen dataclass holding only a profile name round-trips).""" + + profile: str + + def provenance(self) -> str: + return f"profile:{self.profile} (via credentials=)" + + def _load_botocore_credentials(self): + import botocore.session + + return botocore.session.Session(profile=self.profile).get_credentials() + + +@dataclass(frozen=True) +class CallableCredentials(CredentialProvider): + """A picklable zero-arg callable returning a mapping with keys + ``access_key``/``secret_key`` (aliases: ``key``/``secret``), optional + ``token``, optional ``expires_at``/``expiry_time`` (ISO 8601). With an + expiry, credentials are refreshable — the callable is re-invoked by + botocore's refresh machinery (advisory 900 s / mandatory 600 s before + expiry).""" + + fetch: Callable[[], Mapping] + + def provenance(self) -> str: + name = getattr(self.fetch, "__qualname__", None) or repr(type(self.fetch)) + return f"callable:{name}" + + @staticmethod + def _normalize(result: Mapping) -> dict: + def pick(*names): + for name in names: + if name in result: + return result[name] + return None + + creds = dict( + access_key=pick("access_key", "key", "aws_access_key_id"), + secret_key=pick("secret_key", "secret", "aws_secret_access_key"), + token=pick("token", "aws_session_token"), + expiry_time=pick("expires_at", "expiry_time"), + ) + if not creds["access_key"] or not creds["secret_key"]: + raise ConfigurationError( + "The credentials callable must return a mapping with " + "'access_key' and 'secret_key' (aliases: 'key'/'secret'); got " + f"keys {sorted(result)}." + ) + return creds + + def _load_botocore_credentials(self): + from botocore.credentials import Credentials, RefreshableCredentials + + first = self._normalize(self.fetch()) + if first["expiry_time"] is None: + return Credentials( + first["access_key"], + first["secret_key"], + first["token"], + method="s3dol-callable", + ) + + def refresh(): + fresh = self._normalize(self.fetch()) + return { + "access_key": fresh["access_key"], + "secret_key": fresh["secret_key"], + "token": fresh["token"], + "expiry_time": fresh["expiry_time"], + } + + return RefreshableCredentials.create_from_metadata( + metadata=refresh(), refresh_using=refresh, method="s3dol-callable" + ) + + +CredentialsInput = Union[ + None, str, tuple, list, Mapping, CredentialProvider, Callable[[], Mapping] +] + +_AWS_CRED_KEYS = ("aws_access_key_id", "aws_secret_access_key", "aws_session_token") + + +def _is_live_botocore_object(value: Any) -> bool: + """Detect live boto3/botocore objects *without importing them* — + construction must not import boto3 (ADR-0002).""" + module = type(value).__module__ or "" + return module.startswith(("botocore", "boto3")) + + +def normalize_credentials(value: CredentialsInput) -> Optional[CredentialProvider]: + """Normalise every accepted credential shape; reject the rest, each with + its own message (ADR-0012 D5). The only door is ``S3Connection.__post_init__``. + + >>> normalize_credentials(None) is None + True + >>> normalize_credentials('prod') + ProfileCredentials(profile='prod') + >>> normalize_credentials(('AKIAXX', 'sec')) + StaticCredentials(access_key='AKIA'…, secret_key=***) + """ + if value is None: + return None + if isinstance(value, CredentialProvider): + return value + if isinstance(value, type) and issubclass(value, CredentialProvider): + raise ConfigurationError( + f"credentials= got the class {value.__name__}; pass an *instance* " + f"(e.g. {value.__name__}(...))." + ) + if _is_live_botocore_object(value): + raise ConfigurationError( + f"credentials= got a live boto3/botocore object " + f"({type(value).__name__}). A resolved credential snapshot cannot " + f"refresh and cannot pickle, and sessions/clients hold sockets. " + f"Pass a profile name, a (key, secret[, token]) tuple, a mapping of " + f"aws_* keys, or a picklable zero-arg callable." + ) + if isinstance(value, str): + return ProfileCredentials(value) # a bare str is ALWAYS a profile name + if isinstance(value, (tuple, list)): + if len(value) not in (2, 3): + raise ConfigurationError( + f"credentials= tuple must be (key, secret) or " + f"(key, secret, token); got {len(value)} items." + ) + return StaticCredentials(*map(str, value)) + if isinstance(value, Mapping): + unknown = set(value) - set(_AWS_CRED_KEYS) + if unknown: + raise ConfigurationError( + f"credentials= mapping has unknown key(s) {sorted(unknown)}; " + f"allowed: {list(_AWS_CRED_KEYS)}." + ) + if not value.get("aws_access_key_id") or not value.get("aws_secret_access_key"): + raise ConfigurationError( + "credentials= mapping needs both 'aws_access_key_id' and " + "'aws_secret_access_key' (a token alone leaves " + "Credentials(None, None, TOKEN) — a measured boto3 hazard)." + ) + return StaticCredentials( + access_key=value["aws_access_key_id"], + secret_key=value["aws_secret_access_key"], + token=value.get("aws_session_token"), + ) + if callable(value): + try: + pickle.dumps(value) + except Exception as error: + raise ConfigurationError( + f"credentials= callable {value!r} is not picklable " + f"({type(error).__name__}). Every s3dol store pickles " + f"(ADR-0012 D1); use a module-level function or a " + f"functools.partial of one." + ) from error + return CallableCredentials(value) + raise ConfigurationError( + f"credentials= got a {type(value).__name__}; accepted: None, a profile " + f"name (str), (key, secret[, token]), a mapping of aws_* keys, a " + f"CredentialProvider instance, or a picklable zero-arg callable." + ) + + +# --------------------------------------------------------------------------- # +# Resolution — pure, provenance-carrying (ADR-0012 D2/D3/D4) +# --------------------------------------------------------------------------- # + + +class Sourced(NamedTuple): + """A resolved value, where it came from, and whether s3dol passes it to + botocore explicitly (``passed=False`` == the key is omitted so the channel + below stays open — "omit, never None").""" + + value: Any + source: str + passed: bool = False + + +@dataclass(frozen=True) +class Resolution: + """The result of :func:`resolve` — everything ``diagnose()`` prints and + everything client-build consumes. Carries provenance labels, never a + credential value.""" + + endpoint_url: Sourced + region_name: Sourced + signature_version: Sourced + addressing_style: Sourced + checksum: Sourced + payload_signing_enabled: Sourced + verify: Sourced + preset: Optional[Preset] + preset_source: str # 'named' | 'detected' | 'fallback:' | 'none' + credential_provenance: str + capabilities: Capabilities + #: The non-secret environment keys consulted, with their values. + environ_consulted: tuple[tuple[str, Optional[str]], ...] + #: Collected (warning class, message) pairs. resolve() never emits them — + #: the impure caller decides (client build warns; diagnose prints rows). + notes: tuple[tuple[type, str], ...] + + +_ENV_ENDPOINT_KEYS = ("AWS_ENDPOINT_URL_S3", "AWS_ENDPOINT_URL") +_ENV_CONSULTED_KEYS = _ENV_ENDPOINT_KEYS + ( + "AWS_DEFAULT_REGION", # botocore does NOT read AWS_REGION; neither do we + "AWS_PROFILE", + "AWS_REQUEST_CHECKSUM_CALCULATION", + "AWS_RESPONSE_CHECKSUM_VALIDATION", +) + +#: Region-looking tokens: ``us-east-1``, ``fr-par``, ``us-west-004``, hetzner's +#: ``fsn1``, and R2's literal ``auto``. +_REGION_TOKEN = re.compile(r"[a-z]{2}-[a-z]+-\d+|[a-z]{2}-[a-z]{3,}|[a-z]{3}\d|auto") + + +def _region_from_host(endpoint_url: Optional[str]) -> Optional[str]: + """Best-effort region capture from a resolved endpoint host.""" + if not endpoint_url: + return None + from urllib.parse import urlsplit + + host = urlsplit(endpoint_url).hostname or "" + match = re.search(r"(?:^|\.)s3[.-]([a-z0-9-]+)\.", host) + candidates = [match.group(1)] if match else [] + labels = host.split(".") + if labels: + candidates.append(labels[0]) + for candidate in candidates: + if _REGION_TOKEN.fullmatch(candidate): + return candidate + return None + + +_CREDENTIAL_KEY_MARKERS = ( + "access_key", + "secret", + "token", + "password", + "credential_process", + "sso_", +) + + +def _scrub_credentials(config: Mapping) -> dict: + """Recursively drop credential-shaped keys from an aws-config mapping.""" + + def scrub(node): + if isinstance(node, Mapping): + return { + k: scrub(v) + for k, v in node.items() + if not any( + marker in str(k).lower() for marker in _CREDENTIAL_KEY_MARKERS + ) + } + return node + + return scrub(config) + + +def load_aws_config() -> Mapping: + """The named, *impure* aws-config loader (ADR-0012 D2): loads the shared + config via botocore (~16 ms — never on the common path; pass this function + as a thunk and :func:`resolve` calls it only if a rung needs it), scrubbed + of anything credential-shaped.""" + import botocore.session + + return _scrub_credentials(botocore.session.Session().full_config) + + +def _profile_section( + aws_config: Mapping, environ: Mapping, profile: Optional[str] +) -> Mapping: + profiles = aws_config.get("profiles", {}) if aws_config else {} + name = profile or environ.get("AWS_PROFILE") or "default" + return profiles.get(name, {}) + + +def _lower_rung_endpoint( + environ: Mapping, aws_config: Mapping, profile: Optional[str] +) -> Optional[tuple[str, str]]: + """The endpoint the rungs *below* s3dol's two would supply — read purely, + for diagnose() and the C1/C2 guards **only**, never to override (ADR-0012 + D3; and never pass ``Config(ignore_configured_endpoint_urls=True)``).""" + for env_key in _ENV_ENDPOINT_KEYS: + value = environ.get(env_key) + if value: + return value, f"env:{env_key}" + if aws_config: + try: + from botocore.configprovider import ConfiguredEndpointProvider + + provider = ConfiguredEndpointProvider( + full_config=aws_config, + scoped_config=_profile_section(aws_config, environ, profile), + client_name="s3", + environ={}, # env rungs already handled above + ) + value = provider.provide() + if value: + return str(value), "aws-config" + except Exception: # config parsing must never fail resolution + return None + return None + + +def _row_region(row: Preset) -> Optional[str]: + """The row's region literal with params bound; ``None`` if it makes no + claim (absent, or a template with unbound placeholders).""" + if not row.region_name: + return None + value, missing = row._bound(row.region_name) + return None if missing else value + + +def resolve( + spec: "S3Connection", + environ: Mapping[str, str], + aws_config: Union[Mapping, Callable[[], Mapping], None] = None, +) -> Resolution: + """Pure resolution: no ``os.environ``, no disk, no network (ADR-0012 D2). + + ``environ`` is a plain mapping (pass ``os.environ`` at real call sites, + a dict in tests). ``aws_config`` is a pre-loaded, credential-scrubbed + config mapping, or a thunk for one (:func:`load_aws_config`) — called only + if a rung needs it. + """ + notes: list[tuple[type, str]] = [] + aws_config_cache: dict = {} + + def config() -> Mapping: + if "value" not in aws_config_cache: + if aws_config is None: + aws_config_cache["value"] = {} + elif callable(aws_config): + aws_config_cache["value"] = aws_config() or {} + else: + aws_config_cache["value"] = aws_config + return aws_config_cache["value"] + + # -- the named row (pass 1 input) + if spec.preset is None: + named = None + elif isinstance(spec.preset, Preset): + named = spec.preset + else: + named = get_preset(spec.preset) + + # -- endpoint ladder (D3): s3dol owns two rungs, then stops + endpoint: Optional[Sourced] = None + if spec.endpoint_url: + endpoint = Sourced(spec.endpoint_url, "endpoint_url=", passed=True) + elif named is not None: + bound = named.bound_endpoint(region_name=spec.region_name) + if bound is not None: + validate_endpoint_hygiene(bound, what=f"preset:{named.name} endpoint") + endpoint = Sourced(bound, f"preset:{named.name}", passed=True) + + lower = None + if endpoint is None: + lower = _lower_rung_endpoint(environ, config(), spec.profile) + if lower is not None: + endpoint = Sourced(lower[0], lower[1], passed=False) + elif named is not None: + missing = named.missing_endpoint_params(region_name=spec.region_name) + if named.requires_endpoint or missing: + raise MissingEndpoint( + f"Preset {named.name!r} needs an endpoint and nothing " + f"supplied one" + + ( + f" (unbound template parameter(s): {sorted(missing)} — " + f"bind with get_preset({named.name!r}).bind(...))" + if missing + else "" + ) + + ". Pass endpoint_url=, or set AWS_ENDPOINT_URL_S3. " + f"Falling back to AWS silently is structurally impossible " + f"(ADR-0012 C1)." + ) + endpoint = Sourced(None, "SDK resolver", passed=False) + else: + endpoint = Sourced(None, "SDK resolver", passed=False) + + # -- C2 guard: a named, patterned row whose endpoint arrived from below + if ( + named is not None + and not named.soft + and named.host_patterns + and not endpoint.passed + and endpoint.value + ): + if not named.matches_host(endpoint.value): + other = detect_preset(endpoint.value) + if other is not None and other.name != named.name: + raise PresetHostMismatch( + f"preset={named.name!r} was named, but the endpoint that " + f"resolved from the environment ({endpoint.source}) matches " + f"{other.name!r}'s host patterns: {endpoint.value!r}. " + f"Pass endpoint_url= explicitly (rung 1 never triggers " + f"this guard), or name the right preset." + ) + notes.append( + ( + AmbiguousResolution, + f"preset={named.name!r} was named, but the endpoint that " + f"resolved from the environment ({endpoint.source}: " + f"{endpoint.value!r}) matches none of its host patterns " + f"{named.host_patterns}. Proceeding; pass endpoint_url= to " + f"silence this.", + ) + ) + + # -- pass 2: pick the settings row (named > detected > fallback) + if named is not None: + row, preset_source = named, "named" + elif endpoint.value: + detected = detect_preset(endpoint.value) + if detected is not None: + row, preset_source = detected, "detected" + else: + row, preset_source = get_preset("generic-s3"), "fallback:generic-s3" + else: + row, preset_source = get_preset("aws"), "fallback:aws" + + # -- region ladder + row_region = _row_region(row) + if spec.region_name: + if row.pinned and row_region and spec.region_name != row_region: + raise PresetConflict( + f"region_name={spec.region_name!r} contradicts the verified " + f"preset {row.name!r}, which pins region {row_region!r}." + ) + region = Sourced(spec.region_name, "region_name=", passed=True) + elif row_region: + region = Sourced(row_region, f"preset:{row.name}", passed=True) + else: + captured = _region_from_host(endpoint.value) + env_region = environ.get("AWS_DEFAULT_REGION") + if captured: + region = Sourced(captured, "endpoint host (best-effort)", passed=True) + elif env_region: + region = Sourced(env_region, "env:AWS_DEFAULT_REGION", passed=True) + elif _profile_section(config(), environ, spec.profile).get("region"): + region = Sourced( + _profile_section(config(), environ, spec.profile)["region"], + "aws-config profile", + passed=True, + ) + else: + region = Sourced("us-east-1", "default", passed=True) + + # -- addressing style ladder: kwarg > row > profile > omit. + # Soft rows (aws, generic-s3) document the expected behaviour but do NOT + # pass it — omitting the key preserves the profile channel and botocore's + # endpoint-dependent default (ADR-0012 D3: omit, never None). + profile_addressing = ( + _profile_section(config(), environ, spec.profile).get("s3", {}) or {} + ).get("addressing_style") + if spec.addressing_style: + addressing = Sourced(spec.addressing_style, "addressing_style=", passed=True) + elif not row.soft: + addressing = Sourced(row.addressing_style, f"preset:{row.name}", passed=True) + elif profile_addressing: + addressing = Sourced(profile_addressing, "aws-config profile", passed=False) + else: + addressing = Sourced( + row.addressing_style, + f"botocore default (≈{row.addressing_style})", + passed=False, + ) + + # -- checksum ladder: kwarg > row > env > botocore default. The env and + # default rungs OMIT the keys, so a user's AWS_REQUEST_CHECKSUM_* env + # workaround keeps working (measured: Config beats env both ways). + env_checksum = environ.get("AWS_REQUEST_CHECKSUM_CALCULATION") or environ.get( + "AWS_RESPONSE_CHECKSUM_VALIDATION" + ) + if spec.checksum: + checksum = Sourced(spec.checksum, "checksum=", passed=True) + elif row.checksum: + checksum = Sourced(row.checksum, f"preset:{row.name}", passed=True) + elif env_checksum: + checksum = Sourced(env_checksum, "env:AWS_*_CHECKSUM_*", passed=False) + else: + checksum = Sourced("when_supported", "botocore default", passed=False) + + # -- payload signing: kwarg > row > omit (no env channel exists) + if spec.payload_signing_enabled is not None: + payload_signing = Sourced( + spec.payload_signing_enabled, "payload_signing_enabled=", passed=True + ) + elif row.payload_signing_enabled is not None: + payload_signing = Sourced( + row.payload_signing_enabled, f"preset:{row.name}", passed=True + ) + else: + payload_signing = Sourced(None, "omitted", passed=False) + + # -- signature version: kwarg > 's3v4'; never from a row, never from env, + # never left to botocore (which silently downgrades to SigV2 in the 12 + # regions endpoints.json lists as v2-capable — ADR-0003 §4 / ADR-0012). + signature = Sourced( + spec.signature_version, + "signature_version=" if spec.signature_version != "s3v4" else "s3dol default", + passed=True, + ) + + verify = Sourced( + spec.verify, + "verify=" if spec.verify is not None else "omitted", + passed=spec.verify is not None, + ) + + # -- credential provenance label (never a value) + if spec.anon: + provenance = "anonymous (UNSIGNED)" + elif spec.credentials is not None: + provenance = spec.credentials.provenance() + elif spec.profile: + provenance = f"profile:{spec.profile}" + elif environ.get("AWS_ACCESS_KEY_ID"): + provenance = "chain (env credentials present)" + else: + provenance = "chain (config file / SSO / IMDS / none)" + + return Resolution( + endpoint_url=endpoint, + region_name=region, + signature_version=signature, + addressing_style=addressing, + checksum=checksum, + payload_signing_enabled=payload_signing, + verify=verify, + preset=row, + preset_source=preset_source, + credential_provenance=provenance, + capabilities=row.capabilities, + environ_consulted=tuple((key, environ.get(key)) for key in _ENV_CONSULTED_KEYS), + notes=tuple(notes), + ) + + +# --------------------------------------------------------------------------- # +# S3Connection (ADR-0012 D1) +# --------------------------------------------------------------------------- # + + +@dataclass(frozen=True) +class S3Connection: + """A frozen, picklable connection spec. No live objects — ever. + + Construction performs no I/O and does not import boto3 (ADR-0002); the + client is built lazily, once, under a per-instance lock (CPython ≥3.12 + removed ``cached_property``'s lock deliberately — measured: 8 racing + threads → 8 clients without one). + + ``__post_init__`` is the only door: every normalisation and contradiction + check happens here, so a bad shape dies at construction with a message, + not later inside botocore. + + >>> S3Connection(anon=True, profile='prod') + Traceback (most recent call last): + ... + s3dol.errors.ConfigurationError: ... + """ + + # WHERE + preset: Union[str, Preset, None] = None + endpoint_url: Optional[str] = None + region_name: Optional[str] = None + # WHO + profile: Optional[str] = None + credentials: Optional[CredentialProvider] = None # normalised in __post_init__ + anon: bool = False + # HOW (None == "do not override" — the ladder continues; see resolve()) + signature_version: str = "s3v4" # never left to botocore (ADR-0003 §4) + addressing_style: Optional[Literal["path", "virtual"]] = None + checksum: Optional[Literal["when_supported", "when_required"]] = None + payload_signing_enabled: Optional[bool] = None + verify: Union[bool, str, None] = None # self-signed MinIO / corporate CA + client_kwargs: tuple = () + # POLICY + deny_means_absent: bool = False # ADR-0004 §3 + + def __post_init__(self): + object.__setattr__(self, "credentials", normalize_credentials(self.credentials)) + if isinstance(self.preset, str): + get_preset(self.preset) # unknown names die here, loudly + if self.endpoint_url is not None: + validate_endpoint_hygiene(self.endpoint_url) + if self.addressing_style not in (None, "path", "virtual"): + raise ConfigurationError( + f"addressing_style must be 'path', 'virtual', or None (ladder " + f"continues); got {self.addressing_style!r}. 'auto' is illegal." + ) + if self.checksum not in (None, "when_supported", "when_required"): + raise ConfigurationError( + f"checksum must be 'when_supported', 'when_required', or None; " + f"got {self.checksum!r}." + ) + if isinstance(self.client_kwargs, Mapping): + object.__setattr__( + self, "client_kwargs", tuple(sorted(self.client_kwargs.items())) + ) + if self.anon: + if self.credentials is not None or self.profile: + raise ConfigurationError( + "anon=True with credentials/profile: UNSIGNED would " + "silently discard them (measured — the request signer's " + "credentials become None). Drop anon, or drop the " + "credentials." + ) + if self.deny_means_absent: + raise ConfigurationError( + "anon=True with deny_means_absent=True: an anonymous HEAD " + "gives 403 on existing keys and 404 on missing ones, so " + "every key would read as absent — a silently empty store " + "(measured under moto; banned by ADR-0001 goal 4)." + ) + object.__setattr__(self, "_lock", threading.Lock()) + + # -- the client -------------------------------------------------------- # + + @property + def client(self): + """The boto3 S3 client — built lazily, once, under the lock.""" + built = self.__dict__.get("_client") + if built is None: + with self._lock: + built = self.__dict__.get("_client") + if built is None: + built = self._build_client() + object.__setattr__(self, "_client", built) + return built + + def resolution( + self, + environ: Optional[Mapping[str, str]] = None, + aws_config: Union[Mapping, Callable, None] = load_aws_config, + ) -> Resolution: + """Resolve against the real environment by default (a thin impure + wrapper over the pure :func:`resolve`).""" + import os + + return resolve(self, environ if environ is not None else os.environ, aws_config) + + def _build_client(self): + import warnings + + import boto3 + import botocore.session + from botocore import UNSIGNED + from botocore.config import Config + from botocore.credentials import CredentialResolver + + resolution = self.resolution() + for warning_class, message in resolution.notes: + warnings.warn(message, warning_class, stacklevel=4) + + s3_settings: dict = {} + if resolution.addressing_style.passed: + s3_settings["addressing_style"] = resolution.addressing_style.value + if resolution.payload_signing_enabled.passed: + s3_settings["payload_signing_enabled"] = ( + resolution.payload_signing_enabled.value + ) + config_kwargs: dict = dict(resolution.preset.config_kwargs or ()) + if resolution.checksum.passed: + config_kwargs["request_checksum_calculation"] = resolution.checksum.value + config_kwargs["response_checksum_validation"] = resolution.checksum.value + if s3_settings: + config_kwargs["s3"] = s3_settings + config = Config( + # UNSIGNED never enters the dataclass (unpicklable) — build-time only + signature_version=( + UNSIGNED if self.anon else resolution.signature_version.value + ), + **config_kwargs, + ) + + botocore_session = ( + botocore.session.Session(profile=self.profile) + if self.profile + else botocore.session.Session() + ) + if self.credentials is not None: + botocore_session.register_component( + "credential_provider", + CredentialResolver(providers=[self.credentials.botocore_provider()]), + ) + + client_kwargs = dict(self.client_kwargs) + if self.verify is not None: + client_kwargs["verify"] = self.verify + return boto3.Session(botocore_session=botocore_session).client( + "s3", + endpoint_url=( + resolution.endpoint_url.value + if resolution.endpoint_url.passed + else None + ), + region_name=resolution.region_name.value, + config=config, + **client_kwargs, + ) + + # -- pickling: mandatory for every connection (ADR-0012 D1 §3) --------- # + + def __getstate__(self): + # Enumerating dataclass fields drops every cache by construction + # (the boto3 client is unpicklable, full stop). + return {f.name: getattr(self, f.name) for f in fields(self)} + + def __setstate__(self, state): + for name, value in state.items(): + object.__setattr__(self, name, value) + object.__setattr__(self, "_lock", threading.Lock()) + + # -- repr: non-default fields only; secrets never (D1 §4) -------------- # + + def __repr__(self): + shown = [] + for f in fields(self): + value = getattr(self, f.name) + if value != f.default: + shown.append(f"{f.name}={value!r}") + return f"{type(self).__name__}({', '.join(shown)})" diff --git a/s3dol/errors.py b/s3dol/errors.py new file mode 100644 index 0000000..fcf1ccc --- /dev/null +++ b/s3dol/errors.py @@ -0,0 +1,550 @@ +"""The s3dol error taxonomy and the single error-translation seam. + +Design record: `misc/docs/decisions/0004-error-taxonomy.md` (the taxonomy, the +``(operation, code, status)`` classification, the 403 policy, ``ObjectArchived``) +and `misc/docs/decisions/0012-credential-and-endpoint-resolution.md` (the +``ConfigurationError`` tree and the ``S3DolWarning`` tree). + +Three rules govern this module: + +1. **One seam.** :func:`translate_s3_errors` is the only place in the package + that catches botocore exceptions. Everything else propagates. +2. **Auth, config and transport errors are never translated to ``KeyError``.** + A missing permission must not look like a missing key. +3. **Classification keys on ``(operation, code, status)``, never on exception + type.** HEAD responses have no body, so botocore's modelled exceptions + (``client.exceptions.NoSuchKey`` …) never fire for them. + +Everything classification-related here is pure: :func:`classify_client_error` +takes plain values (or a synthesized ``ClientError``) and touches no network, +so the bulk of error tests are tier-1 unit tests — including body-less HEAD +errors, which moto cannot produce. + +>>> classify_client_error('GetObject', 'NoSuchKey', 404).kind +'object_absent' +>>> classify_client_error('HeadObject', '404', 404).kind # HEAD is ambiguous +'ambiguous_head' +>>> classify_client_error('GetObject', 'AccessDenied', 403).kind +'access_denied' +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from functools import wraps +from typing import Any, Callable, Iterable, Mapping, Optional + +# --------------------------------------------------------------------------- # +# The exception tree +# --------------------------------------------------------------------------- # + + +class S3Error(Exception): + """Base of every s3dol exception — ``except S3Error`` catches the package.""" + + +class _InformativeKeyError(KeyError): + """A ``KeyError`` whose message is not repr-mangled. + + ``KeyError.__str__`` uses ``repr`` of its args, so a multi-line, helpful + message becomes one line full of ``\\n`` escapes. Every s3dol ``KeyError`` + subclass therefore overrides ``__str__`` (ADR-0004 §5). + + >>> raw = KeyError('line one\\nline two') + >>> '\\\\n' in str(raw) # stdlib mangles + True + >>> ours = _InformativeKeyError('line one\\nline two') + >>> '\\\\n' in str(ours) + False + """ + + def __str__(self): + return str(self.args[0]) if self.args else "" + + +class ObjectNotFound(S3Error, _InformativeKeyError): + """The object is absent. The Mapping contract's ``KeyError``.""" + + +class BucketNotFound(S3Error, _InformativeKeyError): + """The bucket is absent. + + Still a ``KeyError``: for an object operation it is a key-space problem for + the caller; for a bucket operation it is the key of the endpoint mapping. + """ + + +class KeyNotValid(S3Error, _InformativeKeyError, ValueError): + """The key is syntactically invalid (checked before the wire). + + Inherits from both ``KeyError`` and ``ValueError`` deliberately, so either + ``except`` works. Named ``KeyNotValid`` (not ``KeyValidationError``) because + dol already ships two *different* classes called ``KeyValidationError`` + (ADR-0004 §5). + """ + + +class ObjectArchived(S3Error, _InformativeKeyError): + """The object exists but is archived (e.g. Glacier) and cannot be read now. + + A ``KeyError``, deliberately and arguably (ADR-0004 §4): ``k in store`` + stays ``True``, but ``store[k]`` fails as a ``KeyError`` so that + ``store.get(k, default)`` degrades to the not-available branch. Note that + only the *defaulted accessors* degrade — ``dict(store)``, ``.items()`` and + ``Mapping.__eq__`` still raise. ``setdefault``/``pop`` are overridden in the + store to re-raise this instead of silently overwriting the archived object. + """ + + def __init__( + self, + message: str, + *, + storage_class: Optional[str] = None, + restore_status: Optional[str] = None, + restore: Optional[Callable] = None, + ): + super().__init__(message) + self.storage_class = storage_class + self.restore_status = restore_status + self._restore = restore + + def restore(self, days: int = 1, *, tier: str = "Standard"): + """Request restoration of the archived object (if a handle was bound).""" + if self._restore is None: + raise NotSupported( + "No restore capability was bound to this error. " + "Use s3dol.handles(store)[key].restore(...) instead." + ) + return self._restore(days=days, tier=tier) + + +class AccessDenied(S3Error): + """Permission denied. **Not** a ``KeyError`` — a missing permission must + not look like a missing key (ADR-0004 §1/§3).""" + + +class CredentialsError(S3Error): + """Credentials are missing, expired, or invalid.""" + + +class NotSupported(S3Error): + """The operation is not supported by this provider (names both).""" + + +class BucketNotEmpty(S3Error): + """Refusal to delete a non-empty bucket. Use + ``s3dol.delete_bucket(endpoint, name, force=True)`` for the explicit + cascading form (ADR-0010 §3).""" + + +class S3PartialFailure(S3Error): + """A bulk operation partially failed (ADR-0010 §2). + + Not an ``ExceptionGroup`` — the package supports Python 3.10. + + Attributes: + succeeded: keys the backend reported as deleted (absent keys are + reported as deleted too — S3's semantics, documented, not hidden). + failures: mapping of key -> the error for that key. + """ + + def __init__(self, message: str, *, succeeded: list, failures: dict): + super().__init__(message) + self.succeeded = succeeded + self.failures = failures + + +class ConfigurationError(S3Error, ValueError): + """A construction- or resolution-time misconfiguration (ADR-0012 D1).""" + + +class PresetConflict(ConfigurationError): + """An explicit argument contradicts a *pinned* (verified) preset value.""" + + +class PresetHostMismatch(ConfigurationError): + """The resolved endpoint's host matches a *different* provider's patterns + than the preset you named (ADR-0012 C2).""" + + +class MissingEndpoint(ConfigurationError): + """The named preset needs an endpoint (or endpoint parameters) that nothing + supplied (ADR-0012 C1).""" + + +class MissingPresetParam(ConfigurationError): + """A preset endpoint template has an unbound required parameter.""" + + +class UnknownPresetParam(ConfigurationError): + """A parameter was bound that the preset's templates do not use.""" + + +class InvalidEndpoint(ConfigurationError): + """An endpoint URL failed hygiene checks (userinfo/query/fragment — + ADR-0012 D4: a URL is the residual credential carrier).""" + + +# --------------------------------------------------------------------------- # +# The warning tree (ADR-0012 D1). Strict mode is +# ``filterwarnings('error', category=s3dol.errors.S3DolWarning)`` — there is +# deliberately no ``strict=`` kwarg. +# --------------------------------------------------------------------------- # + + +class S3DolWarning(UserWarning): + """Base of every s3dol warning.""" + + +class S3DolResolutionChanged(S3DolWarning): + """v1 resolves this call differently than v0 would have (ADR-0012 D7). + Emitted by the compat shim; dies with it.""" + + +class AmbiguousResolution(S3DolWarning): + """Resolution is legal but ambiguous (e.g. a named preset whose endpoint, + supplied from the environment, matches no known host pattern).""" + + +class AnonymousFallback(S3DolWarning): + """Anonymous access was used as a fallback (reserved for a future + ``anon='auto'`` — ADR-0012 D5 defers that feature to v1.x).""" + + +# --------------------------------------------------------------------------- # +# Redaction +# --------------------------------------------------------------------------- # + +#: Query parameters whose values are secrets (presigned-URL machinery). +_SECRET_QUERY_PARAMS = ( + "X-Amz-Signature", + "X-Amz-Credential", + "X-Amz-Security-Token", + "Signature", + "AWSAccessKeyId", +) + +_SECRET_PARAM_RE = re.compile( + r"(?P{})=[^&\s]*".format("|".join(map(re.escape, _SECRET_QUERY_PARAMS))), + flags=re.IGNORECASE, +) + + +def redact(text: str) -> str: + """Redact signed-URL query secrets from ``text``. + + Error messages may name the operation, bucket, key, endpoint and code, but + never a credential, token, or signed URL — redaction is a tested function, + not a convention (ADR-0004 §7). + + >>> redact('https://b.s3.amazonaws.com/k?X-Amz-Signature=abc123&x=1') + 'https://b.s3.amazonaws.com/k?X-Amz-Signature=REDACTED&x=1' + """ + return _SECRET_PARAM_RE.sub(lambda m: f"{m.group('name')}=REDACTED", text) + + +# --------------------------------------------------------------------------- # +# Classification — pure, table-driven, per-provider-overridable +# --------------------------------------------------------------------------- # + +#: Kinds a classified error can be. Layer B maps kinds to behaviour; in +#: particular ``ambiguous_head`` triggers the one-``HeadBucket`` +#: disambiguation of ADR-0004 §2 (HEAD has no body, so a missing key and a +#: missing bucket are both ``('404', 404)`` on real AWS). +KINDS = ( + "object_absent", + "bucket_absent", + "ambiguous_head", # HeadObject 404: missing key OR missing bucket + "access_denied", + "credentials", + "invalid_key", + "archived", + "not_supported", + "wrong_region", + "delete_marker", # 405 on HEAD of a delete marker (versioned bucket) + "transient", + "unknown", +) + + +@dataclass(frozen=True) +class Classified: + """The result of classifying a backend error.""" + + kind: str + exception_type: Optional[type] = None # None => re-raise the original + + +#: Rule rows: ``((operation, code, status), kind)``. ``None`` is a wildcard. +#: First match wins; more specific rows come first. +Rule = tuple[tuple[Optional[str], Optional[str], Optional[int]], str] + +_KIND_TO_EXC: dict[str, Optional[type]] = { + "object_absent": ObjectNotFound, + "bucket_absent": BucketNotFound, + "ambiguous_head": ObjectNotFound, # resolved by Layer B before raising + "access_denied": AccessDenied, + "credentials": CredentialsError, + "invalid_key": KeyNotValid, + "archived": ObjectArchived, + "not_supported": NotSupported, + "wrong_region": ConfigurationError, + "delete_marker": ObjectNotFound, + "transient": None, # propagate; botocore retries + "unknown": None, # propagate untouched +} + +#: The default classification table (ADR-0004 §2, measured against botocore). +DEFAULT_RULES: tuple[Rule, ...] = ( + # -- modelled codes (GET/DELETE have bodies, so codes are real words) + ((None, "NoSuchKey", None), "object_absent"), + ((None, "NoSuchBucket", None), "bucket_absent"), + ((None, "NoSuchUpload", None), "object_absent"), + # -- HEAD has no body: botocore synthesizes the code from the HTTP status. + # On real AWS, HeadObject-on-missing-key and HeadObject-on-missing-bucket + # are BOTH ('404', 404) — indistinguishable. moto returns a body + # (Code='NoSuchBucket'/'NoSuchKey'), which the modelled rows above catch, + # so this row only fires on real, body-less responses. + (("HeadObject", "404", 404), "ambiguous_head"), + (("HeadBucket", "404", 404), "bucket_absent"), + (("HeadObject", "404", None), "ambiguous_head"), + (("HeadBucket", "404", None), "bucket_absent"), + # -- versioned-bucket delete marker: HEAD gives 405 Method Not Allowed + (("HeadObject", None, 405), "delete_marker"), + # -- wrong region + ((None, None, 301), "wrong_region"), + ((None, "PermanentRedirect", None), "wrong_region"), + ((None, "AuthorizationHeaderMalformed", None), "wrong_region"), + # -- archived objects + ((None, "InvalidObjectState", None), "archived"), + # -- credentials + ((None, "ExpiredToken", None), "credentials"), + ((None, "InvalidAccessKeyId", None), "credentials"), + ((None, "SignatureDoesNotMatch", None), "credentials"), + ((None, "TokenRefreshRequired", None), "credentials"), + # -- permissions (see ADR-0004 §3 for the deny_means_absent policy) + ((None, "AccessDenied", None), "access_denied"), + ((None, "AllAccessDisabled", None), "access_denied"), + ((None, "403", 403), "access_denied"), + ((None, None, 403), "access_denied"), + # -- key validity that escaped the client-side check + ((None, "KeyTooLongError", None), "invalid_key"), + # -- throttling / transient: propagate, botocore retries + ((None, "SlowDown", None), "transient"), + ((None, "RequestTimeout", None), "transient"), + ((None, "ServiceUnavailable", None), "transient"), + ((None, "InternalError", None), "transient"), + ((None, None, 503), "transient"), +) + +#: Provider overrides live on the preset (ADR-0004 §6 — per-connection, +#: explicit, inspectable; NOT process-global). E.g. Supabase returns 400 where +#: AWS returns 404 on HeadBucket: +#: ``(('HeadBucket', None, 400), 'bucket_absent')``. + + +def _match(rule_key, operation, code, status) -> bool: + op, c, s = rule_key + return ( + (op is None or op == operation) + and (c is None or c == code) + and (s is None or s == status) + ) + + +def classify_client_error( + operation: Optional[str], + code: Optional[str], + status: Optional[int], + *, + overrides: Iterable[Rule] = (), +) -> Classified: + """Classify a backend error by ``(operation, code, status)``. + + ``overrides`` rows (typically from a preset) are consulted first, so a + provider divergence is a table row, never a code branch. + + >>> classify_client_error('HeadBucket', None, 400, + ... overrides=((('HeadBucket', None, 400), 'bucket_absent'),)).kind + 'bucket_absent' + """ + for rules in (tuple(overrides), DEFAULT_RULES): + for key, kind in rules: + if _match(key, operation, code, status): + return Classified(kind=kind, exception_type=_KIND_TO_EXC.get(kind)) + return Classified(kind="unknown", exception_type=None) + + +def code_of(client_error: Any) -> Optional[str]: + """The ``Error.Code`` of a botocore ``ClientError`` (or ``None``).""" + try: + return client_error.response["Error"]["Code"] + except (AttributeError, KeyError, TypeError): + return None + + +def status_of(client_error: Any) -> Optional[int]: + """The HTTP status of a botocore ``ClientError`` (or ``None``).""" + try: + return int(client_error.response["ResponseMetadata"]["HTTPStatusCode"]) + except (AttributeError, KeyError, TypeError, ValueError): + return None + + +def _describe( + *, + operation: str, + code: Optional[str], + bucket: Optional[str], + key: Optional[str], + endpoint: Optional[str], + detail: str = "", +) -> str: + """Build the standard error message: operation, bucket, key, endpoint, + code — and nothing secret (ADR-0004 §7).""" + parts = [f"{operation} failed"] + if bucket is not None: + parts.append(f"bucket={bucket!r}") + if key is not None: + parts.append(f"key={key!r}") + if endpoint is not None: + parts.append(f"endpoint={endpoint!r}") + if code is not None: + parts.append(f"code={code}") + msg = ": ".join([parts[0], ", ".join(parts[1:])]) if parts[1:] else parts[0] + if detail: + msg = f"{msg}. {detail}" + return redact(msg) + + +def translate_client_error( + error: Any, + *, + operation: str, + bucket: Optional[str] = None, + key: Optional[str] = None, + endpoint: Optional[str] = None, + overrides: Iterable[Rule] = (), + deny_means_absent: bool = False, + head_ambiguity_resolver: Optional[Callable[[], bool]] = None, +) -> Optional[S3Error]: + """Translate a botocore ``ClientError`` into the s3dol taxonomy. + + Returns the replacement exception, or ``None`` meaning *propagate the + original* (transient/unknown — rule 2 of the module docstring). + + ``head_ambiguity_resolver`` resolves the body-less ``HeadObject`` 404 + (ADR-0004 §2): it returns ``True`` iff the bucket is known reachable, in + which case the miss really is the object's. It is only called for the + ambiguous case, so the common path costs nothing. + + ``deny_means_absent=True`` classifies an object-op 403 as + ``ObjectNotFound`` — for deployments knowingly running the canonical + least-privilege policy that omits ``s3:ListBucket`` (ADR-0004 §3). + """ + code, status = code_of(error), status_of(error) + classified = classify_client_error(operation, code, status, overrides=overrides) + kind = classified.kind + + if kind == "ambiguous_head": + if head_ambiguity_resolver is not None and head_ambiguity_resolver(): + kind = "object_absent" + elif head_ambiguity_resolver is not None: + kind = "bucket_absent" + else: + kind = "object_absent" # documented default when no resolver bound + + if kind == "access_denied": + if deny_means_absent and key is not None: + kind = "object_absent" + else: + return AccessDenied( + _describe( + operation=operation, + code=code, + bucket=bucket, + key=key, + endpoint=endpoint, + detail=( + "Permission denied. If this key may simply be absent, note " + "that the canonical least-privilege policy (no s3:ListBucket) " + "makes every miss a 403; opt into that reading with " + "S3Connection(deny_means_absent=True)." + ), + ) + ) + + if kind in ("transient", "unknown"): + return None + + exc_type = _KIND_TO_EXC[kind] + detail = { + "wrong_region": "The bucket lives in a different region than the client.", + "delete_marker": "The current version is a delete marker (versioned bucket).", + "credentials": "Credentials are missing, expired, or invalid.", + }.get(kind, "") + message = _describe( + operation=operation, + code=code, + bucket=bucket, + key=key, + endpoint=endpoint, + detail=detail, + ) + if exc_type is ObjectArchived: + return ObjectArchived(message) + return exc_type(message) + + +def translate_s3_errors( + *, + operation: str, + key_arg: Optional[str] = None, + **translate_kwargs, +): + """The one seam: decorate a Layer B method so botocore's ``ClientError`` + becomes the s3dol taxonomy (ADR-0004 §1). + + ``key_arg`` names the decorated function's key *parameter* (resolved via + the call's kwargs/positionals — never an attribute, so the key is not lost + the way azuredol's ``key_arg='blob'`` loses it). The store instance is + expected to provide ``_error_context()`` returning + ``dict(bucket=..., endpoint=..., overrides=..., deny_means_absent=..., + head_ambiguity_resolver=...)`` — Layer B owns that context. + """ + + def decorator(func): + import inspect + + sig = inspect.signature(func) + + @wraps(func) + def wrapper(self, *args, **kwargs): + from botocore.exceptions import ClientError, NoCredentialsError + + try: + return func(self, *args, **kwargs) + except ClientError as error: + key = None + if key_arg is not None: + bound = sig.bind(self, *args, **kwargs) + key = bound.arguments.get(key_arg) + context = dict(getattr(self, "_error_context", dict)()) + context.update(translate_kwargs) + replacement = translate_client_error( + error, operation=operation, key=key, **context + ) + if replacement is None: + raise + raise replacement from error + except NoCredentialsError as error: + raise CredentialsError( + "No AWS credentials found. Configure credentials (env vars, " + "~/.aws/credentials, SSO, or S3Connection(credentials=...)), " + "or pass anon=True for public data." + ) from error + + return wrapper + + return decorator diff --git a/s3dol/presets.py b/s3dol/presets.py new file mode 100644 index 0000000..2a14a50 --- /dev/null +++ b/s3dol/presets.py @@ -0,0 +1,439 @@ +"""Provider presets: config rows, not subclasses. + +Design record: `misc/docs/decisions/0003-provider-presets-and-capabilities.md` +(the survey of 16 backends, the checksum analysis, capabilities policy) amended +by `misc/docs/decisions/0012-credential-and-endpoint-resolution.md` §D4 +(``params``/``host_patterns``/``requires_endpoint``/``soft``/``config_kwargs``, +derived ``pinned``; ``signature_version`` and ``client_kwargs`` moved to the +connection; ``addressing_style`` mandatory with ``'auto'`` illegal; the +structural no-credential rule). + +A provider divergence is a row in this registry — never a subclass, never an +endpoint-substring branch in the hot path. Users register their own rows: + +>>> row = Preset(name='mycorp', addressing_style='path', +... endpoint_url='https://s3.mycorp.example', requires_endpoint=False) +>>> register(row) +>>> get_preset('mycorp').endpoint_url +'https://s3.mycorp.example' +>>> unregister('mycorp') # (cleanup for this doctest) + +Most builtin rows are **doc-sourced, not verified against a live endpoint** — +they say so via ``verified=False``. Wrong values mostly fail loudly, which is +the acceptable failure mode; ``pinned`` (which makes a row's region literal +conflict-raising) is derived and *gated on* ``verified``, so it is dormant +until a row earns evidence (ADR-0012 D4). +""" + +from __future__ import annotations + +import string +from dataclasses import dataclass, replace +from fnmatch import fnmatch +from typing import Any, Literal, Optional +from urllib.parse import urlsplit + +from s3dol.errors import ( + ConfigurationError, + InvalidEndpoint, + MissingPresetParam, + UnknownPresetParam, + Rule, +) + +# --------------------------------------------------------------------------- # +# Capabilities +# --------------------------------------------------------------------------- # + + +@dataclass(frozen=True) +class Capabilities: + """What a backend supports. A *static declared table*, never a probe + (a probe costs a round-trip, is wrong under partial permissions, and can't + be trusted anyway — ADR-0003 §2). Missing capabilities are handled + per-capability: emulate when exact (batch delete -> loop), substitute when + equivalent (ListObjectsV2 -> V1), raise ``NotSupported`` when there is no + honest fallback. Never silently degrade correctness.""" + + list_objects_v2: bool = True + batch_delete: bool = True + presigned_post: bool = True + multipart: bool = True + max_multipart_parts: int = 10_000 + min_part_size: int = 5 * 2**20 + object_tagging: bool = True + versioning: bool = True + conditional_writes: bool = True + consistency: Literal["strong", "read-after-write", "eventual"] = "strong" + + +DEFAULT_CAPABILITIES = Capabilities() + + +# --------------------------------------------------------------------------- # +# Preset +# --------------------------------------------------------------------------- # + +#: ``Config``-constructor kwargs a preset row may carry. Scalars only, and +#: ``proxies``/``proxies_config``/``client_context_params`` are structurally +#: excluded — a preset must not be able to smuggle a credential-bearing proxy +#: URL (ADR-0012 D4). +CONFIG_KWARGS_ALLOWLIST = frozenset( + {"connect_timeout", "read_timeout", "max_pool_connections", "tcp_keepalive"} +) + +_SCALAR_TYPES = (str, int, float, bool, type(None)) + + +def _template_params(template: str) -> set: + """Placeholder names in a ``str.format``-style template. + + >>> sorted(_template_params('https://{account_id}.r2.example/{x}')) + ['account_id', 'x'] + """ + return { + name for _, name, _, _ in string.Formatter().parse(template) if name is not None + } + + +def validate_endpoint_hygiene(url: str, *, what: str = "endpoint_url") -> None: + """Refuse credential-carrying URL shapes: userinfo in the netloc, a query + string, or a fragment (ADR-0012 D4 — a query-string endpoint is accepted by + botocore and retained verbatim in ``client.meta.endpoint_url``, so it would + leak into ``diagnose()`` and tracebacks). + + >>> validate_endpoint_hygiene('https://s3.example.com') + >>> validate_endpoint_hygiene('https://user:pw@s3.example.com') + Traceback (most recent call last): + ... + s3dol.errors.InvalidEndpoint: ... + """ + parts = urlsplit(url) + if "@" in parts.netloc: + raise InvalidEndpoint( + f"{what} must not carry userinfo (user[:password]@host): got a URL " + f"with '@' in its netloc. Pass credentials via " + f"S3Connection(credentials=...), never in a URL." + ) + if parts.query: + raise InvalidEndpoint(f"{what} must not carry a query string: {url!r}") + if parts.fragment: + raise InvalidEndpoint(f"{what} must not carry a fragment: {url!r}") + + +@dataclass(frozen=True) +class Preset: + """One provider's facts. Public, shareable, committable config — which is + why there is **no credential slot**, structurally (ADR-0012 D4). + + ``endpoint_url``/``region_name`` may contain ``{placeholders}``; bind them + with :meth:`bind`. A row whose endpoint template has an unbound *required* + placeholder contributes no endpoint rung (ADR-0012 C1). + """ + + name: str + #: Mandatory; no default; ``'auto'`` is illegal (ADR-0012 D4). + addressing_style: Literal["path", "virtual"] + endpoint_url: Optional[str] = None + region_name: Optional[str] = None + #: ``None`` == the row makes no claim (the ladder continues — ADR-0012 D3). + checksum: Optional[Literal["when_supported", "when_required"]] = None + payload_signing_enabled: Optional[bool] = None + capabilities: Capabilities = DEFAULT_CAPABILITIES + #: Presigning sometimes genuinely differs from the API config (Hetzner + #: presigns virtual while the API is path; R2 presigns only on the S3 API + #: domain) — ADR-0003 §1. + presign_endpoint_url: Optional[str] = None + presign_addressing_style: Optional[Literal["path", "virtual"]] = None + #: Bound template parameters, e.g. ``(('account_id', 'abc'),)``. + params: tuple[tuple[str, str], ...] = () + #: ``fnmatch`` patterns over the endpoint *hostname*; used for detection + #: (pass 2) and the C2 named-vs-resolved guard. + host_patterns: tuple[str, ...] = () + #: The provider cannot be reached without an explicit endpoint. + requires_endpoint: bool = False + #: Soft rows are last-resort fallbacks (``aws``, ``generic-s3``): they + #: never win detection over a non-soft row and never trigger the C2 guard. + soft: bool = False + #: Extra ``botocore.Config`` kwargs — allowlisted, scalars only. + config_kwargs: tuple[tuple[str, Any], ...] = () + #: Error-classification overrides, consulted before the default table + #: (ADR-0004 §2/§6): rows of ``((operation, code, status), kind)``. + error_overrides: tuple[Rule, ...] = () + #: ``True`` only when the row was verified against a live endpoint (with a + #: date in the row's docstring/comment). Builtin rows are doc-sourced. + verified: bool = False + + @property + def pinned(self) -> bool: + """Whether the row's region literal is authoritative enough to make a + conflicting explicit ``region_name=`` raise ``PresetConflict``. + Derived, and gated on ``verified`` — dormant while rows are + doc-sourced (ADR-0012 D4).""" + return ( + self.verified + and self.region_name is not None + and not _template_params(self.region_name) + ) + + def __post_init__(self): + if self.addressing_style not in ("path", "virtual"): + raise ConfigurationError( + f"Preset {self.name!r}: addressing_style must be 'path' or " + f"'virtual' (got {self.addressing_style!r}). 'auto' is illegal: " + f"every row must state what the provider actually needs " + f"(ADR-0012 D4)." + ) + for url_field in ("endpoint_url", "presign_endpoint_url"): + url = getattr(self, url_field) + if url is not None: + validate_endpoint_hygiene(url, what=f"Preset.{url_field}") + for key, value in self.config_kwargs: + if key not in CONFIG_KWARGS_ALLOWLIST: + raise ConfigurationError( + f"Preset {self.name!r}: config_kwargs key {key!r} is not " + f"allowlisted (allowed: {sorted(CONFIG_KWARGS_ALLOWLIST)}). " + f"This list is closed to keep presets credential-free." + ) + if not isinstance(value, _SCALAR_TYPES): + raise ConfigurationError( + f"Preset {self.name!r}: config_kwargs values must be " + f"scalars; {key!r} is a {type(value).__name__}." + ) + unknown = set(dict(self.params)) - self.template_params() + if unknown: + raise UnknownPresetParam( + f"Preset {self.name!r}: bound params {sorted(unknown)} appear in " + f"no template. Templates use: {sorted(self.template_params())}." + ) + + def template_params(self) -> set: + """All placeholder names across this row's templates.""" + names = set() + for template in ( + self.endpoint_url, + self.region_name, + self.presign_endpoint_url, + ): + if template: + names |= _template_params(template) + return names + + def bind(self, **params: str) -> "Preset": + """A copy of this row with template ``params`` bound. + + >>> r2 = get_preset('r2').bind(account_id='abc123') + >>> r2.bound_endpoint() + 'https://abc123.r2.cloudflarestorage.com' + """ + unknown = set(params) - self.template_params() + if unknown: + raise UnknownPresetParam( + f"Preset {self.name!r} has no template parameter(s) " + f"{sorted(unknown)}; it uses {sorted(self.template_params())}." + ) + merged = {**dict(self.params), **{k: str(v) for k, v in params.items()}} + return replace(self, params=tuple(sorted(merged.items()))) + + def _bound(self, template: Optional[str], *, extra: Optional[dict] = None): + """``(value, missing_params)`` for a template under current bindings.""" + if not template: + return None, set() + bindings = {**dict(self.params), **(extra or {})} + missing = _template_params(template) - set(bindings) + if missing: + return None, missing + return template.format(**bindings), set() + + def bound_endpoint(self, *, region_name: Optional[str] = None) -> Optional[str]: + """The endpoint URL with params bound, or ``None`` if the template has + unbound placeholders (C1: an unbound row contributes no rung — + ``preset='minio'`` silently becoming AWS is structurally impossible). + + ``region_name`` binds a ``{region_name}`` placeholder, per the region + ladder's first rung (ADR-0012 D3).""" + extra = {"region_name": region_name} if region_name else None + value, _missing = self._bound(self.endpoint_url, extra=extra) + return value + + def missing_endpoint_params(self, *, region_name: Optional[str] = None) -> set: + """Which placeholders keep :meth:`bound_endpoint` from producing a URL.""" + extra = {"region_name": region_name} if region_name else None + _value, missing = self._bound(self.endpoint_url, extra=extra) + return missing + + def matches_host(self, endpoint_url: str) -> bool: + """Whether ``endpoint_url``'s hostname matches this row's patterns.""" + host = urlsplit(endpoint_url).hostname or "" + return any(fnmatch(host, pattern) for pattern in self.host_patterns) + + +# --------------------------------------------------------------------------- # +# The registry +# --------------------------------------------------------------------------- # + +_REGISTRY: dict[str, Preset] = {} + + +def register(preset: Preset, *, overwrite: bool = False) -> None: + """Add a row. Open-closed: adding a provider is adding a row.""" + if not overwrite and preset.name in _REGISTRY: + raise ConfigurationError( + f"A preset named {preset.name!r} is already registered. " + f"Pass overwrite=True to replace it." + ) + _REGISTRY[preset.name] = preset + + +def unregister(name: str) -> None: + """Remove a row (mostly for tests).""" + _REGISTRY.pop(name, None) + + +def get_preset(name: str) -> Preset: + """Look a row up by name; unknown names raise, naming the registry.""" + try: + return _REGISTRY[name] + except KeyError: + raise ConfigurationError( + f"Unknown preset {name!r}. Registered: {sorted(_REGISTRY)}. " + f"Register your own with s3dol.presets.register(Preset(...))." + ) from None + + +def available_presets() -> tuple[str, ...]: + return tuple(sorted(_REGISTRY)) + + +def detect_preset(endpoint_url: str) -> Optional[Preset]: + """Detection (resolution pass 2): the non-soft row whose ``host_patterns`` + match the resolved endpoint's host. Two matching rows is genuinely + ambiguous and raises; zero yields ``None`` (the caller falls back to + ``generic-s3``/``aws`` — ADR-0012 D4).""" + matches = [ + p for p in _REGISTRY.values() if not p.soft and p.matches_host(endpoint_url) + ] + if len(matches) > 1: + names = sorted(p.name for p in matches) + raise ConfigurationError( + f"Endpoint {endpoint_url!r} matches more than one preset's host " + f"patterns: {names}. Name one explicitly with preset=." + ) + return matches[0] if matches else None + + +# --------------------------------------------------------------------------- # +# Builtin rows (ADR-0003 §1's table; doc-sourced — verified=False throughout) +# --------------------------------------------------------------------------- # + +_SUPABASE_ERROR_OVERRIDES: tuple[Rule, ...] = ( + # Supabase returns 400 where AWS returns 404 on HeadBucket (ADR-0004 §2). + (("HeadBucket", None, 400), "bucket_absent"), +) + +_BUILTIN_PRESETS = ( + # AWS: the reference semantics. SDK-resolved endpoint; soft fallback row. + Preset( + name="aws", + addressing_style="virtual", + soft=True, + host_patterns=("*.amazonaws.com",), + ), + # Fallback for "some endpoint resolved, nothing matched": makes no claim + # beyond path addressing (botocore's own behaviour for custom endpoints), + # so it is byte-identical to today for endpoint-passing callers. + Preset(name="generic-s3", addressing_style="path", soft=True), + Preset( + name="minio", + addressing_style="path", + region_name="us-east-1", # conventional + requires_endpoint=True, + ), + Preset( + name="r2", + addressing_style="virtual", + endpoint_url="https://{account_id}.r2.cloudflarestorage.com", + region_name="auto", # R2's literal region string + host_patterns=("*.r2.cloudflarestorage.com",), + # Presign only on the S3 API domain, never a custom domain (docs). + ), + Preset( + name="scaleway", + addressing_style="virtual", + endpoint_url="https://s3.{region_name}.scw.cloud", + host_patterns=("s3.*.scw.cloud",), + capabilities=Capabilities(max_multipart_parts=1000), + ), + Preset( + name="hetzner", + addressing_style="path", # virtual breaks plain Get/Put (TLS cert) + presign_addressing_style="virtual", # docs: needed for presigned URLs + endpoint_url="https://{location}.your-objectstorage.com", + region_name="{location}", # must repeat the location + payload_signing_enabled=False, + host_patterns=("*.your-objectstorage.com",), + ), + Preset( + name="backblaze", + addressing_style="virtual", + endpoint_url="https://s3.{region_name}.backblazeb2.com", + checksum="when_required", # mandatory: rejects default SDK checksums + host_patterns=("s3.*.backblazeb2.com",), + capabilities=Capabilities(presigned_post=False), + ), + Preset( + name="wasabi", + addressing_style="path", # the vendor's own advice + endpoint_url="https://s3.{region_name}.wasabisys.com", + host_patterns=("s3.*.wasabisys.com",), + ), + Preset( + name="gcs", + addressing_style="virtual", + endpoint_url="https://storage.googleapis.com", + host_patterns=("storage.googleapis.com",), + capabilities=Capabilities(list_objects_v2=False, batch_delete=False), + ), + Preset( + name="supabase", + addressing_style="path", # forced + endpoint_url="https://{project_ref}.storage.supabase.co/storage/v1/s3", + checksum="when_required", # the ADR-0003 §3 corruption fix + host_patterns=("*.supabase.co", "*.supabase.in"), + error_overrides=_SUPABASE_ERROR_OVERRIDES, + ), + Preset( + name="digitalocean", + addressing_style="virtual", + endpoint_url="https://{region_name}.digitaloceanspaces.com", + host_patterns=("*.digitaloceanspaces.com",), + ), + Preset( + name="tigris", + addressing_style="virtual", + endpoint_url="https://fly.storage.tigris.dev", + host_patterns=("fly.storage.tigris.dev", "*.storage.tigris.dev"), + ), + Preset( + name="ceph", + addressing_style="path", + requires_endpoint=True, # behaviour varies more by release than vendor + ), + # Test doubles + Preset( + name="localstack", + addressing_style="path", + endpoint_url="http://localhost:4566", + region_name="us-east-1", + ), + Preset( + name="moto", + addressing_style="path", + endpoint_url="http://localhost:5000", + region_name="us-east-1", + ), +) + +for _preset in _BUILTIN_PRESETS: + register(_preset) +del _preset diff --git a/s3dol/tests/test_connection.py b/s3dol/tests/test_connection.py new file mode 100644 index 0000000..34fc75c --- /dev/null +++ b/s3dol/tests/test_connection.py @@ -0,0 +1,519 @@ +"""Tests for ``S3Connection`` and the pure resolver (ADR-0012). + +Tier 1 throughout (fake environ dicts, zero I/O), plus a small moto tier-2 +section at the bottom proving the built client's presigning is SigV4 — the +regression test for issue #10, at the layer that fixes it. +""" + +import pickle +import threading + +import pytest + +from s3dol.connection import ( + CallableCredentials, + CredentialProvider, + ProfileCredentials, + Resolution, + S3Connection, + StaticCredentials, + load_aws_config, + normalize_credentials, + resolve, +) +from s3dol.errors import ( + AmbiguousResolution, + ConfigurationError, + MissingEndpoint, + PresetHostMismatch, +) +from s3dol.presets import get_preset + + +def fetch_test_credentials(): + """Module-level (picklable) credential callable for tests.""" + return {"access_key": "AKIATEST", "secret_key": "sEkR3t"} + + +# --------------------------------------------------------------------------- # +# credential normalisation (D5) +# --------------------------------------------------------------------------- # + + +def test_none_stays_none_meaning_the_chain(): + assert normalize_credentials(None) is None + + +def test_bare_str_is_always_a_profile_name(): + provider = normalize_credentials("prod") + assert isinstance(provider, ProfileCredentials) and provider.profile == "prod" + + +@pytest.mark.parametrize("shape", [("AK", "SK"), ("AK", "SK", "TOK"), ["AK", "SK"]]) +def test_tuple_shapes(shape): + provider = normalize_credentials(shape) + assert isinstance(provider, StaticCredentials) + assert provider.access_key == "AK" and provider.secret_key == "SK" + + +def test_tuple_wrong_arity_rejected(): + with pytest.raises(ConfigurationError, match="tuple"): + normalize_credentials(("only-one",)) + + +def test_mapping_of_aws_keys(): + provider = normalize_credentials( + {"aws_access_key_id": "AK", "aws_secret_access_key": "SK"} + ) + assert isinstance(provider, StaticCredentials) + + +def test_mapping_unknown_key_rejected_with_the_allowed_list(): + with pytest.raises(ConfigurationError, match="aws_access_key_id"): + normalize_credentials({"access_key": "AK"}) + + +def test_token_only_mapping_rejected(): + # boto3 would leave Credentials(None, None, TOKEN) — a measured hazard. + with pytest.raises(ConfigurationError, match="token"): + normalize_credentials({"aws_session_token": "TOK"}) + + +def test_provider_instance_passes_through(): + provider = StaticCredentials("AK", "SK") + assert normalize_credentials(provider) is provider + + +def test_provider_class_rejected(): + with pytest.raises(ConfigurationError, match="instance"): + normalize_credentials(StaticCredentials) + + +def test_live_botocore_object_rejected(): + from botocore.credentials import Credentials + + with pytest.raises(ConfigurationError, match="live"): + normalize_credentials(Credentials("AK", "SK")) + + +def test_unpicklable_callable_rejected(): + with pytest.raises(ConfigurationError, match="picklable"): + normalize_credentials(lambda: {"access_key": "a", "secret_key": "b"}) + + +def test_picklable_callable_accepted(): + provider = normalize_credentials(fetch_test_credentials) + assert isinstance(provider, CallableCredentials) + assert isinstance(provider, CredentialProvider) + + +def test_alien_type_rejected(): + with pytest.raises(ConfigurationError, match="accepted"): + normalize_credentials(42) + + +def test_static_credentials_repr_redacts(): + provider = StaticCredentials("AKIAEXAMPLE", "sEkR3tVALUE", "tOkEnVALUE") + text = repr(provider) + assert "sEkR3tVALUE" not in text and "tOkEnVALUE" not in text + assert "AKIA" in text # identity by prefix only + + +def test_callable_credentials_normalize_aliases_and_reject_garbage(): + assert CallableCredentials._normalize({"key": "a", "secret": "b"})["access_key"] == "a" + with pytest.raises(ConfigurationError, match="access_key"): + CallableCredentials._normalize({"nope": 1}) + + +# --------------------------------------------------------------------------- # +# S3Connection construction (D1) +# --------------------------------------------------------------------------- # + + +def test_construction_is_lazy_no_client_no_io(): + spec = S3Connection(endpoint_url="http://localhost:9000") + assert "_client" not in spec.__dict__ + + +def test_anon_with_credentials_raises(): + with pytest.raises(ConfigurationError, match="anon"): + S3Connection(anon=True, credentials=("AK", "SK")) + with pytest.raises(ConfigurationError, match="anon"): + S3Connection(anon=True, profile="prod") + + +def test_anon_with_deny_means_absent_raises(): + with pytest.raises(ConfigurationError, match="silently empty"): + S3Connection(anon=True, deny_means_absent=True) + + +def test_auto_addressing_style_is_illegal(): + with pytest.raises(ConfigurationError, match="auto"): + S3Connection(addressing_style="auto") + + +def test_bad_checksum_rejected(): + with pytest.raises(ConfigurationError, match="checksum"): + S3Connection(checksum="always") + + +def test_unknown_preset_name_dies_at_construction(): + with pytest.raises(ConfigurationError, match="Unknown preset"): + S3Connection(preset="no-such-provider") + + +def test_endpoint_hygiene_applies_to_the_kwarg(): + with pytest.raises(ConfigurationError): + S3Connection(endpoint_url="https://user:pw@host") + + +def test_client_kwargs_mapping_normalised_to_tuple(): + spec = S3Connection(client_kwargs={"use_ssl": False}) + assert spec.client_kwargs == (("use_ssl", False),) + hash(spec) # stays hashable + + +def test_equal_specs_hash_equal_so_caches_are_safe(): + a = S3Connection(endpoint_url="http://x:1", credentials=("AK", "SK")) + b = S3Connection(endpoint_url="http://x:1", credentials=("AK", "SK")) + assert a == b and hash(a) == hash(b) + assert a != S3Connection(endpoint_url="http://x:2", credentials=("AK", "SK")) + + +def test_repr_shows_only_non_defaults_and_no_secrets(): + spec = S3Connection( + endpoint_url="http://x:1", credentials=("AKIAEXAMPLE", "sEkR3tVALUE") + ) + text = repr(spec) + assert "sEkR3tVALUE" not in text + assert "http://x:1" in text + assert "deny_means_absent" not in text # default fields stay quiet + + +# --------------------------------------------------------------------------- # +# pickling (D1 §3: mandatory for every connection) +# --------------------------------------------------------------------------- # + + +def test_pickle_round_trip_preserves_equality(): + spec = S3Connection( + preset="r2", + credentials=("AK", "SK", "TOK"), + region_name="auto", + client_kwargs={"use_ssl": True}, + ) + clone = pickle.loads(pickle.dumps(spec)) + assert clone == spec + assert isinstance(clone._lock, type(threading.Lock())) + + +def test_pickle_drops_a_cached_client(): + spec = S3Connection(endpoint_url="http://x:1") + object.__setattr__(spec, "_client", object()) # simulate a built client + clone = pickle.loads(pickle.dumps(spec)) + assert "_client" not in clone.__dict__ + + +def test_callable_credentials_pickle(): + spec = S3Connection(credentials=fetch_test_credentials) + clone = pickle.loads(pickle.dumps(spec)) + assert isinstance(clone.credentials, CallableCredentials) + + +# --------------------------------------------------------------------------- # +# the client is built once, under the lock (D1 §2) +# --------------------------------------------------------------------------- # + + +def test_client_build_races_produce_one_client(monkeypatch): + builds = [] + barrier = threading.Barrier(8) + + def fake_build(self): + builds.append(1) + return object() + + monkeypatch.setattr(S3Connection, "_build_client", fake_build) + spec = S3Connection(endpoint_url="http://x:1") + clients = [] + + def grab(): + barrier.wait() + clients.append(spec.client) + + threads = [threading.Thread(target=grab) for _ in range(8)] + for t in threads: + t.start() + for t in threads: + t.join() + assert len(builds) == 1 + assert len({id(c) for c in clients}) == 1 + + +# --------------------------------------------------------------------------- # +# resolve(): purity and the ladders (D2/D3/D4) +# --------------------------------------------------------------------------- # + + +def test_resolve_is_pure_wrt_the_process_environment(monkeypatch): + monkeypatch.setenv("AWS_ENDPOINT_URL_S3", "http://should-not-be-seen:1") + resolution = resolve(S3Connection(), {}) + assert resolution.endpoint_url.value is None + assert resolution.endpoint_url.source == "SDK resolver" + + +def test_endpoint_rung1_kwarg_wins_over_everything(): + spec = S3Connection( + endpoint_url="http://explicit:1", preset=get_preset("r2").bind(account_id="a") + ) + resolution = resolve(spec, {"AWS_ENDPOINT_URL_S3": "http://env:1"}) + assert resolution.endpoint_url == ("http://explicit:1", "endpoint_url=", True) + + +def test_endpoint_rung2_bound_preset(): + spec = S3Connection(preset=get_preset("r2").bind(account_id="abc")) + resolution = resolve(spec, {}) + assert resolution.endpoint_url.value == "https://abc.r2.cloudflarestorage.com" + assert resolution.endpoint_url.source == "preset:r2" + assert resolution.endpoint_url.passed is True + + +def test_endpoint_env_rungs_are_reported_not_passed(): + resolution = resolve(S3Connection(), {"AWS_ENDPOINT_URL_S3": "http://env:1"}) + assert resolution.endpoint_url.value == "http://env:1" + assert resolution.endpoint_url.source == "env:AWS_ENDPOINT_URL_S3" + assert resolution.endpoint_url.passed is False # botocore reads it itself + lower = resolve(S3Connection(), {"AWS_ENDPOINT_URL": "http://gen:1"}) + assert lower.endpoint_url.source == "env:AWS_ENDPOINT_URL" + priority = resolve( + S3Connection(), + {"AWS_ENDPOINT_URL_S3": "http://svc:1", "AWS_ENDPOINT_URL": "http://gen:1"}, + ) + assert priority.endpoint_url.value == "http://svc:1" # service beats generic + + +def test_endpoint_aws_config_rung(): + aws_config = {"profiles": {"default": {"endpoint_url": "http://cfg:1"}}} + resolution = resolve(S3Connection(), {}, aws_config) + assert resolution.endpoint_url.value == "http://cfg:1" + assert resolution.endpoint_url.source == "aws-config" + assert resolution.endpoint_url.passed is False + + +def test_c1_requires_endpoint_raises_missing_endpoint(): + with pytest.raises(MissingEndpoint) as info: + resolve(S3Connection(preset="minio"), {}) + assert "AWS_ENDPOINT_URL_S3" in str(info.value) + + +def test_c1_unbound_template_raises_and_names_the_params(): + with pytest.raises(MissingEndpoint) as info: + resolve(S3Connection(preset="r2"), {}) + assert "account_id" in str(info.value) + + +def test_c1_env_endpoint_satisfies_a_requiring_preset(): + resolution = resolve( + S3Connection(preset="minio"), {"AWS_ENDPOINT_URL_S3": "http://minio:9000"} + ) + assert resolution.endpoint_url.value == "http://minio:9000" + assert resolution.preset.name == "minio" + + +def test_c2_env_endpoint_matching_another_provider_raises(): + with pytest.raises(PresetHostMismatch, match="wasabi"): + resolve( + S3Connection(preset="r2"), + {"AWS_ENDPOINT_URL_S3": "https://s3.eu-central-1.wasabisys.com"}, + ) + + +def test_c2_env_endpoint_matching_nothing_warns_not_raises(): + resolution = resolve( + S3Connection(preset="r2"), + {"AWS_ENDPOINT_URL_S3": "https://proxy.mycorp.internal"}, + ) + assert any(cls is AmbiguousResolution for cls, _ in resolution.notes) + + +def test_c2_never_fires_for_rung1(): + # endpoint_url= is the documented escape (rung 1 never triggers C2). + resolution = resolve( + S3Connection(preset="r2", endpoint_url="https://proxy.mycorp.internal"), {} + ) + assert resolution.notes == () + + +def test_pass2_detection_supplies_the_full_row(): + resolution = resolve( + S3Connection(endpoint_url="https://abc.r2.cloudflarestorage.com"), {} + ) + assert resolution.preset.name == "r2" + assert resolution.preset_source == "detected" + assert resolution.addressing_style.value == "virtual" + assert resolution.region_name.value == "auto" # the row's literal + + +def test_pass2_fallbacks(): + with_endpoint = resolve(S3Connection(endpoint_url="http://unknown:1"), {}) + assert with_endpoint.preset.name == "generic-s3" + assert with_endpoint.preset_source == "fallback:generic-s3" + without = resolve(S3Connection(), {}) + assert without.preset.name == "aws" + assert without.preset_source == "fallback:aws" + + +def test_region_ladder(): + kwarg = resolve(S3Connection(region_name="eu-west-3"), {}) + assert kwarg.region_name == ("eu-west-3", "region_name=", True) + captured = resolve( + S3Connection(endpoint_url="https://s3.eu-central-2.amazonaws.com"), {} + ) + assert captured.region_name.value == "eu-central-2" + assert "endpoint host" in captured.region_name.source + env = resolve(S3Connection(), {"AWS_DEFAULT_REGION": "ap-south-1"}) + assert env.region_name == ("ap-south-1", "env:AWS_DEFAULT_REGION", True) + profile = resolve( + S3Connection(), {}, {"profiles": {"default": {"region": "sa-east-1"}}} + ) + assert profile.region_name.value == "sa-east-1" + bottom = resolve(S3Connection(), {}) + assert bottom.region_name == ("us-east-1", "default", True) + + +def test_aws_region_env_var_is_deliberately_ignored(): + # botocore does not read AWS_REGION; v1 must not "helpfully" add it. + resolution = resolve(S3Connection(), {"AWS_REGION": "eu-north-1"}) + assert resolution.region_name.value == "us-east-1" + + +def test_addressing_ladder_soft_rows_omit(): + hard = resolve(S3Connection(endpoint_url="https://storage.googleapis.com"), {}) + assert hard.addressing_style.passed is True # gcs is a real claim + soft = resolve(S3Connection(), {}) # aws fallback: omit, channel open + assert soft.addressing_style.passed is False + profile = resolve( + S3Connection(), + {}, + {"profiles": {"default": {"s3": {"addressing_style": "virtual"}}}}, + ) + assert profile.addressing_style.value == "virtual" + assert profile.addressing_style.source == "aws-config profile" + assert profile.addressing_style.passed is False # omit preserves the channel + kwarg = resolve(S3Connection(addressing_style="path"), {}) + assert kwarg.addressing_style == ("path", "addressing_style=", True) + + +def test_checksum_ladder_env_stays_open(): + row = resolve( + S3Connection(preset=get_preset("backblaze").bind()), + {"AWS_ENDPOINT_URL_S3": "https://s3.us-west-004.backblazeb2.com"}, + ) + assert row.checksum == ("when_required", "preset:backblaze", True) + env = resolve( + S3Connection(), {"AWS_REQUEST_CHECKSUM_CALCULATION": "when_required"} + ) + assert env.checksum.value == "when_required" + assert env.checksum.passed is False # omit the key: env keeps working + bottom = resolve(S3Connection(), {}) + assert bottom.checksum == ("when_supported", "botocore default", False) + kwarg = resolve(S3Connection(checksum="when_required"), {}) + assert kwarg.checksum.passed is True + + +def test_payload_signing_ladder(): + hetzner = resolve( + S3Connection(preset="hetzner", endpoint_url="https://fsn1.your-objectstorage.com"), + {}, + ) + assert hetzner.payload_signing_enabled == (False, "preset:hetzner", True) + default = resolve(S3Connection(), {}) + assert default.payload_signing_enabled == (None, "omitted", False) + + +def test_signature_version_always_passed(): + resolution = resolve(S3Connection(), {}) + assert resolution.signature_version.value == "s3v4" + assert resolution.signature_version.passed is True + + +def test_credential_provenance_labels_never_values(): + explicit = resolve(S3Connection(credentials=("AKIAXYZ", "sEkR3t")), {}) + assert "sEkR3t" not in str(explicit.credential_provenance) + assert "AKIA" in explicit.credential_provenance + profile = resolve(S3Connection(profile="prod"), {}) + assert profile.credential_provenance == "profile:prod" + env = resolve(S3Connection(), {"AWS_ACCESS_KEY_ID": "AK"}) + assert "env" in env.credential_provenance + anon = resolve(S3Connection(anon=True), {}) + assert "anonymous" in anon.credential_provenance + + +def test_resolution_is_a_value_and_consults_named_env_keys(): + resolution = resolve(S3Connection(), {"AWS_DEFAULT_REGION": "eu-west-1"}) + assert isinstance(resolution, Resolution) + consulted = dict(resolution.environ_consulted) + assert consulted["AWS_DEFAULT_REGION"] == "eu-west-1" + assert "AWS_ENDPOINT_URL_S3" in consulted + + +def test_load_aws_config_is_credential_scrubbed(): + config = load_aws_config() # impure by design; must still be safe to print + flat = repr(config).lower() + assert "secret" not in flat + assert "aws_access_key_id" not in flat + assert "session_token" not in flat + + +# --------------------------------------------------------------------------- # +# tier 2 (moto): the built client — SigV4 presigning is issue #10's regression +# --------------------------------------------------------------------------- # + +moto = pytest.importorskip("moto") + + +@pytest.fixture +def _mock_aws_env(monkeypatch): + for var in ("AWS_ENDPOINT_URL", "AWS_ENDPOINT_URL_S3", "AWS_PROFILE"): + monkeypatch.delenv(var, raising=False) + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "testing") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "testing") + monkeypatch.setenv("AWS_DEFAULT_REGION", "us-east-1") + + +def test_client_presigns_sigv4_even_in_us_east_1(_mock_aws_env): + # us-east-1 is precisely a region where botocore, left to itself, silently + # downgrades presigning to SigV2 (ADR-0003 §4). Assert on the URL, never on + # client.meta.config.signature_version — it reports 's3v4' while emitting + # SigV2 (measured). + with moto.mock_aws(): + spec = S3Connection(region_name="us-east-1") + client = spec.client + client.create_bucket(Bucket="test-bucket") + client.put_object(Bucket="test-bucket", Key="k", Body=b"v") + url = client.generate_presigned_url( + "get_object", Params={"Bucket": "test-bucket", "Key": "k"}, ExpiresIn=60 + ) + assert "X-Amz-Algorithm=AWS4-HMAC-SHA256" in url + assert "AWSAccessKeyId" not in url # the SigV2 tell + + +def test_anon_client_is_unsigned_and_still_builds(_mock_aws_env): + from botocore import UNSIGNED + + with moto.mock_aws(): + spec = S3Connection(anon=True, region_name="us-east-1") + assert spec.client.meta.config.signature_version is UNSIGNED + + +def test_static_credentials_reach_the_request(_mock_aws_env): + with moto.mock_aws(): + spec = S3Connection( + credentials=("AKIAINJECTED", "sEkR3t"), region_name="us-east-1" + ) + client = spec.client + client.create_bucket(Bucket="cred-bucket") + url = client.generate_presigned_url( + "get_object", Params={"Bucket": "cred-bucket", "Key": "k"}, ExpiresIn=60 + ) + assert "AKIAINJECTED" in url # the injected identity signed, not the env one diff --git a/s3dol/tests/test_diagnose.py b/s3dol/tests/test_diagnose.py new file mode 100644 index 0000000..402a156 --- /dev/null +++ b/s3dol/tests/test_diagnose.py @@ -0,0 +1,144 @@ +"""Tests for ``s3dol.diagnose()`` (ADR-0012 D6/D7). All tier 1.""" + +import io + +from s3dol._diagnose import _spec_from_v0_args, _v0_mangle, diagnose + + +def run_diagnose(**kwargs): + """diagnose() against a controlled environment, capturing the report.""" + kwargs.setdefault("environ", {}) + return diagnose(file=io.StringIO(), **kwargs) + + +# --------------------------------------------------------------------------- # +# never raises (D6) +# --------------------------------------------------------------------------- # + + +def test_reports_resolution_for_plain_v1_kwargs(): + report = run_diagnose(endpoint_url="http://localhost:9000") + assert "resolution:" in report + assert "http://localhost:9000" in report + assert "endpoint_url=" in report + assert "can_presign: True" in report + + +def test_a_failing_resolution_is_a_row_not_a_raise(): + report = run_diagnose(preset="minio") # C1: MissingEndpoint + assert "RESOLUTION FAILS" in report + assert "MissingEndpoint" in report + + +def test_a_failing_construction_is_a_row_not_a_raise(): + report = run_diagnose(anon=True, profile="prod") + assert "RESOLUTION FAILS" in report + assert "ConfigurationError" in report + + +def test_utter_garbage_never_raises(): + report = run_diagnose(bogus_kwarg=42) + assert isinstance(report, str) and report # something readable came back + + +def test_secret_values_never_echo(): + report = run_diagnose( + bucket_name="b", + aws_access_key_id="AKIAEXAMPLE", + aws_secret_access_key="sEkR3tVALUE", + ) + assert "sEkR3tVALUE" not in report + assert "" in report + + +def test_prints_to_the_given_file_and_returns_the_text(): + sink = io.StringIO() + text = diagnose(endpoint_url="http://x:1", environ={}, file=sink) + assert sink.getvalue() == text + + +# --------------------------------------------------------------------------- # +# the v0 mangler (D7): ~20 frozen lines mapping 1:1 onto v0 +# --------------------------------------------------------------------------- # + + +def test_mangle_endpoint_dropped_under_env_credentials(): + branch, effective = _v0_mangle( + {"endpoint_url": "http://r2:1", "aws_access_key_id": "EXPLICIT"}, + {"AWS_ACCESS_KEY_ID": "ENVKEY", "AWS_SECRET_ACCESS_KEY": "ENVSECRET"}, + ) + assert "endpoint dropped" in branch + assert effective["endpoint_url"] is None # the v0 data-misrouting bug + assert effective["aws_access_key_id"] == "ENVKEY" # env over explicit + assert effective["aws_session_token"] is None # nulled, silently + assert effective["region_name"] is None # nulled, silently + + +def test_mangle_explicit_branch_when_no_env_credentials(): + branch, effective = _v0_mangle( + {"endpoint_url": "http://r2:1", "aws_access_key_id": "AK"}, {} + ) + assert branch == "explicit" + assert effective["endpoint_url"] == "http://r2:1" # honoured on this path + + +def test_mangle_profile_with_explicit_creds_discards_profile(): + branch, _ = _v0_mangle( + {"profile_name": "prod", "aws_access_key_id": "AK"}, {} + ) + assert "discarded" in branch + + +def test_mangle_supabase_and_skip_branches_are_excluded(): + assert _v0_mangle({"endpoint_url": "https://x.supabase.co/s3"}, {})[1] is None + assert _v0_mangle({"is_supabase_endpoint": True}, {})[1] is None + assert _v0_mangle({"skip_bucket_check": True}, {})[1] is None + + +def test_spec_from_v0_args_maps_env_profile_to_the_chain(): + spec = _spec_from_v0_args({"profile_name": "environment variables"}) + assert spec.profile is None # env creds are the top of botocore's own chain + + +# --------------------------------------------------------------------------- # +# the divergence table +# --------------------------------------------------------------------------- # + + +def test_divergence_flags_the_endpoint_drop(): + report = run_diagnose( + bucket_name="b", + endpoint_url="http://r2.example:1", + aws_access_key_id="AK", + aws_secret_access_key="SK", + environ={"AWS_ACCESS_KEY_ID": "ENVK", "AWS_SECRET_ACCESS_KEY": "ENVS"}, + ) + assert "divergence" in report + assert "CHANGES ON UPGRADE" in report + assert "endpoint dropped" in report + + +def test_no_divergence_when_v0_and_v1_agree(): + report = run_diagnose(bucket_name="b", endpoint_url="http://minio:9000") + assert "no divergence" in report + + +def test_excluded_branch_prints_the_reason(): + report = run_diagnose(bucket_name="b", endpoint_url="https://x.supabase.co/s3") + assert "EXCLUDED" in report + assert "process-global" in report + + +def test_credential_identity_divergence_is_a_predicate_not_a_value(): + report = run_diagnose( + bucket_name="b", + aws_access_key_id="AKIAEXPLICIT", + aws_secret_access_key="sEkR3tVALUE", + environ={ + "AWS_ACCESS_KEY_ID": "AKIAFROMENV", + "AWS_SECRET_ACCESS_KEY": "ENVSEKRET99", + }, + ) + assert "credentials:" in report + assert "sEkR3tVALUE" not in report + assert "ENVSEKRET99" not in report # no env secret value either diff --git a/s3dol/tests/test_errors.py b/s3dol/tests/test_errors.py new file mode 100644 index 0000000..461a21a --- /dev/null +++ b/s3dol/tests/test_errors.py @@ -0,0 +1,311 @@ +"""Tier-1 tests for the error taxonomy and the classification seam (ADR-0004). + +Pure unit tests over *synthesized* ``ClientError``s — including body-less HEAD +errors, which moto cannot produce (it returns an error body on HEAD where real +AWS returns none), so tier 2 could never cover them. +""" + +import pytest + +from s3dol.errors import ( + AccessDenied, + BucketNotFound, + Classified, + ConfigurationError, + CredentialsError, + KeyNotValid, + ObjectArchived, + ObjectNotFound, + S3DolWarning, + S3Error, + S3PartialFailure, + classify_client_error, + code_of, + redact, + status_of, + translate_client_error, + translate_s3_errors, +) + + +def synthesized_client_error(code=None, status=None, operation="GetObject"): + """A ClientError like botocore raises, built with no network.""" + from botocore.exceptions import ClientError + + response = {"ResponseMetadata": {"HTTPStatusCode": status}} + if code is not None: + response["Error"] = {"Code": code, "Message": "synthesized"} + return ClientError(response, operation) + + +# --------------------------------------------------------------------------- # +# taxonomy shape +# --------------------------------------------------------------------------- # + + +def test_mapping_contract_errors_are_keyerrors(): + for exc_type in (ObjectNotFound, BucketNotFound, KeyNotValid, ObjectArchived): + assert issubclass(exc_type, KeyError) + assert issubclass(exc_type, S3Error) + + +def test_key_not_valid_is_also_a_valueerror(): + # Deliberately both, so either `except` works (ADR-0004 §5). + assert issubclass(KeyNotValid, ValueError) + + +def test_auth_errors_are_never_keyerrors(): + # A missing permission must not look like a missing key (ADR-0004 §1). + for exc_type in (AccessDenied, CredentialsError): + assert not issubclass(exc_type, KeyError) + assert issubclass(exc_type, S3Error) + + +def test_configuration_error_is_a_valueerror(): + assert issubclass(ConfigurationError, ValueError) + + +def test_except_s3error_catches_the_package(): + with pytest.raises(S3Error): + raise ObjectNotFound("k") + with pytest.raises(S3Error): + raise AccessDenied("nope") + + +def test_keyerror_str_is_not_repr_mangled(): + message = "line one\nline two: helpful detail" + assert str(ObjectNotFound(message)) == message # stdlib KeyError would repr() + assert "\\n" not in str(BucketNotFound(message)) + + +def test_partial_failure_carries_the_split(): + error = S3PartialFailure( + "3 of 5 deleted", succeeded=["a", "b", "c"], failures={"d": AccessDenied("x")} + ) + assert error.succeeded == ["a", "b", "c"] + assert set(error.failures) == {"d"} + + +def test_object_archived_carries_restore_metadata(): + error = ObjectArchived("archived", storage_class="GLACIER", restore_status=None) + assert error.storage_class == "GLACIER" + with pytest.raises(S3Error): + error.restore() # nothing bound -> loud, not silent + + +def test_warning_tree_rooted_in_s3dolwarning(): + from s3dol.errors import ( + AmbiguousResolution, + AnonymousFallback, + S3DolResolutionChanged, + ) + + for warning_class in (S3DolResolutionChanged, AmbiguousResolution, AnonymousFallback): + assert issubclass(warning_class, S3DolWarning) + assert issubclass(warning_class, UserWarning) + + +# --------------------------------------------------------------------------- # +# classification +# --------------------------------------------------------------------------- # + + +@pytest.mark.parametrize( + "operation, code, status, kind", + [ + ("GetObject", "NoSuchKey", 404, "object_absent"), + ("GetObject", "NoSuchBucket", 404, "bucket_absent"), + # HEAD has no body: the code is the synthesized status string. + ("HeadObject", "404", 404, "ambiguous_head"), + ("HeadBucket", "404", 404, "bucket_absent"), + ("HeadObject", "404", None, "ambiguous_head"), + # delete-marker HEAD on a versioned bucket + ("HeadObject", "405", 405, "delete_marker"), + # wrong region + ("GetObject", "PermanentRedirect", 301, "wrong_region"), + ("ListObjectsV2", None, 301, "wrong_region"), + # archived + ("GetObject", "InvalidObjectState", 403, "archived"), + # credentials + ("GetObject", "ExpiredToken", 400, "credentials"), + ("GetObject", "InvalidAccessKeyId", 403, "credentials"), + ("PutObject", "SignatureDoesNotMatch", 403, "credentials"), + # permissions + ("GetObject", "AccessDenied", 403, "access_denied"), + ("HeadObject", "403", 403, "access_denied"), + # transient + ("GetObject", "SlowDown", 503, "transient"), + ("PutObject", None, 503, "transient"), + # unknown propagates + ("GetObject", "TotallyNewCode", 418, "unknown"), + ], +) +def test_classification_table(operation, code, status, kind): + assert classify_client_error(operation, code, status).kind == kind + + +def test_provider_overrides_are_consulted_first(): + # Supabase returns 400 where AWS returns 404 on HeadBucket (ADR-0004 §2). + overrides = ((("HeadBucket", None, 400), "bucket_absent"),) + assert ( + classify_client_error("HeadBucket", "400", 400, overrides=overrides).kind + == "bucket_absent" + ) + # ...and only for the operation they name. + assert ( + classify_client_error("GetObject", "400", 400, overrides=overrides).kind + == "unknown" + ) + + +def test_classified_is_a_value(): + assert Classified(kind="object_absent", exception_type=ObjectNotFound).kind == ( + "object_absent" + ) + + +def test_code_and_status_extractors_are_total(): + error = synthesized_client_error("NoSuchKey", 404) + assert code_of(error) == "NoSuchKey" + assert status_of(error) == 404 + assert code_of(object()) is None + assert status_of(None) is None + + +# --------------------------------------------------------------------------- # +# translation +# --------------------------------------------------------------------------- # + + +def test_translate_object_absent(): + error = synthesized_client_error("NoSuchKey", 404) + replacement = translate_client_error( + error, operation="GetObject", bucket="b", key="k", endpoint="http://e" + ) + assert isinstance(replacement, ObjectNotFound) + for fragment in ("GetObject", "'b'", "'k'", "http://e"): + assert fragment in str(replacement) + + +def test_translate_ambiguous_head_with_resolver(): + error = synthesized_client_error("404", 404, operation="HeadObject") + bucket_reachable = translate_client_error( + error, operation="HeadObject", key="k", head_ambiguity_resolver=lambda: True + ) + assert isinstance(bucket_reachable, ObjectNotFound) + bucket_missing = translate_client_error( + error, operation="HeadObject", key="k", head_ambiguity_resolver=lambda: False + ) + assert isinstance(bucket_missing, BucketNotFound) + + +def test_translate_access_denied_names_the_policy_cause(): + error = synthesized_client_error("AccessDenied", 403) + replacement = translate_client_error(error, operation="HeadObject", key="k") + assert isinstance(replacement, AccessDenied) + assert "s3:ListBucket" in str(replacement) + assert "deny_means_absent" in str(replacement) + + +def test_deny_means_absent_opt_in(): + error = synthesized_client_error("AccessDenied", 403) + replacement = translate_client_error( + error, operation="HeadObject", key="k", deny_means_absent=True + ) + assert isinstance(replacement, ObjectNotFound) + + +def test_transient_and_unknown_propagate(): + assert ( + translate_client_error( + synthesized_client_error("SlowDown", 503), operation="GetObject" + ) + is None + ) + assert ( + translate_client_error( + synthesized_client_error("BrandNew", 418), operation="GetObject" + ) + is None + ) + + +# --------------------------------------------------------------------------- # +# the decorator seam +# --------------------------------------------------------------------------- # + + +class _FakeStore: + def __init__(self, error): + self._error = error + + def _error_context(self): + return dict(bucket="b", endpoint="http://e") + + @translate_s3_errors(operation="GetObject", key_arg="k") + def read(self, k): + raise self._error + + @translate_s3_errors(operation="GetObject", key_arg="k") + def ok(self, k): + return k.upper() + + +def test_decorator_translates_and_carries_the_key(): + store = _FakeStore(synthesized_client_error("NoSuchKey", 404)) + with pytest.raises(ObjectNotFound) as info: + store.read("my-key") + assert "my-key" in str(info.value) + assert "'b'" in str(info.value) + # the botocore original is chained, not swallowed + assert info.value.__cause__ is not None + + +def test_decorator_extracts_key_passed_by_keyword(): + store = _FakeStore(synthesized_client_error("NoSuchKey", 404)) + with pytest.raises(ObjectNotFound) as info: + store.read(k="kw-key") + assert "kw-key" in str(info.value) + + +def test_decorator_propagates_unknown_errors_untouched(): + original = synthesized_client_error("BrandNewCode", 418) + store = _FakeStore(original) + with pytest.raises(type(original)) as info: + store.read("k") + assert info.value is original + + +def test_decorator_translates_missing_credentials(): + from botocore.exceptions import NoCredentialsError + + store = _FakeStore(NoCredentialsError()) + with pytest.raises(CredentialsError) as info: + store.read("k") + assert "anon=True" in str(info.value) + + +def test_decorator_leaves_success_alone(): + store = _FakeStore(None) + assert store.ok("abc") == "ABC" + + +# --------------------------------------------------------------------------- # +# redaction +# --------------------------------------------------------------------------- # + + +@pytest.mark.parametrize( + "secret_param", + ["X-Amz-Signature", "X-Amz-Credential", "X-Amz-Security-Token", "AWSAccessKeyId"], +) +def test_redact_presigned_url_secrets(secret_param): + url = f"https://b.s3.amazonaws.com/k?{secret_param}=SECRETVALUE&keep=1" + redacted = redact(url) + assert "SECRETVALUE" not in redacted + assert "keep=1" in redacted + + +def test_redact_is_case_insensitive_and_leaves_plain_text_alone(): + assert redact("nothing secret here") == "nothing secret here" + assert "hunter2" not in redact("?x-amz-signature=hunter2") diff --git a/s3dol/tests/test_presets.py b/s3dol/tests/test_presets.py new file mode 100644 index 0000000..23199dc --- /dev/null +++ b/s3dol/tests/test_presets.py @@ -0,0 +1,226 @@ +"""Tier-1 tests for the preset registry (ADR-0003 as amended by ADR-0012 D4).""" + +import pytest + +from s3dol.errors import ConfigurationError, InvalidEndpoint, UnknownPresetParam +from s3dol.presets import ( + CONFIG_KWARGS_ALLOWLIST, + Capabilities, + Preset, + _REGISTRY, + available_presets, + detect_preset, + get_preset, + register, + unregister, + validate_endpoint_hygiene, +) + + +@pytest.fixture +def scratch_registry(): + """Snapshot/restore the registry so tests can register freely.""" + snapshot = dict(_REGISTRY) + yield + _REGISTRY.clear() + _REGISTRY.update(snapshot) + + +# --------------------------------------------------------------------------- # +# row validation +# --------------------------------------------------------------------------- # + + +def test_addressing_style_is_mandatory_and_auto_is_illegal(): + with pytest.raises(TypeError): + Preset(name="x") # no default: every row must state it + with pytest.raises(ConfigurationError, match="auto"): + Preset(name="x", addressing_style="auto") + + +@pytest.mark.parametrize( + "bad_url", + [ + "https://user:pw@s3.example.com", # userinfo: the credential carrier + "https://s3.example.com?token=abc", # query + "https://s3.example.com#frag", # fragment + ], +) +def test_endpoint_hygiene(bad_url): + with pytest.raises(InvalidEndpoint): + validate_endpoint_hygiene(bad_url) + with pytest.raises(InvalidEndpoint): + Preset(name="x", addressing_style="path", endpoint_url=bad_url) + + +def test_config_kwargs_allowlist_is_closed(): + with pytest.raises(ConfigurationError, match="allowlisted"): + Preset( + name="x", + addressing_style="path", + config_kwargs=(("proxies", "http://user:pw@proxy"),), + ) + with pytest.raises(ConfigurationError, match="scalar"): + Preset( + name="x", + addressing_style="path", + config_kwargs=(("connect_timeout", {"nested": 1}),), + ) + ok = Preset( + name="x", addressing_style="path", config_kwargs=(("connect_timeout", 5),) + ) + assert dict(ok.config_kwargs)["connect_timeout"] == 5 + assert "proxies" not in CONFIG_KWARGS_ALLOWLIST + + +def test_unknown_bound_param_raises(): + with pytest.raises(UnknownPresetParam): + Preset( + name="x", + addressing_style="path", + endpoint_url="https://{a}.example.com", + params=(("b", "1"),), + ) + + +# --------------------------------------------------------------------------- # +# template binding (C1's raw material) +# --------------------------------------------------------------------------- # + + +def test_bind_and_bound_endpoint(): + r2 = get_preset("r2") + assert r2.bound_endpoint() is None # unbound template -> no rung + assert r2.missing_endpoint_params() == {"account_id"} + bound = r2.bind(account_id="abc123") + assert bound.bound_endpoint() == "https://abc123.r2.cloudflarestorage.com" + assert bound.missing_endpoint_params() == set() + + +def test_bind_unknown_param_raises(): + with pytest.raises(UnknownPresetParam, match="account_id"): + get_preset("r2").bind(nonsense="x") + + +def test_region_kwarg_binds_region_template(): + scaleway = get_preset("scaleway") + assert scaleway.bound_endpoint() is None + assert scaleway.bound_endpoint(region_name="fr-par") == ( + "https://s3.fr-par.scw.cloud" + ) + + +# --------------------------------------------------------------------------- # +# pinned is derived, gated on verified (dormant in v1.0) +# --------------------------------------------------------------------------- # + + +def test_pinned_is_dormant_while_doc_sourced(): + row = Preset( + name="x", addressing_style="path", region_name="eu-west-1", verified=False + ) + assert row.pinned is False + verified = Preset( + name="x", addressing_style="path", region_name="eu-west-1", verified=True + ) + assert verified.pinned is True + + +def test_templated_region_never_pins(): + row = Preset( + name="x", addressing_style="path", region_name="{location}", verified=True + ) + assert row.pinned is False + + +# --------------------------------------------------------------------------- # +# registry +# --------------------------------------------------------------------------- # + + +def test_register_refuses_silent_overwrite(scratch_registry): + row = Preset(name="mine", addressing_style="path") + register(row) + with pytest.raises(ConfigurationError, match="already registered"): + register(row) + register(row, overwrite=True) # explicit is fine + unregister("mine") + assert "mine" not in available_presets() + + +def test_get_preset_unknown_names_the_registry(): + with pytest.raises(ConfigurationError) as info: + get_preset("no-such-provider") + assert "register" in str(info.value).lower() + assert "aws" in str(info.value) # the row list is in the message + + +# --------------------------------------------------------------------------- # +# detection (resolution pass 2) +# --------------------------------------------------------------------------- # + + +def test_detection_by_host_pattern(): + detected = detect_preset("https://abc123.r2.cloudflarestorage.com") + assert detected is not None and detected.name == "r2" + assert detect_preset("https://storage.googleapis.com").name == "gcs" + assert detect_preset("https://myref.storage.supabase.co/storage/v1/s3").name == ( + "supabase" + ) + + +def test_soft_rows_never_win_detection(): + # aws is a soft fallback; detection yielding None routes the caller to the + # generic-s3/aws fallback logic instead. + assert detect_preset("https://s3.eu-central-1.amazonaws.com") is None + assert detect_preset("https://unknown.example.com") is None + + +def test_ambiguous_detection_raises(scratch_registry): + register( + Preset( + name="clone-a", addressing_style="path", host_patterns=("*.clone.test",) + ) + ) + register( + Preset( + name="clone-b", addressing_style="path", host_patterns=("*.clone.test",) + ) + ) + with pytest.raises(ConfigurationError, match="more than one preset"): + detect_preset("https://x.clone.test") + + +# --------------------------------------------------------------------------- # +# builtin rows sanity +# --------------------------------------------------------------------------- # + + +def test_every_builtin_row_is_well_formed(): + for name in available_presets(): + row = get_preset(name) + assert row.addressing_style in ("path", "virtual") + assert row.verified is False # doc-sourced until proven live + + +def test_the_load_bearing_rows(): + assert get_preset("backblaze").checksum == "when_required" # mandatory + assert get_preset("supabase").checksum == "when_required" # the §3 fix + assert get_preset("supabase").error_overrides # 400-means-404 + assert get_preset("hetzner").payload_signing_enabled is False + assert get_preset("hetzner").presign_addressing_style == "virtual" + assert get_preset("scaleway").capabilities.max_multipart_parts == 1000 + gcs = get_preset("gcs").capabilities + assert gcs.list_objects_v2 is False and gcs.batch_delete is False + assert get_preset("minio").requires_endpoint is True + assert get_preset("ceph").requires_endpoint is True + assert get_preset("r2").region_name == "auto" # R2's literal region string + for soft_name in ("aws", "generic-s3"): + assert get_preset(soft_name).soft is True + + +def test_capabilities_defaults_are_aws_reference_semantics(): + caps = Capabilities() + assert caps.list_objects_v2 and caps.batch_delete and caps.multipart + assert caps.max_multipart_parts == 10_000 + assert caps.consistency == "strong"