From ebde6a69f164195928bb244277d9d9959337c39a Mon Sep 17 00:00:00 2001 From: Thor Whalen <1906276+thorwhalen@users.noreply.github.com> Date: Tue, 11 Aug 2026 17:12:29 +0100 Subject: [PATCH 1/3] =?UTF-8?q?feat:=20Layer=20A=20=E2=80=94=20errors,=20p?= =?UTF-8?q?resets,=20connection,=20diagnose=20(ADRs=200003/0004/0012)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Additive, non-breaking 0.1.x change: four new modules, no v0 code touched. This is ADR-0007 §5 step 0 — s3dol.diagnose() ships BEFORE any resolution behaviour changes, so dependents can record what their environment resolves to (and see the v0-vs-v1 divergence table) ahead of the v1 release. - errors.py: the ADR-0004 taxonomy (KeyError family with unmangled __str__, auth errors never KeyError), pure (operation, code, status) classification incl. the body-less HEAD ambiguity moto cannot produce, per-provider override rows, tested redaction, the one translate_s3_errors seam, and the ADR-0012 ConfigurationError + S3DolWarning trees. - presets.py: Preset/Capabilities rows per ADR-0003 amended by ADR-0012 D4 (host_patterns/params/requires_endpoint/soft/config_kwargs, derived pinned gated on verified, mandatory addressing_style with 'auto' illegal, structural no-credential rule, endpoint hygiene). 15 doc-sourced rows. - connection.py: frozen picklable S3Connection (no live objects, per-instance build lock, mandatory __getstate__/__setstate__, redacting repr); pure resolve(spec, environ, aws_config) with per-field provenance, the two-rung endpoint ladder + C1/C2 guards, two-pass preset selection, omit-not-None Config semantics; credential-shape normalisation with per-shape rejections; explicit SigV4 always (the #10 root cause, fixed at this layer). - _diagnose.py: never-raising diagnose() with provenance table and the ADR-0012 D7 v0-vs-v1 differential (v0_mangle run through v1's own resolver twice). Module named _diagnose so s3dol.diagnose is unambiguously the function (a diagnose.py submodule would shadow it after first import). 134 new tier-1/2 tests + 12 doctests; full suite green. Refs #11, refs #10 (closed when the S3Store path switches to S3Connection in the v1 PR). Claude-Session: https://claude.ai/code/session_01KqivR5Be21Wh7yRdEG4fiY --- README.md | 25 + conftest.py | 11 +- s3dol/__init__.py | 38 ++ s3dol/_diagnose.py | 317 ++++++++++++ s3dol/connection.py | 872 +++++++++++++++++++++++++++++++++ s3dol/errors.py | 550 +++++++++++++++++++++ s3dol/presets.py | 439 +++++++++++++++++ s3dol/tests/test_connection.py | 519 ++++++++++++++++++++ s3dol/tests/test_diagnose.py | 144 ++++++ s3dol/tests/test_errors.py | 311 ++++++++++++ s3dol/tests/test_presets.py | 226 +++++++++ 11 files changed, 3451 insertions(+), 1 deletion(-) create mode 100644 s3dol/_diagnose.py create mode 100644 s3dol/connection.py create mode 100644 s3dol/errors.py create mode 100644 s3dol/presets.py create mode 100644 s3dol/tests/test_connection.py create mode 100644 s3dol/tests/test_diagnose.py create mode 100644 s3dol/tests/test_errors.py create mode 100644 s3dol/tests/test_presets.py diff --git a/README.md b/README.md index c0f4296..e58eb99 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,31 @@ To install: ```pip install s3dol``` [Documentation](https://i2mint.github.io/s3dol/) +## Heads-up: v1 is coming — run `s3dol.diagnose()` now + +A major, behaviour-clarifying v1 is in progress +(design: [`misc/docs/architecture.md`](misc/docs/architecture.md) and the ADRs in +[`misc/docs/decisions/`](misc/docs/decisions/)). Today's `S3Store` will keep +working through a compatibility shim, but a few v0 behaviours were bugs +(e.g. an explicit `endpoint_url=` being silently dropped whenever +`AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY` are exported) and v1 fixes them — +which can *move where your data goes*. + +Before upgrading to v1, run the diagnosis with the same arguments you pass +`S3Store` today, in the environment you deploy in: + +```python +import s3dol + +s3dol.diagnose(bucket_name='my-bucket', endpoint_url='...', path='...') +``` + +It prints what resolves (endpoint, region, signing, credential *source* — never +a secret), where each value came from, and a **v0-vs-v1 divergence table** +telling you whether the upgrade changes anything for your call. It never +raises; a failing resolution is part of the report. + + ## Set up credentials Recommended prerequisite to make getting started easier but not required. diff --git a/conftest.py b/conftest.py index 65254ed..1b661d5 100644 --- a/conftest.py +++ b/conftest.py @@ -30,7 +30,16 @@ #: module not listed here is deselected when no endpoint is reachable, so a new #: endpoint-using test can never silently fail the local gate. Widen only after #: verifying a module's tests are genuinely endpoint-free. -_S3_FREE_TEST_STEMS = frozenset({"test_url_for"}) +_S3_FREE_TEST_STEMS = frozenset( + { + "test_url_for", + # Layer A (v1) suites: tier-1 pure + moto-mocked tier-2 — no endpoint. + "test_errors", + "test_presets", + "test_connection", + "test_diagnose", + } +) def _probe_host_port(): diff --git a/s3dol/__init__.py b/s3dol/__init__.py index cce333d..edae93b 100644 --- a/s3dol/__init__.py +++ b/s3dol/__init__.py @@ -4,3 +4,41 @@ from s3dol.base import S3Dol from s3dol.store import S3Store + +# New (v1) layer-A names, loaded lazily (PEP 562) so `import s3dol` does not +# pay for what a script does not use. `s3dol.diagnose(...)` is the ADR-0007 §5 +# step-0 migration safety mechanism — run it with your current S3Store kwargs +# to see what resolves, from where, and whether the upcoming v1 moves it. +_LAZY_ATTRIBUTES = { + "diagnose": ("s3dol._diagnose", "diagnose"), + "S3Connection": "s3dol.connection", + "resolve": "s3dol.connection", + "Resolution": "s3dol.connection", + "Preset": "s3dol.presets", + "Capabilities": "s3dol.presets", + "register_preset": ("s3dol.presets", "register"), +} + + +def __getattr__(name): + try: + target = _LAZY_ATTRIBUTES[name] + except KeyError: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") from None + import importlib + + if isinstance(target, tuple): + module_path, attribute = target + else: + module_path, attribute = target, name + value = getattr(importlib.import_module(module_path), attribute) + # Cache the resolved attribute. This ALSO deliberately overrides the + # submodule binding that `import s3dol.diagnose` just created on this + # package — without it, `s3dol.diagnose` would resolve to the module on + # every access after the first, shadowing the function. + globals()[name] = value + return value + + +def __dir__(): + return sorted(set(globals()) | set(_LAZY_ATTRIBUTES)) diff --git a/s3dol/_diagnose.py b/s3dol/_diagnose.py new file mode 100644 index 0000000..32f335e --- /dev/null +++ b/s3dol/_diagnose.py @@ -0,0 +1,317 @@ +"""``s3dol.diagnose()`` — print what would resolve, from where, and whether v1 +moves it. The step-0 migration safety mechanism (ADR-0007 §5, ADR-0012 D6/D7). + +Run it with the same arguments you pass ``s3dol.store.S3Store`` today:: + + import s3dol + s3dol.diagnose(bucket_name='my-bucket', endpoint_url='...', path='logs') + +Non-negotiables (ADR-0012 D6): + +- it **never raises** — a failed resolution is a *row*; this function exists to + report exactly the misconfigurations that raise; +- it holds **no credential value**, only a provenance label; it never reads + ``.access_key`` (which can trigger a network refresh); credential identity is + discussed as a predicate, never a snapshot. + +The v0-vs-v1 divergence table is a **differential, not a frozen simulation** +(ADR-0012 D7): one ~20-line argument-mangler (:func:`_v0_mangle`, mapping 1:1 +onto v0's ``get_client``/``_find_default_credentials``) is run through v1's own +resolver twice, so resolver imprecision cancels. It dies in the same commit as +the compat shim. +""" + +from __future__ import annotations + +import io +import re +import sys +from typing import Mapping, Optional + +from s3dol.connection import S3Connection, Resolution, load_aws_config, resolve +from s3dol.errors import redact + +#: v0 ``S3Store`` parameter names (the shim signature, ADR-0007 §3 — including +#: ``is_supabase_endpoint``, which v0 accepts and an early shim draft forgot). +_V0_ARG_NAMES = frozenset( + { + "bucket_name", + "make_bucket", + "path", + "aws_access_key_id", + "aws_secret_access_key", + "aws_session_token", + "endpoint_url", + "region_name", + "profile_name", + "skip_bucket_check", + "is_supabase_endpoint", + } +) + +_SECRET_ARG_NAMES = frozenset({"aws_secret_access_key", "aws_session_token"}) + +#: Reasons a v0 branch is excluded from the differential (ADR-0012 D7): these +#: two branches build clients through the module-level ``boto3.client(...)``, +#: which resolves through the process-global cached DEFAULT_SESSION — they are +#: not functions of their arguments. +_EXCLUDED_BRANCHES = { + "supabase": ( + "v0's Supabase branch builds its client through boto3's process-global " + "session, so its behaviour is not a function of these arguments. Note: " + "a long-lived v0 process that builds one Supabase store pins its " + "credentials for the life of the interpreter." + ), + "skip-bucket-check": ( + "v0's skip_bucket_check branch builds its client through boto3's " + "process-global session, so its behaviour is not a function of these " + "arguments." + ), +} + + +def _v0_mangle(args: Mapping, environ: Mapping) -> tuple[str, Optional[dict]]: + """What v0's ``S3Store`` *effectively* does with these arguments. + + ~20 frozen lines mapping 1:1 onto v0 (``store.py`` + ``base.py:get_client`` + / ``_find_default_credentials``). Returns ``(branch, effective_args)``; + ``effective_args=None`` means the branch is excluded (see + :data:`_EXCLUDED_BRANCHES`). + """ + endpoint = args.get("endpoint_url") + is_supabase = args.get("is_supabase_endpoint") + if is_supabase is None: + is_supabase = bool(endpoint and ".supabase." in endpoint) + if is_supabase: + return "supabase", None + if args.get("skip_bucket_check"): + return "skip-bucket-check", None + effective = dict(args) + profile = args.get("profile_name") + env_creds = bool( + environ.get("AWS_ACCESS_KEY_ID") and environ.get("AWS_SECRET_ACCESS_KEY") + ) + if profile is None and env_creds: + # get_client(None) -> _find_default_credentials -> the env branch: + # env creds OVERRIDE explicit kwargs ({**kwargs, **env}), env's (maybe + # unset) token/region NULL explicit ones, and endpoint_url is DROPPED + # (never forwarded to get_client('environment variables', ...)). + effective.update( + aws_access_key_id=environ.get("AWS_ACCESS_KEY_ID"), + aws_secret_access_key=environ.get("AWS_SECRET_ACCESS_KEY"), + aws_session_token=environ.get("AWS_SESSION_TOKEN"), + region_name=environ.get("AWS_DEFAULT_REGION"), + endpoint_url=None, + ) + return "env-credentials (endpoint dropped)", effective + if profile == "environment variables": + effective.update( + aws_access_key_id=environ.get("AWS_ACCESS_KEY_ID"), + aws_secret_access_key=environ.get("AWS_SECRET_ACCESS_KEY"), + aws_session_token=environ.get("AWS_SESSION_TOKEN"), + region_name=environ.get("AWS_DEFAULT_REGION"), + ) + return "env-profile", effective + if profile is not None and any( + args.get(k) + for k in ("aws_access_key_id", "aws_secret_access_key", "aws_session_token") + ): + # boto3: profile + any explicit credential kwarg silently discards the + # profile's credentials (measured, ADR-0012 D5). + return "profile+explicit-creds (profile credentials discarded)", effective + return ("profile" if profile else "explicit"), effective + + +def _spec_from_v0_args(args: Mapping) -> S3Connection: + """The v1 spec equivalent of a v0 ``S3Store`` call (connection axes only).""" + access = args.get("aws_access_key_id") + secret = args.get("aws_secret_access_key") + token = args.get("aws_session_token") + credentials = None + if access and secret: + credentials = (access, secret, token) if token else (access, secret) + profile = args.get("profile_name") + if profile == "environment variables": + profile = None # env credentials are the top of botocore's own chain + return S3Connection( + endpoint_url=args.get("endpoint_url") or None, + region_name=args.get("region_name") or None, + profile=profile, + credentials=credentials, + ) + + +def _redact_url(url: Optional[str]) -> Optional[str]: + """Redact query secrets and any userinfo from a URL for display.""" + if url is None: + return None + return re.sub(r"://[^@/]+@", "://REDACTED@", redact(url)) + + +def _field_rows(resolution: Resolution): + for name in ( + "endpoint_url", + "region_name", + "signature_version", + "addressing_style", + "checksum", + "payload_signing_enabled", + "verify", + ): + sourced = getattr(resolution, name) + value = sourced.value + if name == "endpoint_url": + value = _redact_url(value) + handed = "passed" if sourced.passed else "omitted (channel open)" + yield name, value, sourced.source, handed + + +def _divergence_rows(args: Mapping, environ: Mapping, aws_config) -> list[str]: + branch, effective = _v0_mangle(args, environ) + if effective is None: + return [ + f"v0 branch: {branch} — EXCLUDED from the differential.", + _EXCLUDED_BRANCHES[branch], + ] + rows = [f"v0 branch: {branch}"] + v1_res = resolve(_spec_from_v0_args(args), environ, aws_config) + v0_res = resolve(_spec_from_v0_args(effective), environ, aws_config) + moved = False + for axis in ("endpoint_url", "region_name"): + v0_value = getattr(v0_res, axis).value + v1_value = getattr(v1_res, axis).value + if v0_value != v1_value: + moved = True + rows.append( + f" {axis}: v0 -> {_redact_url(v0_value)!r}, " + f"v1 -> {_redact_url(v1_value)!r} ** CHANGES ON UPGRADE **" + ) + if v0_res.credential_provenance != v1_res.credential_provenance: + moved = True + rows.append( + f" credentials: v0 -> {v0_res.credential_provenance}, " + f"v1 -> {v1_res.credential_provenance} ** CHANGES ON UPGRADE **" + ) + if not moved: + rows.append(" no divergence: v1 resolves this call exactly as v0 does.") + return rows + + +def diagnose( + connection: Optional[S3Connection] = None, + *, + environ: Optional[Mapping[str, str]] = None, + file=None, + **kwargs, +) -> str: + """Print (and return) the resolution report. Never raises. + + Call it either with a ready :class:`~s3dol.connection.S3Connection`, with + ``S3Connection`` kwargs, or — the migration case — with exactly the kwargs + you pass ``s3dol.store.S3Store`` today (``bucket_name=``, ``path=``, + ``profile_name=``, ...), which additionally prints the v0-vs-v1 + divergence table. + + >>> report = diagnose(endpoint_url='http://localhost:9000', + ... environ={}, file=io.StringIO()) + >>> 'endpoint_url' in report and 'resolution' in report.lower() + True + """ + out = io.StringIO() + + def emit(line=""): + print(line, file=out) + + try: + import importlib.metadata + + try: + own_version = importlib.metadata.version("s3dol") + except Exception: + own_version = "unknown" + try: + import botocore + + botocore_version = botocore.__version__ + except Exception: + botocore_version = "NOT IMPORTABLE" + emit("s3dol diagnose") + emit("=" * 60) + emit( + f"s3dol {own_version} | botocore {botocore_version} | " + f"python {sys.version.split()[0]}" + ) + + if environ is None: + import os + + environ = dict(os.environ) + aws_config: object + try: + aws_config = load_aws_config() + except Exception as error: + aws_config = {} + emit(f"note: aws config unreadable ({type(error).__name__}: {error})") + + v0_args = {k: v for k, v in kwargs.items() if k in _V0_ARG_NAMES} + v1_kwargs = {k: v for k, v in kwargs.items() if k not in _V0_ARG_NAMES} + v0_mode = bool(v0_args) and not v1_kwargs and connection is None + + if kwargs: + emit() + emit("arguments:") + for key in sorted(kwargs): + shown = "" if key in _SECRET_ARG_NAMES else repr(kwargs[key]) + emit(f" {key} = {shown}") + + emit() + emit("resolution:") + try: + if connection is None: + if v0_mode: + spec = _spec_from_v0_args(v0_args) + else: + spec = S3Connection(**{**v0_args, **v1_kwargs}) + else: + if kwargs: + emit("note: connection= given; extra kwargs ignored.") + spec = connection + resolution = resolve(spec, environ, aws_config) + except Exception as error: + emit(f" RESOLUTION FAILS: {type(error).__name__}: {error}") + emit() + emit("(That failure is the diagnosis: the same arguments would") + emit(" raise at first use. Fix the row above and re-run.)") + return _finish(out, file) + emit( + f" preset: {resolution.preset.name!r} ({resolution.preset_source})" + + ("" if resolution.preset.verified else " [doc-sourced row]") + ) + for name, value, source, handed in _field_rows(resolution): + emit(f" {name:<24} = {value!r:<28} [{source}; {handed}]") + emit(f" credentials: {resolution.credential_provenance}") + emit(f" can_presign: {not spec.anon}") + emit() + emit("environment consulted (non-secret keys only):") + for key, value in resolution.environ_consulted: + emit(f" {key} = {_redact_url(value)!r}") + for warning_class, message in resolution.notes: + emit(f"warning [{warning_class.__name__}]: {message}") + + if v0_mode: + emit() + emit("v0 (S3Store) vs v1 divergence:") + for row in _divergence_rows(v0_args, environ, aws_config): + emit(f" {row}") + except Exception as error: # never raises — D6 + emit( + f"diagnose itself hit an unexpected error: {type(error).__name__}: {error}" + ) + emit("Please report this at https://github.com/i2mint/s3dol/issues") + return _finish(out, file) + + +def _finish(out: io.StringIO, file) -> str: + text = out.getvalue() + print(text, file=file if file is not None else sys.stdout, end="") + return text diff --git a/s3dol/connection.py b/s3dol/connection.py new file mode 100644 index 0000000..799ff35 --- /dev/null +++ b/s3dol/connection.py @@ -0,0 +1,872 @@ +"""``S3Connection`` — the credential + endpoint SSOT, and the pure resolver. + +Design record: `misc/docs/decisions/0012-credential-and-endpoint-resolution.md` +(all of it) on top of `0002-boto3-as-engine.md` (lazy boto3) and +`0003-provider-presets-and-capabilities.md` (presets, SigV4 mandate). + +The two sentences the module follows from (ADR-0012): + +1. **A preset is the caller's explicit endpoint, spelled by name.** It sits + exactly where ``endpoint_url=`` sits — above every environment and + config-file channel. s3dol owns two rungs of the endpoint ladder, not six. +2. **A ``Config`` value s3dol does not set is a channel it leaves open to the + user.** Every field is a ladder with a documented bottom, and "the row makes + no claim" is spelled by *omitting the key*, never by passing ``None``. + +Structure: + +- :class:`S3Connection` — a frozen, picklable spec. No live objects, no + escape hatch for one. Construction performs no I/O and imports no boto3. +- :func:`resolve` — a **pure** function ``(spec, environ, aws_config) -> + Resolution`` where every field carries its provenance. Tier-1 tests cover + the whole ladder with a fake environ dict and zero I/O. +- :func:`load_aws_config` — the separate, *named impure* loader for the + aws-config rungs; pass it (or its result) to :func:`resolve`. +- Credential shapes (ADR-0012 D5) — normalised in ``__post_init__`` to small + frozen, picklable providers; live botocore objects are rejected with + shape-specific messages. + +>>> spec = S3Connection(endpoint_url='http://localhost:9000', +... credentials=('AKIAEXAMPLE', 'sEkR3tVALUE')) +>>> res = resolve(spec, {}) +>>> res.endpoint_url.value, res.endpoint_url.source +('http://localhost:9000', 'endpoint_url=') +>>> res.signature_version.value # never left to botocore (ADR-0003 §4) +'s3v4' +>>> 'sEkR3tVALUE' in repr(spec) # secrets never in repr (ADR-0012 D1) +False +""" + +from __future__ import annotations + +import pickle +import re +import threading +from dataclasses import dataclass, fields +from typing import ( + Any, + Callable, + Literal, + Mapping, + NamedTuple, + Optional, + Union, +) + +from s3dol.errors import ( + AmbiguousResolution, + ConfigurationError, + MissingEndpoint, + PresetConflict, + PresetHostMismatch, +) +from s3dol.presets import ( + Capabilities, + Preset, + detect_preset, + get_preset, + validate_endpoint_hygiene, +) + +# --------------------------------------------------------------------------- # +# Credential shapes (ADR-0012 D5) +# --------------------------------------------------------------------------- # + + +class CredentialProvider: + """Base of s3dol's picklable credential specs. + + Subclasses are small frozen dataclasses that *describe* how to obtain + credentials; nothing live is held (nothing live pickles — measured, ADR-0012 + D5). At client-build time each is adapted into a botocore provider and + **replaces** botocore's resolver via + ``session.register_component('credential_provider', ...)`` — never + prepended (silent fallthrough) and never ``insert_before`` (raises when a + profile is set). + """ + + def provenance(self) -> str: + """A label for diagnose() — never a credential value.""" + raise NotImplementedError + + def _load_botocore_credentials(self): + raise NotImplementedError + + def botocore_provider(self): + from botocore.credentials import CredentialProvider as _BotocoreBase + + spec = self + + class _Adapter(_BotocoreBase): + METHOD = f"s3dol-{type(spec).__name__}" + CANONICAL_NAME = METHOD + + def load(self): + return spec._load_botocore_credentials() + + return _Adapter() + + +@dataclass(frozen=True) +class StaticCredentials(CredentialProvider): + """An explicit key pair (optionally a session token). + + Note: a *static* token expires and cannot refresh — prefer ``profile=`` or + a callable provider for STS/SSO. Pickling this object carries the secret + (that is what "explicit credentials" means); its repr does not. + """ + + access_key: str + secret_key: str + token: Optional[str] = None + + def provenance(self) -> str: + return f"explicit static (access key {self.access_key[:4]}…)" + + def __repr__(self): # secrets never in repr (ADR-0012 D1 §4) + token = ", token=***" if self.token else "" + return ( + f"{type(self).__name__}(access_key={self.access_key[:4]!r}…, " + f"secret_key=***{token})" + ) + + def _load_botocore_credentials(self): + from botocore.credentials import Credentials + + return Credentials( + self.access_key, self.secret_key, self.token, method="s3dol-static" + ) + + +@dataclass(frozen=True) +class ProfileCredentials(CredentialProvider): + """Credentials from a named profile — *credentials only*; the profile's + other config is not selected by this (that is what ``profile=`` on the + connection is for). Rebuilds live, still-refreshable credentials after + unpickling (a frozen dataclass holding only a profile name round-trips).""" + + profile: str + + def provenance(self) -> str: + return f"profile:{self.profile} (via credentials=)" + + def _load_botocore_credentials(self): + import botocore.session + + return botocore.session.Session(profile=self.profile).get_credentials() + + +@dataclass(frozen=True) +class CallableCredentials(CredentialProvider): + """A picklable zero-arg callable returning a mapping with keys + ``access_key``/``secret_key`` (aliases: ``key``/``secret``), optional + ``token``, optional ``expires_at``/``expiry_time`` (ISO 8601). With an + expiry, credentials are refreshable — the callable is re-invoked by + botocore's refresh machinery (advisory 900 s / mandatory 600 s before + expiry).""" + + fetch: Callable[[], Mapping] + + def provenance(self) -> str: + name = getattr(self.fetch, "__qualname__", None) or repr(type(self.fetch)) + return f"callable:{name}" + + @staticmethod + def _normalize(result: Mapping) -> dict: + def pick(*names): + for name in names: + if name in result: + return result[name] + return None + + creds = dict( + access_key=pick("access_key", "key", "aws_access_key_id"), + secret_key=pick("secret_key", "secret", "aws_secret_access_key"), + token=pick("token", "aws_session_token"), + expiry_time=pick("expires_at", "expiry_time"), + ) + if not creds["access_key"] or not creds["secret_key"]: + raise ConfigurationError( + "The credentials callable must return a mapping with " + "'access_key' and 'secret_key' (aliases: 'key'/'secret'); got " + f"keys {sorted(result)}." + ) + return creds + + def _load_botocore_credentials(self): + from botocore.credentials import Credentials, RefreshableCredentials + + first = self._normalize(self.fetch()) + if first["expiry_time"] is None: + return Credentials( + first["access_key"], + first["secret_key"], + first["token"], + method="s3dol-callable", + ) + + def refresh(): + fresh = self._normalize(self.fetch()) + return { + "access_key": fresh["access_key"], + "secret_key": fresh["secret_key"], + "token": fresh["token"], + "expiry_time": fresh["expiry_time"], + } + + return RefreshableCredentials.create_from_metadata( + metadata=refresh(), refresh_using=refresh, method="s3dol-callable" + ) + + +CredentialsInput = Union[ + None, str, tuple, list, Mapping, CredentialProvider, Callable[[], Mapping] +] + +_AWS_CRED_KEYS = ("aws_access_key_id", "aws_secret_access_key", "aws_session_token") + + +def _is_live_botocore_object(value: Any) -> bool: + """Detect live boto3/botocore objects *without importing them* — + construction must not import boto3 (ADR-0002).""" + module = type(value).__module__ or "" + return module.startswith(("botocore", "boto3")) + + +def normalize_credentials(value: CredentialsInput) -> Optional[CredentialProvider]: + """Normalise every accepted credential shape; reject the rest, each with + its own message (ADR-0012 D5). The only door is ``S3Connection.__post_init__``. + + >>> normalize_credentials(None) is None + True + >>> normalize_credentials('prod') + ProfileCredentials(profile='prod') + >>> normalize_credentials(('AKIAXX', 'sec')) + StaticCredentials(access_key='AKIA'…, secret_key=***) + """ + if value is None: + return None + if isinstance(value, CredentialProvider): + return value + if isinstance(value, type) and issubclass(value, CredentialProvider): + raise ConfigurationError( + f"credentials= got the class {value.__name__}; pass an *instance* " + f"(e.g. {value.__name__}(...))." + ) + if _is_live_botocore_object(value): + raise ConfigurationError( + f"credentials= got a live boto3/botocore object " + f"({type(value).__name__}). A resolved credential snapshot cannot " + f"refresh and cannot pickle, and sessions/clients hold sockets. " + f"Pass a profile name, a (key, secret[, token]) tuple, a mapping of " + f"aws_* keys, or a picklable zero-arg callable." + ) + if isinstance(value, str): + return ProfileCredentials(value) # a bare str is ALWAYS a profile name + if isinstance(value, (tuple, list)): + if len(value) not in (2, 3): + raise ConfigurationError( + f"credentials= tuple must be (key, secret) or " + f"(key, secret, token); got {len(value)} items." + ) + return StaticCredentials(*map(str, value)) + if isinstance(value, Mapping): + unknown = set(value) - set(_AWS_CRED_KEYS) + if unknown: + raise ConfigurationError( + f"credentials= mapping has unknown key(s) {sorted(unknown)}; " + f"allowed: {list(_AWS_CRED_KEYS)}." + ) + if not value.get("aws_access_key_id") or not value.get("aws_secret_access_key"): + raise ConfigurationError( + "credentials= mapping needs both 'aws_access_key_id' and " + "'aws_secret_access_key' (a token alone leaves " + "Credentials(None, None, TOKEN) — a measured boto3 hazard)." + ) + return StaticCredentials( + access_key=value["aws_access_key_id"], + secret_key=value["aws_secret_access_key"], + token=value.get("aws_session_token"), + ) + if callable(value): + try: + pickle.dumps(value) + except Exception as error: + raise ConfigurationError( + f"credentials= callable {value!r} is not picklable " + f"({type(error).__name__}). Every s3dol store pickles " + f"(ADR-0012 D1); use a module-level function or a " + f"functools.partial of one." + ) from error + return CallableCredentials(value) + raise ConfigurationError( + f"credentials= got a {type(value).__name__}; accepted: None, a profile " + f"name (str), (key, secret[, token]), a mapping of aws_* keys, a " + f"CredentialProvider instance, or a picklable zero-arg callable." + ) + + +# --------------------------------------------------------------------------- # +# Resolution — pure, provenance-carrying (ADR-0012 D2/D3/D4) +# --------------------------------------------------------------------------- # + + +class Sourced(NamedTuple): + """A resolved value, where it came from, and whether s3dol passes it to + botocore explicitly (``passed=False`` == the key is omitted so the channel + below stays open — "omit, never None").""" + + value: Any + source: str + passed: bool = False + + +@dataclass(frozen=True) +class Resolution: + """The result of :func:`resolve` — everything ``diagnose()`` prints and + everything client-build consumes. Carries provenance labels, never a + credential value.""" + + endpoint_url: Sourced + region_name: Sourced + signature_version: Sourced + addressing_style: Sourced + checksum: Sourced + payload_signing_enabled: Sourced + verify: Sourced + preset: Optional[Preset] + preset_source: str # 'named' | 'detected' | 'fallback:' | 'none' + credential_provenance: str + capabilities: Capabilities + #: The non-secret environment keys consulted, with their values. + environ_consulted: tuple[tuple[str, Optional[str]], ...] + #: Collected (warning class, message) pairs. resolve() never emits them — + #: the impure caller decides (client build warns; diagnose prints rows). + notes: tuple[tuple[type, str], ...] + + +_ENV_ENDPOINT_KEYS = ("AWS_ENDPOINT_URL_S3", "AWS_ENDPOINT_URL") +_ENV_CONSULTED_KEYS = _ENV_ENDPOINT_KEYS + ( + "AWS_DEFAULT_REGION", # botocore does NOT read AWS_REGION; neither do we + "AWS_PROFILE", + "AWS_REQUEST_CHECKSUM_CALCULATION", + "AWS_RESPONSE_CHECKSUM_VALIDATION", +) + +#: Region-looking tokens: ``us-east-1``, ``fr-par``, ``us-west-004``, hetzner's +#: ``fsn1``, and R2's literal ``auto``. +_REGION_TOKEN = re.compile(r"[a-z]{2}-[a-z]+-\d+|[a-z]{2}-[a-z]{3,}|[a-z]{3}\d|auto") + + +def _region_from_host(endpoint_url: Optional[str]) -> Optional[str]: + """Best-effort region capture from a resolved endpoint host.""" + if not endpoint_url: + return None + from urllib.parse import urlsplit + + host = urlsplit(endpoint_url).hostname or "" + match = re.search(r"(?:^|\.)s3[.-]([a-z0-9-]+)\.", host) + candidates = [match.group(1)] if match else [] + labels = host.split(".") + if labels: + candidates.append(labels[0]) + for candidate in candidates: + if _REGION_TOKEN.fullmatch(candidate): + return candidate + return None + + +_CREDENTIAL_KEY_MARKERS = ( + "access_key", + "secret", + "token", + "password", + "credential_process", + "sso_", +) + + +def _scrub_credentials(config: Mapping) -> dict: + """Recursively drop credential-shaped keys from an aws-config mapping.""" + + def scrub(node): + if isinstance(node, Mapping): + return { + k: scrub(v) + for k, v in node.items() + if not any( + marker in str(k).lower() for marker in _CREDENTIAL_KEY_MARKERS + ) + } + return node + + return scrub(config) + + +def load_aws_config() -> Mapping: + """The named, *impure* aws-config loader (ADR-0012 D2): loads the shared + config via botocore (~16 ms — never on the common path; pass this function + as a thunk and :func:`resolve` calls it only if a rung needs it), scrubbed + of anything credential-shaped.""" + import botocore.session + + return _scrub_credentials(botocore.session.Session().full_config) + + +def _profile_section( + aws_config: Mapping, environ: Mapping, profile: Optional[str] +) -> Mapping: + profiles = aws_config.get("profiles", {}) if aws_config else {} + name = profile or environ.get("AWS_PROFILE") or "default" + return profiles.get(name, {}) + + +def _lower_rung_endpoint( + environ: Mapping, aws_config: Mapping, profile: Optional[str] +) -> Optional[tuple[str, str]]: + """The endpoint the rungs *below* s3dol's two would supply — read purely, + for diagnose() and the C1/C2 guards **only**, never to override (ADR-0012 + D3; and never pass ``Config(ignore_configured_endpoint_urls=True)``).""" + for env_key in _ENV_ENDPOINT_KEYS: + value = environ.get(env_key) + if value: + return value, f"env:{env_key}" + if aws_config: + try: + from botocore.configprovider import ConfiguredEndpointProvider + + provider = ConfiguredEndpointProvider( + full_config=aws_config, + scoped_config=_profile_section(aws_config, environ, profile), + client_name="s3", + environ={}, # env rungs already handled above + ) + value = provider.provide() + if value: + return str(value), "aws-config" + except Exception: # config parsing must never fail resolution + return None + return None + + +def _row_region(row: Preset) -> Optional[str]: + """The row's region literal with params bound; ``None`` if it makes no + claim (absent, or a template with unbound placeholders).""" + if not row.region_name: + return None + value, missing = row._bound(row.region_name) + return None if missing else value + + +def resolve( + spec: "S3Connection", + environ: Mapping[str, str], + aws_config: Union[Mapping, Callable[[], Mapping], None] = None, +) -> Resolution: + """Pure resolution: no ``os.environ``, no disk, no network (ADR-0012 D2). + + ``environ`` is a plain mapping (pass ``os.environ`` at real call sites, + a dict in tests). ``aws_config`` is a pre-loaded, credential-scrubbed + config mapping, or a thunk for one (:func:`load_aws_config`) — called only + if a rung needs it. + """ + notes: list[tuple[type, str]] = [] + aws_config_cache: dict = {} + + def config() -> Mapping: + if "value" not in aws_config_cache: + if aws_config is None: + aws_config_cache["value"] = {} + elif callable(aws_config): + aws_config_cache["value"] = aws_config() or {} + else: + aws_config_cache["value"] = aws_config + return aws_config_cache["value"] + + # -- the named row (pass 1 input) + if spec.preset is None: + named = None + elif isinstance(spec.preset, Preset): + named = spec.preset + else: + named = get_preset(spec.preset) + + # -- endpoint ladder (D3): s3dol owns two rungs, then stops + endpoint: Optional[Sourced] = None + if spec.endpoint_url: + endpoint = Sourced(spec.endpoint_url, "endpoint_url=", passed=True) + elif named is not None: + bound = named.bound_endpoint(region_name=spec.region_name) + if bound is not None: + validate_endpoint_hygiene(bound, what=f"preset:{named.name} endpoint") + endpoint = Sourced(bound, f"preset:{named.name}", passed=True) + + lower = None + if endpoint is None: + lower = _lower_rung_endpoint(environ, config(), spec.profile) + if lower is not None: + endpoint = Sourced(lower[0], lower[1], passed=False) + elif named is not None: + missing = named.missing_endpoint_params(region_name=spec.region_name) + if named.requires_endpoint or missing: + raise MissingEndpoint( + f"Preset {named.name!r} needs an endpoint and nothing " + f"supplied one" + + ( + f" (unbound template parameter(s): {sorted(missing)} — " + f"bind with get_preset({named.name!r}).bind(...))" + if missing + else "" + ) + + ". Pass endpoint_url=, or set AWS_ENDPOINT_URL_S3. " + f"Falling back to AWS silently is structurally impossible " + f"(ADR-0012 C1)." + ) + endpoint = Sourced(None, "SDK resolver", passed=False) + else: + endpoint = Sourced(None, "SDK resolver", passed=False) + + # -- C2 guard: a named, patterned row whose endpoint arrived from below + if ( + named is not None + and not named.soft + and named.host_patterns + and not endpoint.passed + and endpoint.value + ): + if not named.matches_host(endpoint.value): + other = detect_preset(endpoint.value) + if other is not None and other.name != named.name: + raise PresetHostMismatch( + f"preset={named.name!r} was named, but the endpoint that " + f"resolved from the environment ({endpoint.source}) matches " + f"{other.name!r}'s host patterns: {endpoint.value!r}. " + f"Pass endpoint_url= explicitly (rung 1 never triggers " + f"this guard), or name the right preset." + ) + notes.append( + ( + AmbiguousResolution, + f"preset={named.name!r} was named, but the endpoint that " + f"resolved from the environment ({endpoint.source}: " + f"{endpoint.value!r}) matches none of its host patterns " + f"{named.host_patterns}. Proceeding; pass endpoint_url= to " + f"silence this.", + ) + ) + + # -- pass 2: pick the settings row (named > detected > fallback) + if named is not None: + row, preset_source = named, "named" + elif endpoint.value: + detected = detect_preset(endpoint.value) + if detected is not None: + row, preset_source = detected, "detected" + else: + row, preset_source = get_preset("generic-s3"), "fallback:generic-s3" + else: + row, preset_source = get_preset("aws"), "fallback:aws" + + # -- region ladder + row_region = _row_region(row) + if spec.region_name: + if row.pinned and row_region and spec.region_name != row_region: + raise PresetConflict( + f"region_name={spec.region_name!r} contradicts the verified " + f"preset {row.name!r}, which pins region {row_region!r}." + ) + region = Sourced(spec.region_name, "region_name=", passed=True) + elif row_region: + region = Sourced(row_region, f"preset:{row.name}", passed=True) + else: + captured = _region_from_host(endpoint.value) + env_region = environ.get("AWS_DEFAULT_REGION") + if captured: + region = Sourced(captured, "endpoint host (best-effort)", passed=True) + elif env_region: + region = Sourced(env_region, "env:AWS_DEFAULT_REGION", passed=True) + elif _profile_section(config(), environ, spec.profile).get("region"): + region = Sourced( + _profile_section(config(), environ, spec.profile)["region"], + "aws-config profile", + passed=True, + ) + else: + region = Sourced("us-east-1", "default", passed=True) + + # -- addressing style ladder: kwarg > row > profile > omit. + # Soft rows (aws, generic-s3) document the expected behaviour but do NOT + # pass it — omitting the key preserves the profile channel and botocore's + # endpoint-dependent default (ADR-0012 D3: omit, never None). + profile_addressing = ( + _profile_section(config(), environ, spec.profile).get("s3", {}) or {} + ).get("addressing_style") + if spec.addressing_style: + addressing = Sourced(spec.addressing_style, "addressing_style=", passed=True) + elif not row.soft: + addressing = Sourced(row.addressing_style, f"preset:{row.name}", passed=True) + elif profile_addressing: + addressing = Sourced(profile_addressing, "aws-config profile", passed=False) + else: + addressing = Sourced( + row.addressing_style, + f"botocore default (≈{row.addressing_style})", + passed=False, + ) + + # -- checksum ladder: kwarg > row > env > botocore default. The env and + # default rungs OMIT the keys, so a user's AWS_REQUEST_CHECKSUM_* env + # workaround keeps working (measured: Config beats env both ways). + env_checksum = environ.get("AWS_REQUEST_CHECKSUM_CALCULATION") or environ.get( + "AWS_RESPONSE_CHECKSUM_VALIDATION" + ) + if spec.checksum: + checksum = Sourced(spec.checksum, "checksum=", passed=True) + elif row.checksum: + checksum = Sourced(row.checksum, f"preset:{row.name}", passed=True) + elif env_checksum: + checksum = Sourced(env_checksum, "env:AWS_*_CHECKSUM_*", passed=False) + else: + checksum = Sourced("when_supported", "botocore default", passed=False) + + # -- payload signing: kwarg > row > omit (no env channel exists) + if spec.payload_signing_enabled is not None: + payload_signing = Sourced( + spec.payload_signing_enabled, "payload_signing_enabled=", passed=True + ) + elif row.payload_signing_enabled is not None: + payload_signing = Sourced( + row.payload_signing_enabled, f"preset:{row.name}", passed=True + ) + else: + payload_signing = Sourced(None, "omitted", passed=False) + + # -- signature version: kwarg > 's3v4'; never from a row, never from env, + # never left to botocore (which silently downgrades to SigV2 in the 12 + # regions endpoints.json lists as v2-capable — ADR-0003 §4 / ADR-0012). + signature = Sourced( + spec.signature_version, + "signature_version=" if spec.signature_version != "s3v4" else "s3dol default", + passed=True, + ) + + verify = Sourced( + spec.verify, + "verify=" if spec.verify is not None else "omitted", + passed=spec.verify is not None, + ) + + # -- credential provenance label (never a value) + if spec.anon: + provenance = "anonymous (UNSIGNED)" + elif spec.credentials is not None: + provenance = spec.credentials.provenance() + elif spec.profile: + provenance = f"profile:{spec.profile}" + elif environ.get("AWS_ACCESS_KEY_ID"): + provenance = "chain (env credentials present)" + else: + provenance = "chain (config file / SSO / IMDS / none)" + + return Resolution( + endpoint_url=endpoint, + region_name=region, + signature_version=signature, + addressing_style=addressing, + checksum=checksum, + payload_signing_enabled=payload_signing, + verify=verify, + preset=row, + preset_source=preset_source, + credential_provenance=provenance, + capabilities=row.capabilities, + environ_consulted=tuple((key, environ.get(key)) for key in _ENV_CONSULTED_KEYS), + notes=tuple(notes), + ) + + +# --------------------------------------------------------------------------- # +# S3Connection (ADR-0012 D1) +# --------------------------------------------------------------------------- # + + +@dataclass(frozen=True) +class S3Connection: + """A frozen, picklable connection spec. No live objects — ever. + + Construction performs no I/O and does not import boto3 (ADR-0002); the + client is built lazily, once, under a per-instance lock (CPython ≥3.12 + removed ``cached_property``'s lock deliberately — measured: 8 racing + threads → 8 clients without one). + + ``__post_init__`` is the only door: every normalisation and contradiction + check happens here, so a bad shape dies at construction with a message, + not later inside botocore. + + >>> S3Connection(anon=True, profile='prod') + Traceback (most recent call last): + ... + s3dol.errors.ConfigurationError: ... + """ + + # WHERE + preset: Union[str, Preset, None] = None + endpoint_url: Optional[str] = None + region_name: Optional[str] = None + # WHO + profile: Optional[str] = None + credentials: Optional[CredentialProvider] = None # normalised in __post_init__ + anon: bool = False + # HOW (None == "do not override" — the ladder continues; see resolve()) + signature_version: str = "s3v4" # never left to botocore (ADR-0003 §4) + addressing_style: Optional[Literal["path", "virtual"]] = None + checksum: Optional[Literal["when_supported", "when_required"]] = None + payload_signing_enabled: Optional[bool] = None + verify: Union[bool, str, None] = None # self-signed MinIO / corporate CA + client_kwargs: tuple = () + # POLICY + deny_means_absent: bool = False # ADR-0004 §3 + + def __post_init__(self): + object.__setattr__(self, "credentials", normalize_credentials(self.credentials)) + if isinstance(self.preset, str): + get_preset(self.preset) # unknown names die here, loudly + if self.endpoint_url is not None: + validate_endpoint_hygiene(self.endpoint_url) + if self.addressing_style not in (None, "path", "virtual"): + raise ConfigurationError( + f"addressing_style must be 'path', 'virtual', or None (ladder " + f"continues); got {self.addressing_style!r}. 'auto' is illegal." + ) + if self.checksum not in (None, "when_supported", "when_required"): + raise ConfigurationError( + f"checksum must be 'when_supported', 'when_required', or None; " + f"got {self.checksum!r}." + ) + if isinstance(self.client_kwargs, Mapping): + object.__setattr__( + self, "client_kwargs", tuple(sorted(self.client_kwargs.items())) + ) + if self.anon: + if self.credentials is not None or self.profile: + raise ConfigurationError( + "anon=True with credentials/profile: UNSIGNED would " + "silently discard them (measured — the request signer's " + "credentials become None). Drop anon, or drop the " + "credentials." + ) + if self.deny_means_absent: + raise ConfigurationError( + "anon=True with deny_means_absent=True: an anonymous HEAD " + "gives 403 on existing keys and 404 on missing ones, so " + "every key would read as absent — a silently empty store " + "(measured under moto; banned by ADR-0001 goal 4)." + ) + object.__setattr__(self, "_lock", threading.Lock()) + + # -- the client -------------------------------------------------------- # + + @property + def client(self): + """The boto3 S3 client — built lazily, once, under the lock.""" + built = self.__dict__.get("_client") + if built is None: + with self._lock: + built = self.__dict__.get("_client") + if built is None: + built = self._build_client() + object.__setattr__(self, "_client", built) + return built + + def resolution( + self, + environ: Optional[Mapping[str, str]] = None, + aws_config: Union[Mapping, Callable, None] = load_aws_config, + ) -> Resolution: + """Resolve against the real environment by default (a thin impure + wrapper over the pure :func:`resolve`).""" + import os + + return resolve(self, environ if environ is not None else os.environ, aws_config) + + def _build_client(self): + import warnings + + import boto3 + import botocore.session + from botocore import UNSIGNED + from botocore.config import Config + from botocore.credentials import CredentialResolver + + resolution = self.resolution() + for warning_class, message in resolution.notes: + warnings.warn(message, warning_class, stacklevel=4) + + s3_settings: dict = {} + if resolution.addressing_style.passed: + s3_settings["addressing_style"] = resolution.addressing_style.value + if resolution.payload_signing_enabled.passed: + s3_settings["payload_signing_enabled"] = ( + resolution.payload_signing_enabled.value + ) + config_kwargs: dict = dict(resolution.preset.config_kwargs or ()) + if resolution.checksum.passed: + config_kwargs["request_checksum_calculation"] = resolution.checksum.value + config_kwargs["response_checksum_validation"] = resolution.checksum.value + if s3_settings: + config_kwargs["s3"] = s3_settings + config = Config( + # UNSIGNED never enters the dataclass (unpicklable) — build-time only + signature_version=( + UNSIGNED if self.anon else resolution.signature_version.value + ), + **config_kwargs, + ) + + botocore_session = ( + botocore.session.Session(profile=self.profile) + if self.profile + else botocore.session.Session() + ) + if self.credentials is not None: + botocore_session.register_component( + "credential_provider", + CredentialResolver(providers=[self.credentials.botocore_provider()]), + ) + + client_kwargs = dict(self.client_kwargs) + if self.verify is not None: + client_kwargs["verify"] = self.verify + return boto3.Session(botocore_session=botocore_session).client( + "s3", + endpoint_url=( + resolution.endpoint_url.value + if resolution.endpoint_url.passed + else None + ), + region_name=resolution.region_name.value, + config=config, + **client_kwargs, + ) + + # -- pickling: mandatory for every connection (ADR-0012 D1 §3) --------- # + + def __getstate__(self): + # Enumerating dataclass fields drops every cache by construction + # (the boto3 client is unpicklable, full stop). + return {f.name: getattr(self, f.name) for f in fields(self)} + + def __setstate__(self, state): + for name, value in state.items(): + object.__setattr__(self, name, value) + object.__setattr__(self, "_lock", threading.Lock()) + + # -- repr: non-default fields only; secrets never (D1 §4) -------------- # + + def __repr__(self): + shown = [] + for f in fields(self): + value = getattr(self, f.name) + if value != f.default: + shown.append(f"{f.name}={value!r}") + return f"{type(self).__name__}({', '.join(shown)})" diff --git a/s3dol/errors.py b/s3dol/errors.py new file mode 100644 index 0000000..fcf1ccc --- /dev/null +++ b/s3dol/errors.py @@ -0,0 +1,550 @@ +"""The s3dol error taxonomy and the single error-translation seam. + +Design record: `misc/docs/decisions/0004-error-taxonomy.md` (the taxonomy, the +``(operation, code, status)`` classification, the 403 policy, ``ObjectArchived``) +and `misc/docs/decisions/0012-credential-and-endpoint-resolution.md` (the +``ConfigurationError`` tree and the ``S3DolWarning`` tree). + +Three rules govern this module: + +1. **One seam.** :func:`translate_s3_errors` is the only place in the package + that catches botocore exceptions. Everything else propagates. +2. **Auth, config and transport errors are never translated to ``KeyError``.** + A missing permission must not look like a missing key. +3. **Classification keys on ``(operation, code, status)``, never on exception + type.** HEAD responses have no body, so botocore's modelled exceptions + (``client.exceptions.NoSuchKey`` …) never fire for them. + +Everything classification-related here is pure: :func:`classify_client_error` +takes plain values (or a synthesized ``ClientError``) and touches no network, +so the bulk of error tests are tier-1 unit tests — including body-less HEAD +errors, which moto cannot produce. + +>>> classify_client_error('GetObject', 'NoSuchKey', 404).kind +'object_absent' +>>> classify_client_error('HeadObject', '404', 404).kind # HEAD is ambiguous +'ambiguous_head' +>>> classify_client_error('GetObject', 'AccessDenied', 403).kind +'access_denied' +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from functools import wraps +from typing import Any, Callable, Iterable, Mapping, Optional + +# --------------------------------------------------------------------------- # +# The exception tree +# --------------------------------------------------------------------------- # + + +class S3Error(Exception): + """Base of every s3dol exception — ``except S3Error`` catches the package.""" + + +class _InformativeKeyError(KeyError): + """A ``KeyError`` whose message is not repr-mangled. + + ``KeyError.__str__`` uses ``repr`` of its args, so a multi-line, helpful + message becomes one line full of ``\\n`` escapes. Every s3dol ``KeyError`` + subclass therefore overrides ``__str__`` (ADR-0004 §5). + + >>> raw = KeyError('line one\\nline two') + >>> '\\\\n' in str(raw) # stdlib mangles + True + >>> ours = _InformativeKeyError('line one\\nline two') + >>> '\\\\n' in str(ours) + False + """ + + def __str__(self): + return str(self.args[0]) if self.args else "" + + +class ObjectNotFound(S3Error, _InformativeKeyError): + """The object is absent. The Mapping contract's ``KeyError``.""" + + +class BucketNotFound(S3Error, _InformativeKeyError): + """The bucket is absent. + + Still a ``KeyError``: for an object operation it is a key-space problem for + the caller; for a bucket operation it is the key of the endpoint mapping. + """ + + +class KeyNotValid(S3Error, _InformativeKeyError, ValueError): + """The key is syntactically invalid (checked before the wire). + + Inherits from both ``KeyError`` and ``ValueError`` deliberately, so either + ``except`` works. Named ``KeyNotValid`` (not ``KeyValidationError``) because + dol already ships two *different* classes called ``KeyValidationError`` + (ADR-0004 §5). + """ + + +class ObjectArchived(S3Error, _InformativeKeyError): + """The object exists but is archived (e.g. Glacier) and cannot be read now. + + A ``KeyError``, deliberately and arguably (ADR-0004 §4): ``k in store`` + stays ``True``, but ``store[k]`` fails as a ``KeyError`` so that + ``store.get(k, default)`` degrades to the not-available branch. Note that + only the *defaulted accessors* degrade — ``dict(store)``, ``.items()`` and + ``Mapping.__eq__`` still raise. ``setdefault``/``pop`` are overridden in the + store to re-raise this instead of silently overwriting the archived object. + """ + + def __init__( + self, + message: str, + *, + storage_class: Optional[str] = None, + restore_status: Optional[str] = None, + restore: Optional[Callable] = None, + ): + super().__init__(message) + self.storage_class = storage_class + self.restore_status = restore_status + self._restore = restore + + def restore(self, days: int = 1, *, tier: str = "Standard"): + """Request restoration of the archived object (if a handle was bound).""" + if self._restore is None: + raise NotSupported( + "No restore capability was bound to this error. " + "Use s3dol.handles(store)[key].restore(...) instead." + ) + return self._restore(days=days, tier=tier) + + +class AccessDenied(S3Error): + """Permission denied. **Not** a ``KeyError`` — a missing permission must + not look like a missing key (ADR-0004 §1/§3).""" + + +class CredentialsError(S3Error): + """Credentials are missing, expired, or invalid.""" + + +class NotSupported(S3Error): + """The operation is not supported by this provider (names both).""" + + +class BucketNotEmpty(S3Error): + """Refusal to delete a non-empty bucket. Use + ``s3dol.delete_bucket(endpoint, name, force=True)`` for the explicit + cascading form (ADR-0010 §3).""" + + +class S3PartialFailure(S3Error): + """A bulk operation partially failed (ADR-0010 §2). + + Not an ``ExceptionGroup`` — the package supports Python 3.10. + + Attributes: + succeeded: keys the backend reported as deleted (absent keys are + reported as deleted too — S3's semantics, documented, not hidden). + failures: mapping of key -> the error for that key. + """ + + def __init__(self, message: str, *, succeeded: list, failures: dict): + super().__init__(message) + self.succeeded = succeeded + self.failures = failures + + +class ConfigurationError(S3Error, ValueError): + """A construction- or resolution-time misconfiguration (ADR-0012 D1).""" + + +class PresetConflict(ConfigurationError): + """An explicit argument contradicts a *pinned* (verified) preset value.""" + + +class PresetHostMismatch(ConfigurationError): + """The resolved endpoint's host matches a *different* provider's patterns + than the preset you named (ADR-0012 C2).""" + + +class MissingEndpoint(ConfigurationError): + """The named preset needs an endpoint (or endpoint parameters) that nothing + supplied (ADR-0012 C1).""" + + +class MissingPresetParam(ConfigurationError): + """A preset endpoint template has an unbound required parameter.""" + + +class UnknownPresetParam(ConfigurationError): + """A parameter was bound that the preset's templates do not use.""" + + +class InvalidEndpoint(ConfigurationError): + """An endpoint URL failed hygiene checks (userinfo/query/fragment — + ADR-0012 D4: a URL is the residual credential carrier).""" + + +# --------------------------------------------------------------------------- # +# The warning tree (ADR-0012 D1). Strict mode is +# ``filterwarnings('error', category=s3dol.errors.S3DolWarning)`` — there is +# deliberately no ``strict=`` kwarg. +# --------------------------------------------------------------------------- # + + +class S3DolWarning(UserWarning): + """Base of every s3dol warning.""" + + +class S3DolResolutionChanged(S3DolWarning): + """v1 resolves this call differently than v0 would have (ADR-0012 D7). + Emitted by the compat shim; dies with it.""" + + +class AmbiguousResolution(S3DolWarning): + """Resolution is legal but ambiguous (e.g. a named preset whose endpoint, + supplied from the environment, matches no known host pattern).""" + + +class AnonymousFallback(S3DolWarning): + """Anonymous access was used as a fallback (reserved for a future + ``anon='auto'`` — ADR-0012 D5 defers that feature to v1.x).""" + + +# --------------------------------------------------------------------------- # +# Redaction +# --------------------------------------------------------------------------- # + +#: Query parameters whose values are secrets (presigned-URL machinery). +_SECRET_QUERY_PARAMS = ( + "X-Amz-Signature", + "X-Amz-Credential", + "X-Amz-Security-Token", + "Signature", + "AWSAccessKeyId", +) + +_SECRET_PARAM_RE = re.compile( + r"(?P{})=[^&\s]*".format("|".join(map(re.escape, _SECRET_QUERY_PARAMS))), + flags=re.IGNORECASE, +) + + +def redact(text: str) -> str: + """Redact signed-URL query secrets from ``text``. + + Error messages may name the operation, bucket, key, endpoint and code, but + never a credential, token, or signed URL — redaction is a tested function, + not a convention (ADR-0004 §7). + + >>> redact('https://b.s3.amazonaws.com/k?X-Amz-Signature=abc123&x=1') + 'https://b.s3.amazonaws.com/k?X-Amz-Signature=REDACTED&x=1' + """ + return _SECRET_PARAM_RE.sub(lambda m: f"{m.group('name')}=REDACTED", text) + + +# --------------------------------------------------------------------------- # +# Classification — pure, table-driven, per-provider-overridable +# --------------------------------------------------------------------------- # + +#: Kinds a classified error can be. Layer B maps kinds to behaviour; in +#: particular ``ambiguous_head`` triggers the one-``HeadBucket`` +#: disambiguation of ADR-0004 §2 (HEAD has no body, so a missing key and a +#: missing bucket are both ``('404', 404)`` on real AWS). +KINDS = ( + "object_absent", + "bucket_absent", + "ambiguous_head", # HeadObject 404: missing key OR missing bucket + "access_denied", + "credentials", + "invalid_key", + "archived", + "not_supported", + "wrong_region", + "delete_marker", # 405 on HEAD of a delete marker (versioned bucket) + "transient", + "unknown", +) + + +@dataclass(frozen=True) +class Classified: + """The result of classifying a backend error.""" + + kind: str + exception_type: Optional[type] = None # None => re-raise the original + + +#: Rule rows: ``((operation, code, status), kind)``. ``None`` is a wildcard. +#: First match wins; more specific rows come first. +Rule = tuple[tuple[Optional[str], Optional[str], Optional[int]], str] + +_KIND_TO_EXC: dict[str, Optional[type]] = { + "object_absent": ObjectNotFound, + "bucket_absent": BucketNotFound, + "ambiguous_head": ObjectNotFound, # resolved by Layer B before raising + "access_denied": AccessDenied, + "credentials": CredentialsError, + "invalid_key": KeyNotValid, + "archived": ObjectArchived, + "not_supported": NotSupported, + "wrong_region": ConfigurationError, + "delete_marker": ObjectNotFound, + "transient": None, # propagate; botocore retries + "unknown": None, # propagate untouched +} + +#: The default classification table (ADR-0004 §2, measured against botocore). +DEFAULT_RULES: tuple[Rule, ...] = ( + # -- modelled codes (GET/DELETE have bodies, so codes are real words) + ((None, "NoSuchKey", None), "object_absent"), + ((None, "NoSuchBucket", None), "bucket_absent"), + ((None, "NoSuchUpload", None), "object_absent"), + # -- HEAD has no body: botocore synthesizes the code from the HTTP status. + # On real AWS, HeadObject-on-missing-key and HeadObject-on-missing-bucket + # are BOTH ('404', 404) — indistinguishable. moto returns a body + # (Code='NoSuchBucket'/'NoSuchKey'), which the modelled rows above catch, + # so this row only fires on real, body-less responses. + (("HeadObject", "404", 404), "ambiguous_head"), + (("HeadBucket", "404", 404), "bucket_absent"), + (("HeadObject", "404", None), "ambiguous_head"), + (("HeadBucket", "404", None), "bucket_absent"), + # -- versioned-bucket delete marker: HEAD gives 405 Method Not Allowed + (("HeadObject", None, 405), "delete_marker"), + # -- wrong region + ((None, None, 301), "wrong_region"), + ((None, "PermanentRedirect", None), "wrong_region"), + ((None, "AuthorizationHeaderMalformed", None), "wrong_region"), + # -- archived objects + ((None, "InvalidObjectState", None), "archived"), + # -- credentials + ((None, "ExpiredToken", None), "credentials"), + ((None, "InvalidAccessKeyId", None), "credentials"), + ((None, "SignatureDoesNotMatch", None), "credentials"), + ((None, "TokenRefreshRequired", None), "credentials"), + # -- permissions (see ADR-0004 §3 for the deny_means_absent policy) + ((None, "AccessDenied", None), "access_denied"), + ((None, "AllAccessDisabled", None), "access_denied"), + ((None, "403", 403), "access_denied"), + ((None, None, 403), "access_denied"), + # -- key validity that escaped the client-side check + ((None, "KeyTooLongError", None), "invalid_key"), + # -- throttling / transient: propagate, botocore retries + ((None, "SlowDown", None), "transient"), + ((None, "RequestTimeout", None), "transient"), + ((None, "ServiceUnavailable", None), "transient"), + ((None, "InternalError", None), "transient"), + ((None, None, 503), "transient"), +) + +#: Provider overrides live on the preset (ADR-0004 §6 — per-connection, +#: explicit, inspectable; NOT process-global). E.g. Supabase returns 400 where +#: AWS returns 404 on HeadBucket: +#: ``(('HeadBucket', None, 400), 'bucket_absent')``. + + +def _match(rule_key, operation, code, status) -> bool: + op, c, s = rule_key + return ( + (op is None or op == operation) + and (c is None or c == code) + and (s is None or s == status) + ) + + +def classify_client_error( + operation: Optional[str], + code: Optional[str], + status: Optional[int], + *, + overrides: Iterable[Rule] = (), +) -> Classified: + """Classify a backend error by ``(operation, code, status)``. + + ``overrides`` rows (typically from a preset) are consulted first, so a + provider divergence is a table row, never a code branch. + + >>> classify_client_error('HeadBucket', None, 400, + ... overrides=((('HeadBucket', None, 400), 'bucket_absent'),)).kind + 'bucket_absent' + """ + for rules in (tuple(overrides), DEFAULT_RULES): + for key, kind in rules: + if _match(key, operation, code, status): + return Classified(kind=kind, exception_type=_KIND_TO_EXC.get(kind)) + return Classified(kind="unknown", exception_type=None) + + +def code_of(client_error: Any) -> Optional[str]: + """The ``Error.Code`` of a botocore ``ClientError`` (or ``None``).""" + try: + return client_error.response["Error"]["Code"] + except (AttributeError, KeyError, TypeError): + return None + + +def status_of(client_error: Any) -> Optional[int]: + """The HTTP status of a botocore ``ClientError`` (or ``None``).""" + try: + return int(client_error.response["ResponseMetadata"]["HTTPStatusCode"]) + except (AttributeError, KeyError, TypeError, ValueError): + return None + + +def _describe( + *, + operation: str, + code: Optional[str], + bucket: Optional[str], + key: Optional[str], + endpoint: Optional[str], + detail: str = "", +) -> str: + """Build the standard error message: operation, bucket, key, endpoint, + code — and nothing secret (ADR-0004 §7).""" + parts = [f"{operation} failed"] + if bucket is not None: + parts.append(f"bucket={bucket!r}") + if key is not None: + parts.append(f"key={key!r}") + if endpoint is not None: + parts.append(f"endpoint={endpoint!r}") + if code is not None: + parts.append(f"code={code}") + msg = ": ".join([parts[0], ", ".join(parts[1:])]) if parts[1:] else parts[0] + if detail: + msg = f"{msg}. {detail}" + return redact(msg) + + +def translate_client_error( + error: Any, + *, + operation: str, + bucket: Optional[str] = None, + key: Optional[str] = None, + endpoint: Optional[str] = None, + overrides: Iterable[Rule] = (), + deny_means_absent: bool = False, + head_ambiguity_resolver: Optional[Callable[[], bool]] = None, +) -> Optional[S3Error]: + """Translate a botocore ``ClientError`` into the s3dol taxonomy. + + Returns the replacement exception, or ``None`` meaning *propagate the + original* (transient/unknown — rule 2 of the module docstring). + + ``head_ambiguity_resolver`` resolves the body-less ``HeadObject`` 404 + (ADR-0004 §2): it returns ``True`` iff the bucket is known reachable, in + which case the miss really is the object's. It is only called for the + ambiguous case, so the common path costs nothing. + + ``deny_means_absent=True`` classifies an object-op 403 as + ``ObjectNotFound`` — for deployments knowingly running the canonical + least-privilege policy that omits ``s3:ListBucket`` (ADR-0004 §3). + """ + code, status = code_of(error), status_of(error) + classified = classify_client_error(operation, code, status, overrides=overrides) + kind = classified.kind + + if kind == "ambiguous_head": + if head_ambiguity_resolver is not None and head_ambiguity_resolver(): + kind = "object_absent" + elif head_ambiguity_resolver is not None: + kind = "bucket_absent" + else: + kind = "object_absent" # documented default when no resolver bound + + if kind == "access_denied": + if deny_means_absent and key is not None: + kind = "object_absent" + else: + return AccessDenied( + _describe( + operation=operation, + code=code, + bucket=bucket, + key=key, + endpoint=endpoint, + detail=( + "Permission denied. If this key may simply be absent, note " + "that the canonical least-privilege policy (no s3:ListBucket) " + "makes every miss a 403; opt into that reading with " + "S3Connection(deny_means_absent=True)." + ), + ) + ) + + if kind in ("transient", "unknown"): + return None + + exc_type = _KIND_TO_EXC[kind] + detail = { + "wrong_region": "The bucket lives in a different region than the client.", + "delete_marker": "The current version is a delete marker (versioned bucket).", + "credentials": "Credentials are missing, expired, or invalid.", + }.get(kind, "") + message = _describe( + operation=operation, + code=code, + bucket=bucket, + key=key, + endpoint=endpoint, + detail=detail, + ) + if exc_type is ObjectArchived: + return ObjectArchived(message) + return exc_type(message) + + +def translate_s3_errors( + *, + operation: str, + key_arg: Optional[str] = None, + **translate_kwargs, +): + """The one seam: decorate a Layer B method so botocore's ``ClientError`` + becomes the s3dol taxonomy (ADR-0004 §1). + + ``key_arg`` names the decorated function's key *parameter* (resolved via + the call's kwargs/positionals — never an attribute, so the key is not lost + the way azuredol's ``key_arg='blob'`` loses it). The store instance is + expected to provide ``_error_context()`` returning + ``dict(bucket=..., endpoint=..., overrides=..., deny_means_absent=..., + head_ambiguity_resolver=...)`` — Layer B owns that context. + """ + + def decorator(func): + import inspect + + sig = inspect.signature(func) + + @wraps(func) + def wrapper(self, *args, **kwargs): + from botocore.exceptions import ClientError, NoCredentialsError + + try: + return func(self, *args, **kwargs) + except ClientError as error: + key = None + if key_arg is not None: + bound = sig.bind(self, *args, **kwargs) + key = bound.arguments.get(key_arg) + context = dict(getattr(self, "_error_context", dict)()) + context.update(translate_kwargs) + replacement = translate_client_error( + error, operation=operation, key=key, **context + ) + if replacement is None: + raise + raise replacement from error + except NoCredentialsError as error: + raise CredentialsError( + "No AWS credentials found. Configure credentials (env vars, " + "~/.aws/credentials, SSO, or S3Connection(credentials=...)), " + "or pass anon=True for public data." + ) from error + + return wrapper + + return decorator diff --git a/s3dol/presets.py b/s3dol/presets.py new file mode 100644 index 0000000..2a14a50 --- /dev/null +++ b/s3dol/presets.py @@ -0,0 +1,439 @@ +"""Provider presets: config rows, not subclasses. + +Design record: `misc/docs/decisions/0003-provider-presets-and-capabilities.md` +(the survey of 16 backends, the checksum analysis, capabilities policy) amended +by `misc/docs/decisions/0012-credential-and-endpoint-resolution.md` §D4 +(``params``/``host_patterns``/``requires_endpoint``/``soft``/``config_kwargs``, +derived ``pinned``; ``signature_version`` and ``client_kwargs`` moved to the +connection; ``addressing_style`` mandatory with ``'auto'`` illegal; the +structural no-credential rule). + +A provider divergence is a row in this registry — never a subclass, never an +endpoint-substring branch in the hot path. Users register their own rows: + +>>> row = Preset(name='mycorp', addressing_style='path', +... endpoint_url='https://s3.mycorp.example', requires_endpoint=False) +>>> register(row) +>>> get_preset('mycorp').endpoint_url +'https://s3.mycorp.example' +>>> unregister('mycorp') # (cleanup for this doctest) + +Most builtin rows are **doc-sourced, not verified against a live endpoint** — +they say so via ``verified=False``. Wrong values mostly fail loudly, which is +the acceptable failure mode; ``pinned`` (which makes a row's region literal +conflict-raising) is derived and *gated on* ``verified``, so it is dormant +until a row earns evidence (ADR-0012 D4). +""" + +from __future__ import annotations + +import string +from dataclasses import dataclass, replace +from fnmatch import fnmatch +from typing import Any, Literal, Optional +from urllib.parse import urlsplit + +from s3dol.errors import ( + ConfigurationError, + InvalidEndpoint, + MissingPresetParam, + UnknownPresetParam, + Rule, +) + +# --------------------------------------------------------------------------- # +# Capabilities +# --------------------------------------------------------------------------- # + + +@dataclass(frozen=True) +class Capabilities: + """What a backend supports. A *static declared table*, never a probe + (a probe costs a round-trip, is wrong under partial permissions, and can't + be trusted anyway — ADR-0003 §2). Missing capabilities are handled + per-capability: emulate when exact (batch delete -> loop), substitute when + equivalent (ListObjectsV2 -> V1), raise ``NotSupported`` when there is no + honest fallback. Never silently degrade correctness.""" + + list_objects_v2: bool = True + batch_delete: bool = True + presigned_post: bool = True + multipart: bool = True + max_multipart_parts: int = 10_000 + min_part_size: int = 5 * 2**20 + object_tagging: bool = True + versioning: bool = True + conditional_writes: bool = True + consistency: Literal["strong", "read-after-write", "eventual"] = "strong" + + +DEFAULT_CAPABILITIES = Capabilities() + + +# --------------------------------------------------------------------------- # +# Preset +# --------------------------------------------------------------------------- # + +#: ``Config``-constructor kwargs a preset row may carry. Scalars only, and +#: ``proxies``/``proxies_config``/``client_context_params`` are structurally +#: excluded — a preset must not be able to smuggle a credential-bearing proxy +#: URL (ADR-0012 D4). +CONFIG_KWARGS_ALLOWLIST = frozenset( + {"connect_timeout", "read_timeout", "max_pool_connections", "tcp_keepalive"} +) + +_SCALAR_TYPES = (str, int, float, bool, type(None)) + + +def _template_params(template: str) -> set: + """Placeholder names in a ``str.format``-style template. + + >>> sorted(_template_params('https://{account_id}.r2.example/{x}')) + ['account_id', 'x'] + """ + return { + name for _, name, _, _ in string.Formatter().parse(template) if name is not None + } + + +def validate_endpoint_hygiene(url: str, *, what: str = "endpoint_url") -> None: + """Refuse credential-carrying URL shapes: userinfo in the netloc, a query + string, or a fragment (ADR-0012 D4 — a query-string endpoint is accepted by + botocore and retained verbatim in ``client.meta.endpoint_url``, so it would + leak into ``diagnose()`` and tracebacks). + + >>> validate_endpoint_hygiene('https://s3.example.com') + >>> validate_endpoint_hygiene('https://user:pw@s3.example.com') + Traceback (most recent call last): + ... + s3dol.errors.InvalidEndpoint: ... + """ + parts = urlsplit(url) + if "@" in parts.netloc: + raise InvalidEndpoint( + f"{what} must not carry userinfo (user[:password]@host): got a URL " + f"with '@' in its netloc. Pass credentials via " + f"S3Connection(credentials=...), never in a URL." + ) + if parts.query: + raise InvalidEndpoint(f"{what} must not carry a query string: {url!r}") + if parts.fragment: + raise InvalidEndpoint(f"{what} must not carry a fragment: {url!r}") + + +@dataclass(frozen=True) +class Preset: + """One provider's facts. Public, shareable, committable config — which is + why there is **no credential slot**, structurally (ADR-0012 D4). + + ``endpoint_url``/``region_name`` may contain ``{placeholders}``; bind them + with :meth:`bind`. A row whose endpoint template has an unbound *required* + placeholder contributes no endpoint rung (ADR-0012 C1). + """ + + name: str + #: Mandatory; no default; ``'auto'`` is illegal (ADR-0012 D4). + addressing_style: Literal["path", "virtual"] + endpoint_url: Optional[str] = None + region_name: Optional[str] = None + #: ``None`` == the row makes no claim (the ladder continues — ADR-0012 D3). + checksum: Optional[Literal["when_supported", "when_required"]] = None + payload_signing_enabled: Optional[bool] = None + capabilities: Capabilities = DEFAULT_CAPABILITIES + #: Presigning sometimes genuinely differs from the API config (Hetzner + #: presigns virtual while the API is path; R2 presigns only on the S3 API + #: domain) — ADR-0003 §1. + presign_endpoint_url: Optional[str] = None + presign_addressing_style: Optional[Literal["path", "virtual"]] = None + #: Bound template parameters, e.g. ``(('account_id', 'abc'),)``. + params: tuple[tuple[str, str], ...] = () + #: ``fnmatch`` patterns over the endpoint *hostname*; used for detection + #: (pass 2) and the C2 named-vs-resolved guard. + host_patterns: tuple[str, ...] = () + #: The provider cannot be reached without an explicit endpoint. + requires_endpoint: bool = False + #: Soft rows are last-resort fallbacks (``aws``, ``generic-s3``): they + #: never win detection over a non-soft row and never trigger the C2 guard. + soft: bool = False + #: Extra ``botocore.Config`` kwargs — allowlisted, scalars only. + config_kwargs: tuple[tuple[str, Any], ...] = () + #: Error-classification overrides, consulted before the default table + #: (ADR-0004 §2/§6): rows of ``((operation, code, status), kind)``. + error_overrides: tuple[Rule, ...] = () + #: ``True`` only when the row was verified against a live endpoint (with a + #: date in the row's docstring/comment). Builtin rows are doc-sourced. + verified: bool = False + + @property + def pinned(self) -> bool: + """Whether the row's region literal is authoritative enough to make a + conflicting explicit ``region_name=`` raise ``PresetConflict``. + Derived, and gated on ``verified`` — dormant while rows are + doc-sourced (ADR-0012 D4).""" + return ( + self.verified + and self.region_name is not None + and not _template_params(self.region_name) + ) + + def __post_init__(self): + if self.addressing_style not in ("path", "virtual"): + raise ConfigurationError( + f"Preset {self.name!r}: addressing_style must be 'path' or " + f"'virtual' (got {self.addressing_style!r}). 'auto' is illegal: " + f"every row must state what the provider actually needs " + f"(ADR-0012 D4)." + ) + for url_field in ("endpoint_url", "presign_endpoint_url"): + url = getattr(self, url_field) + if url is not None: + validate_endpoint_hygiene(url, what=f"Preset.{url_field}") + for key, value in self.config_kwargs: + if key not in CONFIG_KWARGS_ALLOWLIST: + raise ConfigurationError( + f"Preset {self.name!r}: config_kwargs key {key!r} is not " + f"allowlisted (allowed: {sorted(CONFIG_KWARGS_ALLOWLIST)}). " + f"This list is closed to keep presets credential-free." + ) + if not isinstance(value, _SCALAR_TYPES): + raise ConfigurationError( + f"Preset {self.name!r}: config_kwargs values must be " + f"scalars; {key!r} is a {type(value).__name__}." + ) + unknown = set(dict(self.params)) - self.template_params() + if unknown: + raise UnknownPresetParam( + f"Preset {self.name!r}: bound params {sorted(unknown)} appear in " + f"no template. Templates use: {sorted(self.template_params())}." + ) + + def template_params(self) -> set: + """All placeholder names across this row's templates.""" + names = set() + for template in ( + self.endpoint_url, + self.region_name, + self.presign_endpoint_url, + ): + if template: + names |= _template_params(template) + return names + + def bind(self, **params: str) -> "Preset": + """A copy of this row with template ``params`` bound. + + >>> r2 = get_preset('r2').bind(account_id='abc123') + >>> r2.bound_endpoint() + 'https://abc123.r2.cloudflarestorage.com' + """ + unknown = set(params) - self.template_params() + if unknown: + raise UnknownPresetParam( + f"Preset {self.name!r} has no template parameter(s) " + f"{sorted(unknown)}; it uses {sorted(self.template_params())}." + ) + merged = {**dict(self.params), **{k: str(v) for k, v in params.items()}} + return replace(self, params=tuple(sorted(merged.items()))) + + def _bound(self, template: Optional[str], *, extra: Optional[dict] = None): + """``(value, missing_params)`` for a template under current bindings.""" + if not template: + return None, set() + bindings = {**dict(self.params), **(extra or {})} + missing = _template_params(template) - set(bindings) + if missing: + return None, missing + return template.format(**bindings), set() + + def bound_endpoint(self, *, region_name: Optional[str] = None) -> Optional[str]: + """The endpoint URL with params bound, or ``None`` if the template has + unbound placeholders (C1: an unbound row contributes no rung — + ``preset='minio'`` silently becoming AWS is structurally impossible). + + ``region_name`` binds a ``{region_name}`` placeholder, per the region + ladder's first rung (ADR-0012 D3).""" + extra = {"region_name": region_name} if region_name else None + value, _missing = self._bound(self.endpoint_url, extra=extra) + return value + + def missing_endpoint_params(self, *, region_name: Optional[str] = None) -> set: + """Which placeholders keep :meth:`bound_endpoint` from producing a URL.""" + extra = {"region_name": region_name} if region_name else None + _value, missing = self._bound(self.endpoint_url, extra=extra) + return missing + + def matches_host(self, endpoint_url: str) -> bool: + """Whether ``endpoint_url``'s hostname matches this row's patterns.""" + host = urlsplit(endpoint_url).hostname or "" + return any(fnmatch(host, pattern) for pattern in self.host_patterns) + + +# --------------------------------------------------------------------------- # +# The registry +# --------------------------------------------------------------------------- # + +_REGISTRY: dict[str, Preset] = {} + + +def register(preset: Preset, *, overwrite: bool = False) -> None: + """Add a row. Open-closed: adding a provider is adding a row.""" + if not overwrite and preset.name in _REGISTRY: + raise ConfigurationError( + f"A preset named {preset.name!r} is already registered. " + f"Pass overwrite=True to replace it." + ) + _REGISTRY[preset.name] = preset + + +def unregister(name: str) -> None: + """Remove a row (mostly for tests).""" + _REGISTRY.pop(name, None) + + +def get_preset(name: str) -> Preset: + """Look a row up by name; unknown names raise, naming the registry.""" + try: + return _REGISTRY[name] + except KeyError: + raise ConfigurationError( + f"Unknown preset {name!r}. Registered: {sorted(_REGISTRY)}. " + f"Register your own with s3dol.presets.register(Preset(...))." + ) from None + + +def available_presets() -> tuple[str, ...]: + return tuple(sorted(_REGISTRY)) + + +def detect_preset(endpoint_url: str) -> Optional[Preset]: + """Detection (resolution pass 2): the non-soft row whose ``host_patterns`` + match the resolved endpoint's host. Two matching rows is genuinely + ambiguous and raises; zero yields ``None`` (the caller falls back to + ``generic-s3``/``aws`` — ADR-0012 D4).""" + matches = [ + p for p in _REGISTRY.values() if not p.soft and p.matches_host(endpoint_url) + ] + if len(matches) > 1: + names = sorted(p.name for p in matches) + raise ConfigurationError( + f"Endpoint {endpoint_url!r} matches more than one preset's host " + f"patterns: {names}. Name one explicitly with preset=." + ) + return matches[0] if matches else None + + +# --------------------------------------------------------------------------- # +# Builtin rows (ADR-0003 §1's table; doc-sourced — verified=False throughout) +# --------------------------------------------------------------------------- # + +_SUPABASE_ERROR_OVERRIDES: tuple[Rule, ...] = ( + # Supabase returns 400 where AWS returns 404 on HeadBucket (ADR-0004 §2). + (("HeadBucket", None, 400), "bucket_absent"), +) + +_BUILTIN_PRESETS = ( + # AWS: the reference semantics. SDK-resolved endpoint; soft fallback row. + Preset( + name="aws", + addressing_style="virtual", + soft=True, + host_patterns=("*.amazonaws.com",), + ), + # Fallback for "some endpoint resolved, nothing matched": makes no claim + # beyond path addressing (botocore's own behaviour for custom endpoints), + # so it is byte-identical to today for endpoint-passing callers. + Preset(name="generic-s3", addressing_style="path", soft=True), + Preset( + name="minio", + addressing_style="path", + region_name="us-east-1", # conventional + requires_endpoint=True, + ), + Preset( + name="r2", + addressing_style="virtual", + endpoint_url="https://{account_id}.r2.cloudflarestorage.com", + region_name="auto", # R2's literal region string + host_patterns=("*.r2.cloudflarestorage.com",), + # Presign only on the S3 API domain, never a custom domain (docs). + ), + Preset( + name="scaleway", + addressing_style="virtual", + endpoint_url="https://s3.{region_name}.scw.cloud", + host_patterns=("s3.*.scw.cloud",), + capabilities=Capabilities(max_multipart_parts=1000), + ), + Preset( + name="hetzner", + addressing_style="path", # virtual breaks plain Get/Put (TLS cert) + presign_addressing_style="virtual", # docs: needed for presigned URLs + endpoint_url="https://{location}.your-objectstorage.com", + region_name="{location}", # must repeat the location + payload_signing_enabled=False, + host_patterns=("*.your-objectstorage.com",), + ), + Preset( + name="backblaze", + addressing_style="virtual", + endpoint_url="https://s3.{region_name}.backblazeb2.com", + checksum="when_required", # mandatory: rejects default SDK checksums + host_patterns=("s3.*.backblazeb2.com",), + capabilities=Capabilities(presigned_post=False), + ), + Preset( + name="wasabi", + addressing_style="path", # the vendor's own advice + endpoint_url="https://s3.{region_name}.wasabisys.com", + host_patterns=("s3.*.wasabisys.com",), + ), + Preset( + name="gcs", + addressing_style="virtual", + endpoint_url="https://storage.googleapis.com", + host_patterns=("storage.googleapis.com",), + capabilities=Capabilities(list_objects_v2=False, batch_delete=False), + ), + Preset( + name="supabase", + addressing_style="path", # forced + endpoint_url="https://{project_ref}.storage.supabase.co/storage/v1/s3", + checksum="when_required", # the ADR-0003 §3 corruption fix + host_patterns=("*.supabase.co", "*.supabase.in"), + error_overrides=_SUPABASE_ERROR_OVERRIDES, + ), + Preset( + name="digitalocean", + addressing_style="virtual", + endpoint_url="https://{region_name}.digitaloceanspaces.com", + host_patterns=("*.digitaloceanspaces.com",), + ), + Preset( + name="tigris", + addressing_style="virtual", + endpoint_url="https://fly.storage.tigris.dev", + host_patterns=("fly.storage.tigris.dev", "*.storage.tigris.dev"), + ), + Preset( + name="ceph", + addressing_style="path", + requires_endpoint=True, # behaviour varies more by release than vendor + ), + # Test doubles + Preset( + name="localstack", + addressing_style="path", + endpoint_url="http://localhost:4566", + region_name="us-east-1", + ), + Preset( + name="moto", + addressing_style="path", + endpoint_url="http://localhost:5000", + region_name="us-east-1", + ), +) + +for _preset in _BUILTIN_PRESETS: + register(_preset) +del _preset diff --git a/s3dol/tests/test_connection.py b/s3dol/tests/test_connection.py new file mode 100644 index 0000000..34fc75c --- /dev/null +++ b/s3dol/tests/test_connection.py @@ -0,0 +1,519 @@ +"""Tests for ``S3Connection`` and the pure resolver (ADR-0012). + +Tier 1 throughout (fake environ dicts, zero I/O), plus a small moto tier-2 +section at the bottom proving the built client's presigning is SigV4 — the +regression test for issue #10, at the layer that fixes it. +""" + +import pickle +import threading + +import pytest + +from s3dol.connection import ( + CallableCredentials, + CredentialProvider, + ProfileCredentials, + Resolution, + S3Connection, + StaticCredentials, + load_aws_config, + normalize_credentials, + resolve, +) +from s3dol.errors import ( + AmbiguousResolution, + ConfigurationError, + MissingEndpoint, + PresetHostMismatch, +) +from s3dol.presets import get_preset + + +def fetch_test_credentials(): + """Module-level (picklable) credential callable for tests.""" + return {"access_key": "AKIATEST", "secret_key": "sEkR3t"} + + +# --------------------------------------------------------------------------- # +# credential normalisation (D5) +# --------------------------------------------------------------------------- # + + +def test_none_stays_none_meaning_the_chain(): + assert normalize_credentials(None) is None + + +def test_bare_str_is_always_a_profile_name(): + provider = normalize_credentials("prod") + assert isinstance(provider, ProfileCredentials) and provider.profile == "prod" + + +@pytest.mark.parametrize("shape", [("AK", "SK"), ("AK", "SK", "TOK"), ["AK", "SK"]]) +def test_tuple_shapes(shape): + provider = normalize_credentials(shape) + assert isinstance(provider, StaticCredentials) + assert provider.access_key == "AK" and provider.secret_key == "SK" + + +def test_tuple_wrong_arity_rejected(): + with pytest.raises(ConfigurationError, match="tuple"): + normalize_credentials(("only-one",)) + + +def test_mapping_of_aws_keys(): + provider = normalize_credentials( + {"aws_access_key_id": "AK", "aws_secret_access_key": "SK"} + ) + assert isinstance(provider, StaticCredentials) + + +def test_mapping_unknown_key_rejected_with_the_allowed_list(): + with pytest.raises(ConfigurationError, match="aws_access_key_id"): + normalize_credentials({"access_key": "AK"}) + + +def test_token_only_mapping_rejected(): + # boto3 would leave Credentials(None, None, TOKEN) — a measured hazard. + with pytest.raises(ConfigurationError, match="token"): + normalize_credentials({"aws_session_token": "TOK"}) + + +def test_provider_instance_passes_through(): + provider = StaticCredentials("AK", "SK") + assert normalize_credentials(provider) is provider + + +def test_provider_class_rejected(): + with pytest.raises(ConfigurationError, match="instance"): + normalize_credentials(StaticCredentials) + + +def test_live_botocore_object_rejected(): + from botocore.credentials import Credentials + + with pytest.raises(ConfigurationError, match="live"): + normalize_credentials(Credentials("AK", "SK")) + + +def test_unpicklable_callable_rejected(): + with pytest.raises(ConfigurationError, match="picklable"): + normalize_credentials(lambda: {"access_key": "a", "secret_key": "b"}) + + +def test_picklable_callable_accepted(): + provider = normalize_credentials(fetch_test_credentials) + assert isinstance(provider, CallableCredentials) + assert isinstance(provider, CredentialProvider) + + +def test_alien_type_rejected(): + with pytest.raises(ConfigurationError, match="accepted"): + normalize_credentials(42) + + +def test_static_credentials_repr_redacts(): + provider = StaticCredentials("AKIAEXAMPLE", "sEkR3tVALUE", "tOkEnVALUE") + text = repr(provider) + assert "sEkR3tVALUE" not in text and "tOkEnVALUE" not in text + assert "AKIA" in text # identity by prefix only + + +def test_callable_credentials_normalize_aliases_and_reject_garbage(): + assert CallableCredentials._normalize({"key": "a", "secret": "b"})["access_key"] == "a" + with pytest.raises(ConfigurationError, match="access_key"): + CallableCredentials._normalize({"nope": 1}) + + +# --------------------------------------------------------------------------- # +# S3Connection construction (D1) +# --------------------------------------------------------------------------- # + + +def test_construction_is_lazy_no_client_no_io(): + spec = S3Connection(endpoint_url="http://localhost:9000") + assert "_client" not in spec.__dict__ + + +def test_anon_with_credentials_raises(): + with pytest.raises(ConfigurationError, match="anon"): + S3Connection(anon=True, credentials=("AK", "SK")) + with pytest.raises(ConfigurationError, match="anon"): + S3Connection(anon=True, profile="prod") + + +def test_anon_with_deny_means_absent_raises(): + with pytest.raises(ConfigurationError, match="silently empty"): + S3Connection(anon=True, deny_means_absent=True) + + +def test_auto_addressing_style_is_illegal(): + with pytest.raises(ConfigurationError, match="auto"): + S3Connection(addressing_style="auto") + + +def test_bad_checksum_rejected(): + with pytest.raises(ConfigurationError, match="checksum"): + S3Connection(checksum="always") + + +def test_unknown_preset_name_dies_at_construction(): + with pytest.raises(ConfigurationError, match="Unknown preset"): + S3Connection(preset="no-such-provider") + + +def test_endpoint_hygiene_applies_to_the_kwarg(): + with pytest.raises(ConfigurationError): + S3Connection(endpoint_url="https://user:pw@host") + + +def test_client_kwargs_mapping_normalised_to_tuple(): + spec = S3Connection(client_kwargs={"use_ssl": False}) + assert spec.client_kwargs == (("use_ssl", False),) + hash(spec) # stays hashable + + +def test_equal_specs_hash_equal_so_caches_are_safe(): + a = S3Connection(endpoint_url="http://x:1", credentials=("AK", "SK")) + b = S3Connection(endpoint_url="http://x:1", credentials=("AK", "SK")) + assert a == b and hash(a) == hash(b) + assert a != S3Connection(endpoint_url="http://x:2", credentials=("AK", "SK")) + + +def test_repr_shows_only_non_defaults_and_no_secrets(): + spec = S3Connection( + endpoint_url="http://x:1", credentials=("AKIAEXAMPLE", "sEkR3tVALUE") + ) + text = repr(spec) + assert "sEkR3tVALUE" not in text + assert "http://x:1" in text + assert "deny_means_absent" not in text # default fields stay quiet + + +# --------------------------------------------------------------------------- # +# pickling (D1 §3: mandatory for every connection) +# --------------------------------------------------------------------------- # + + +def test_pickle_round_trip_preserves_equality(): + spec = S3Connection( + preset="r2", + credentials=("AK", "SK", "TOK"), + region_name="auto", + client_kwargs={"use_ssl": True}, + ) + clone = pickle.loads(pickle.dumps(spec)) + assert clone == spec + assert isinstance(clone._lock, type(threading.Lock())) + + +def test_pickle_drops_a_cached_client(): + spec = S3Connection(endpoint_url="http://x:1") + object.__setattr__(spec, "_client", object()) # simulate a built client + clone = pickle.loads(pickle.dumps(spec)) + assert "_client" not in clone.__dict__ + + +def test_callable_credentials_pickle(): + spec = S3Connection(credentials=fetch_test_credentials) + clone = pickle.loads(pickle.dumps(spec)) + assert isinstance(clone.credentials, CallableCredentials) + + +# --------------------------------------------------------------------------- # +# the client is built once, under the lock (D1 §2) +# --------------------------------------------------------------------------- # + + +def test_client_build_races_produce_one_client(monkeypatch): + builds = [] + barrier = threading.Barrier(8) + + def fake_build(self): + builds.append(1) + return object() + + monkeypatch.setattr(S3Connection, "_build_client", fake_build) + spec = S3Connection(endpoint_url="http://x:1") + clients = [] + + def grab(): + barrier.wait() + clients.append(spec.client) + + threads = [threading.Thread(target=grab) for _ in range(8)] + for t in threads: + t.start() + for t in threads: + t.join() + assert len(builds) == 1 + assert len({id(c) for c in clients}) == 1 + + +# --------------------------------------------------------------------------- # +# resolve(): purity and the ladders (D2/D3/D4) +# --------------------------------------------------------------------------- # + + +def test_resolve_is_pure_wrt_the_process_environment(monkeypatch): + monkeypatch.setenv("AWS_ENDPOINT_URL_S3", "http://should-not-be-seen:1") + resolution = resolve(S3Connection(), {}) + assert resolution.endpoint_url.value is None + assert resolution.endpoint_url.source == "SDK resolver" + + +def test_endpoint_rung1_kwarg_wins_over_everything(): + spec = S3Connection( + endpoint_url="http://explicit:1", preset=get_preset("r2").bind(account_id="a") + ) + resolution = resolve(spec, {"AWS_ENDPOINT_URL_S3": "http://env:1"}) + assert resolution.endpoint_url == ("http://explicit:1", "endpoint_url=", True) + + +def test_endpoint_rung2_bound_preset(): + spec = S3Connection(preset=get_preset("r2").bind(account_id="abc")) + resolution = resolve(spec, {}) + assert resolution.endpoint_url.value == "https://abc.r2.cloudflarestorage.com" + assert resolution.endpoint_url.source == "preset:r2" + assert resolution.endpoint_url.passed is True + + +def test_endpoint_env_rungs_are_reported_not_passed(): + resolution = resolve(S3Connection(), {"AWS_ENDPOINT_URL_S3": "http://env:1"}) + assert resolution.endpoint_url.value == "http://env:1" + assert resolution.endpoint_url.source == "env:AWS_ENDPOINT_URL_S3" + assert resolution.endpoint_url.passed is False # botocore reads it itself + lower = resolve(S3Connection(), {"AWS_ENDPOINT_URL": "http://gen:1"}) + assert lower.endpoint_url.source == "env:AWS_ENDPOINT_URL" + priority = resolve( + S3Connection(), + {"AWS_ENDPOINT_URL_S3": "http://svc:1", "AWS_ENDPOINT_URL": "http://gen:1"}, + ) + assert priority.endpoint_url.value == "http://svc:1" # service beats generic + + +def test_endpoint_aws_config_rung(): + aws_config = {"profiles": {"default": {"endpoint_url": "http://cfg:1"}}} + resolution = resolve(S3Connection(), {}, aws_config) + assert resolution.endpoint_url.value == "http://cfg:1" + assert resolution.endpoint_url.source == "aws-config" + assert resolution.endpoint_url.passed is False + + +def test_c1_requires_endpoint_raises_missing_endpoint(): + with pytest.raises(MissingEndpoint) as info: + resolve(S3Connection(preset="minio"), {}) + assert "AWS_ENDPOINT_URL_S3" in str(info.value) + + +def test_c1_unbound_template_raises_and_names_the_params(): + with pytest.raises(MissingEndpoint) as info: + resolve(S3Connection(preset="r2"), {}) + assert "account_id" in str(info.value) + + +def test_c1_env_endpoint_satisfies_a_requiring_preset(): + resolution = resolve( + S3Connection(preset="minio"), {"AWS_ENDPOINT_URL_S3": "http://minio:9000"} + ) + assert resolution.endpoint_url.value == "http://minio:9000" + assert resolution.preset.name == "minio" + + +def test_c2_env_endpoint_matching_another_provider_raises(): + with pytest.raises(PresetHostMismatch, match="wasabi"): + resolve( + S3Connection(preset="r2"), + {"AWS_ENDPOINT_URL_S3": "https://s3.eu-central-1.wasabisys.com"}, + ) + + +def test_c2_env_endpoint_matching_nothing_warns_not_raises(): + resolution = resolve( + S3Connection(preset="r2"), + {"AWS_ENDPOINT_URL_S3": "https://proxy.mycorp.internal"}, + ) + assert any(cls is AmbiguousResolution for cls, _ in resolution.notes) + + +def test_c2_never_fires_for_rung1(): + # endpoint_url= is the documented escape (rung 1 never triggers C2). + resolution = resolve( + S3Connection(preset="r2", endpoint_url="https://proxy.mycorp.internal"), {} + ) + assert resolution.notes == () + + +def test_pass2_detection_supplies_the_full_row(): + resolution = resolve( + S3Connection(endpoint_url="https://abc.r2.cloudflarestorage.com"), {} + ) + assert resolution.preset.name == "r2" + assert resolution.preset_source == "detected" + assert resolution.addressing_style.value == "virtual" + assert resolution.region_name.value == "auto" # the row's literal + + +def test_pass2_fallbacks(): + with_endpoint = resolve(S3Connection(endpoint_url="http://unknown:1"), {}) + assert with_endpoint.preset.name == "generic-s3" + assert with_endpoint.preset_source == "fallback:generic-s3" + without = resolve(S3Connection(), {}) + assert without.preset.name == "aws" + assert without.preset_source == "fallback:aws" + + +def test_region_ladder(): + kwarg = resolve(S3Connection(region_name="eu-west-3"), {}) + assert kwarg.region_name == ("eu-west-3", "region_name=", True) + captured = resolve( + S3Connection(endpoint_url="https://s3.eu-central-2.amazonaws.com"), {} + ) + assert captured.region_name.value == "eu-central-2" + assert "endpoint host" in captured.region_name.source + env = resolve(S3Connection(), {"AWS_DEFAULT_REGION": "ap-south-1"}) + assert env.region_name == ("ap-south-1", "env:AWS_DEFAULT_REGION", True) + profile = resolve( + S3Connection(), {}, {"profiles": {"default": {"region": "sa-east-1"}}} + ) + assert profile.region_name.value == "sa-east-1" + bottom = resolve(S3Connection(), {}) + assert bottom.region_name == ("us-east-1", "default", True) + + +def test_aws_region_env_var_is_deliberately_ignored(): + # botocore does not read AWS_REGION; v1 must not "helpfully" add it. + resolution = resolve(S3Connection(), {"AWS_REGION": "eu-north-1"}) + assert resolution.region_name.value == "us-east-1" + + +def test_addressing_ladder_soft_rows_omit(): + hard = resolve(S3Connection(endpoint_url="https://storage.googleapis.com"), {}) + assert hard.addressing_style.passed is True # gcs is a real claim + soft = resolve(S3Connection(), {}) # aws fallback: omit, channel open + assert soft.addressing_style.passed is False + profile = resolve( + S3Connection(), + {}, + {"profiles": {"default": {"s3": {"addressing_style": "virtual"}}}}, + ) + assert profile.addressing_style.value == "virtual" + assert profile.addressing_style.source == "aws-config profile" + assert profile.addressing_style.passed is False # omit preserves the channel + kwarg = resolve(S3Connection(addressing_style="path"), {}) + assert kwarg.addressing_style == ("path", "addressing_style=", True) + + +def test_checksum_ladder_env_stays_open(): + row = resolve( + S3Connection(preset=get_preset("backblaze").bind()), + {"AWS_ENDPOINT_URL_S3": "https://s3.us-west-004.backblazeb2.com"}, + ) + assert row.checksum == ("when_required", "preset:backblaze", True) + env = resolve( + S3Connection(), {"AWS_REQUEST_CHECKSUM_CALCULATION": "when_required"} + ) + assert env.checksum.value == "when_required" + assert env.checksum.passed is False # omit the key: env keeps working + bottom = resolve(S3Connection(), {}) + assert bottom.checksum == ("when_supported", "botocore default", False) + kwarg = resolve(S3Connection(checksum="when_required"), {}) + assert kwarg.checksum.passed is True + + +def test_payload_signing_ladder(): + hetzner = resolve( + S3Connection(preset="hetzner", endpoint_url="https://fsn1.your-objectstorage.com"), + {}, + ) + assert hetzner.payload_signing_enabled == (False, "preset:hetzner", True) + default = resolve(S3Connection(), {}) + assert default.payload_signing_enabled == (None, "omitted", False) + + +def test_signature_version_always_passed(): + resolution = resolve(S3Connection(), {}) + assert resolution.signature_version.value == "s3v4" + assert resolution.signature_version.passed is True + + +def test_credential_provenance_labels_never_values(): + explicit = resolve(S3Connection(credentials=("AKIAXYZ", "sEkR3t")), {}) + assert "sEkR3t" not in str(explicit.credential_provenance) + assert "AKIA" in explicit.credential_provenance + profile = resolve(S3Connection(profile="prod"), {}) + assert profile.credential_provenance == "profile:prod" + env = resolve(S3Connection(), {"AWS_ACCESS_KEY_ID": "AK"}) + assert "env" in env.credential_provenance + anon = resolve(S3Connection(anon=True), {}) + assert "anonymous" in anon.credential_provenance + + +def test_resolution_is_a_value_and_consults_named_env_keys(): + resolution = resolve(S3Connection(), {"AWS_DEFAULT_REGION": "eu-west-1"}) + assert isinstance(resolution, Resolution) + consulted = dict(resolution.environ_consulted) + assert consulted["AWS_DEFAULT_REGION"] == "eu-west-1" + assert "AWS_ENDPOINT_URL_S3" in consulted + + +def test_load_aws_config_is_credential_scrubbed(): + config = load_aws_config() # impure by design; must still be safe to print + flat = repr(config).lower() + assert "secret" not in flat + assert "aws_access_key_id" not in flat + assert "session_token" not in flat + + +# --------------------------------------------------------------------------- # +# tier 2 (moto): the built client — SigV4 presigning is issue #10's regression +# --------------------------------------------------------------------------- # + +moto = pytest.importorskip("moto") + + +@pytest.fixture +def _mock_aws_env(monkeypatch): + for var in ("AWS_ENDPOINT_URL", "AWS_ENDPOINT_URL_S3", "AWS_PROFILE"): + monkeypatch.delenv(var, raising=False) + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "testing") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "testing") + monkeypatch.setenv("AWS_DEFAULT_REGION", "us-east-1") + + +def test_client_presigns_sigv4_even_in_us_east_1(_mock_aws_env): + # us-east-1 is precisely a region where botocore, left to itself, silently + # downgrades presigning to SigV2 (ADR-0003 §4). Assert on the URL, never on + # client.meta.config.signature_version — it reports 's3v4' while emitting + # SigV2 (measured). + with moto.mock_aws(): + spec = S3Connection(region_name="us-east-1") + client = spec.client + client.create_bucket(Bucket="test-bucket") + client.put_object(Bucket="test-bucket", Key="k", Body=b"v") + url = client.generate_presigned_url( + "get_object", Params={"Bucket": "test-bucket", "Key": "k"}, ExpiresIn=60 + ) + assert "X-Amz-Algorithm=AWS4-HMAC-SHA256" in url + assert "AWSAccessKeyId" not in url # the SigV2 tell + + +def test_anon_client_is_unsigned_and_still_builds(_mock_aws_env): + from botocore import UNSIGNED + + with moto.mock_aws(): + spec = S3Connection(anon=True, region_name="us-east-1") + assert spec.client.meta.config.signature_version is UNSIGNED + + +def test_static_credentials_reach_the_request(_mock_aws_env): + with moto.mock_aws(): + spec = S3Connection( + credentials=("AKIAINJECTED", "sEkR3t"), region_name="us-east-1" + ) + client = spec.client + client.create_bucket(Bucket="cred-bucket") + url = client.generate_presigned_url( + "get_object", Params={"Bucket": "cred-bucket", "Key": "k"}, ExpiresIn=60 + ) + assert "AKIAINJECTED" in url # the injected identity signed, not the env one diff --git a/s3dol/tests/test_diagnose.py b/s3dol/tests/test_diagnose.py new file mode 100644 index 0000000..402a156 --- /dev/null +++ b/s3dol/tests/test_diagnose.py @@ -0,0 +1,144 @@ +"""Tests for ``s3dol.diagnose()`` (ADR-0012 D6/D7). All tier 1.""" + +import io + +from s3dol._diagnose import _spec_from_v0_args, _v0_mangle, diagnose + + +def run_diagnose(**kwargs): + """diagnose() against a controlled environment, capturing the report.""" + kwargs.setdefault("environ", {}) + return diagnose(file=io.StringIO(), **kwargs) + + +# --------------------------------------------------------------------------- # +# never raises (D6) +# --------------------------------------------------------------------------- # + + +def test_reports_resolution_for_plain_v1_kwargs(): + report = run_diagnose(endpoint_url="http://localhost:9000") + assert "resolution:" in report + assert "http://localhost:9000" in report + assert "endpoint_url=" in report + assert "can_presign: True" in report + + +def test_a_failing_resolution_is_a_row_not_a_raise(): + report = run_diagnose(preset="minio") # C1: MissingEndpoint + assert "RESOLUTION FAILS" in report + assert "MissingEndpoint" in report + + +def test_a_failing_construction_is_a_row_not_a_raise(): + report = run_diagnose(anon=True, profile="prod") + assert "RESOLUTION FAILS" in report + assert "ConfigurationError" in report + + +def test_utter_garbage_never_raises(): + report = run_diagnose(bogus_kwarg=42) + assert isinstance(report, str) and report # something readable came back + + +def test_secret_values_never_echo(): + report = run_diagnose( + bucket_name="b", + aws_access_key_id="AKIAEXAMPLE", + aws_secret_access_key="sEkR3tVALUE", + ) + assert "sEkR3tVALUE" not in report + assert "" in report + + +def test_prints_to_the_given_file_and_returns_the_text(): + sink = io.StringIO() + text = diagnose(endpoint_url="http://x:1", environ={}, file=sink) + assert sink.getvalue() == text + + +# --------------------------------------------------------------------------- # +# the v0 mangler (D7): ~20 frozen lines mapping 1:1 onto v0 +# --------------------------------------------------------------------------- # + + +def test_mangle_endpoint_dropped_under_env_credentials(): + branch, effective = _v0_mangle( + {"endpoint_url": "http://r2:1", "aws_access_key_id": "EXPLICIT"}, + {"AWS_ACCESS_KEY_ID": "ENVKEY", "AWS_SECRET_ACCESS_KEY": "ENVSECRET"}, + ) + assert "endpoint dropped" in branch + assert effective["endpoint_url"] is None # the v0 data-misrouting bug + assert effective["aws_access_key_id"] == "ENVKEY" # env over explicit + assert effective["aws_session_token"] is None # nulled, silently + assert effective["region_name"] is None # nulled, silently + + +def test_mangle_explicit_branch_when_no_env_credentials(): + branch, effective = _v0_mangle( + {"endpoint_url": "http://r2:1", "aws_access_key_id": "AK"}, {} + ) + assert branch == "explicit" + assert effective["endpoint_url"] == "http://r2:1" # honoured on this path + + +def test_mangle_profile_with_explicit_creds_discards_profile(): + branch, _ = _v0_mangle( + {"profile_name": "prod", "aws_access_key_id": "AK"}, {} + ) + assert "discarded" in branch + + +def test_mangle_supabase_and_skip_branches_are_excluded(): + assert _v0_mangle({"endpoint_url": "https://x.supabase.co/s3"}, {})[1] is None + assert _v0_mangle({"is_supabase_endpoint": True}, {})[1] is None + assert _v0_mangle({"skip_bucket_check": True}, {})[1] is None + + +def test_spec_from_v0_args_maps_env_profile_to_the_chain(): + spec = _spec_from_v0_args({"profile_name": "environment variables"}) + assert spec.profile is None # env creds are the top of botocore's own chain + + +# --------------------------------------------------------------------------- # +# the divergence table +# --------------------------------------------------------------------------- # + + +def test_divergence_flags_the_endpoint_drop(): + report = run_diagnose( + bucket_name="b", + endpoint_url="http://r2.example:1", + aws_access_key_id="AK", + aws_secret_access_key="SK", + environ={"AWS_ACCESS_KEY_ID": "ENVK", "AWS_SECRET_ACCESS_KEY": "ENVS"}, + ) + assert "divergence" in report + assert "CHANGES ON UPGRADE" in report + assert "endpoint dropped" in report + + +def test_no_divergence_when_v0_and_v1_agree(): + report = run_diagnose(bucket_name="b", endpoint_url="http://minio:9000") + assert "no divergence" in report + + +def test_excluded_branch_prints_the_reason(): + report = run_diagnose(bucket_name="b", endpoint_url="https://x.supabase.co/s3") + assert "EXCLUDED" in report + assert "process-global" in report + + +def test_credential_identity_divergence_is_a_predicate_not_a_value(): + report = run_diagnose( + bucket_name="b", + aws_access_key_id="AKIAEXPLICIT", + aws_secret_access_key="sEkR3tVALUE", + environ={ + "AWS_ACCESS_KEY_ID": "AKIAFROMENV", + "AWS_SECRET_ACCESS_KEY": "ENVSEKRET99", + }, + ) + assert "credentials:" in report + assert "sEkR3tVALUE" not in report + assert "ENVSEKRET99" not in report # no env secret value either diff --git a/s3dol/tests/test_errors.py b/s3dol/tests/test_errors.py new file mode 100644 index 0000000..461a21a --- /dev/null +++ b/s3dol/tests/test_errors.py @@ -0,0 +1,311 @@ +"""Tier-1 tests for the error taxonomy and the classification seam (ADR-0004). + +Pure unit tests over *synthesized* ``ClientError``s — including body-less HEAD +errors, which moto cannot produce (it returns an error body on HEAD where real +AWS returns none), so tier 2 could never cover them. +""" + +import pytest + +from s3dol.errors import ( + AccessDenied, + BucketNotFound, + Classified, + ConfigurationError, + CredentialsError, + KeyNotValid, + ObjectArchived, + ObjectNotFound, + S3DolWarning, + S3Error, + S3PartialFailure, + classify_client_error, + code_of, + redact, + status_of, + translate_client_error, + translate_s3_errors, +) + + +def synthesized_client_error(code=None, status=None, operation="GetObject"): + """A ClientError like botocore raises, built with no network.""" + from botocore.exceptions import ClientError + + response = {"ResponseMetadata": {"HTTPStatusCode": status}} + if code is not None: + response["Error"] = {"Code": code, "Message": "synthesized"} + return ClientError(response, operation) + + +# --------------------------------------------------------------------------- # +# taxonomy shape +# --------------------------------------------------------------------------- # + + +def test_mapping_contract_errors_are_keyerrors(): + for exc_type in (ObjectNotFound, BucketNotFound, KeyNotValid, ObjectArchived): + assert issubclass(exc_type, KeyError) + assert issubclass(exc_type, S3Error) + + +def test_key_not_valid_is_also_a_valueerror(): + # Deliberately both, so either `except` works (ADR-0004 §5). + assert issubclass(KeyNotValid, ValueError) + + +def test_auth_errors_are_never_keyerrors(): + # A missing permission must not look like a missing key (ADR-0004 §1). + for exc_type in (AccessDenied, CredentialsError): + assert not issubclass(exc_type, KeyError) + assert issubclass(exc_type, S3Error) + + +def test_configuration_error_is_a_valueerror(): + assert issubclass(ConfigurationError, ValueError) + + +def test_except_s3error_catches_the_package(): + with pytest.raises(S3Error): + raise ObjectNotFound("k") + with pytest.raises(S3Error): + raise AccessDenied("nope") + + +def test_keyerror_str_is_not_repr_mangled(): + message = "line one\nline two: helpful detail" + assert str(ObjectNotFound(message)) == message # stdlib KeyError would repr() + assert "\\n" not in str(BucketNotFound(message)) + + +def test_partial_failure_carries_the_split(): + error = S3PartialFailure( + "3 of 5 deleted", succeeded=["a", "b", "c"], failures={"d": AccessDenied("x")} + ) + assert error.succeeded == ["a", "b", "c"] + assert set(error.failures) == {"d"} + + +def test_object_archived_carries_restore_metadata(): + error = ObjectArchived("archived", storage_class="GLACIER", restore_status=None) + assert error.storage_class == "GLACIER" + with pytest.raises(S3Error): + error.restore() # nothing bound -> loud, not silent + + +def test_warning_tree_rooted_in_s3dolwarning(): + from s3dol.errors import ( + AmbiguousResolution, + AnonymousFallback, + S3DolResolutionChanged, + ) + + for warning_class in (S3DolResolutionChanged, AmbiguousResolution, AnonymousFallback): + assert issubclass(warning_class, S3DolWarning) + assert issubclass(warning_class, UserWarning) + + +# --------------------------------------------------------------------------- # +# classification +# --------------------------------------------------------------------------- # + + +@pytest.mark.parametrize( + "operation, code, status, kind", + [ + ("GetObject", "NoSuchKey", 404, "object_absent"), + ("GetObject", "NoSuchBucket", 404, "bucket_absent"), + # HEAD has no body: the code is the synthesized status string. + ("HeadObject", "404", 404, "ambiguous_head"), + ("HeadBucket", "404", 404, "bucket_absent"), + ("HeadObject", "404", None, "ambiguous_head"), + # delete-marker HEAD on a versioned bucket + ("HeadObject", "405", 405, "delete_marker"), + # wrong region + ("GetObject", "PermanentRedirect", 301, "wrong_region"), + ("ListObjectsV2", None, 301, "wrong_region"), + # archived + ("GetObject", "InvalidObjectState", 403, "archived"), + # credentials + ("GetObject", "ExpiredToken", 400, "credentials"), + ("GetObject", "InvalidAccessKeyId", 403, "credentials"), + ("PutObject", "SignatureDoesNotMatch", 403, "credentials"), + # permissions + ("GetObject", "AccessDenied", 403, "access_denied"), + ("HeadObject", "403", 403, "access_denied"), + # transient + ("GetObject", "SlowDown", 503, "transient"), + ("PutObject", None, 503, "transient"), + # unknown propagates + ("GetObject", "TotallyNewCode", 418, "unknown"), + ], +) +def test_classification_table(operation, code, status, kind): + assert classify_client_error(operation, code, status).kind == kind + + +def test_provider_overrides_are_consulted_first(): + # Supabase returns 400 where AWS returns 404 on HeadBucket (ADR-0004 §2). + overrides = ((("HeadBucket", None, 400), "bucket_absent"),) + assert ( + classify_client_error("HeadBucket", "400", 400, overrides=overrides).kind + == "bucket_absent" + ) + # ...and only for the operation they name. + assert ( + classify_client_error("GetObject", "400", 400, overrides=overrides).kind + == "unknown" + ) + + +def test_classified_is_a_value(): + assert Classified(kind="object_absent", exception_type=ObjectNotFound).kind == ( + "object_absent" + ) + + +def test_code_and_status_extractors_are_total(): + error = synthesized_client_error("NoSuchKey", 404) + assert code_of(error) == "NoSuchKey" + assert status_of(error) == 404 + assert code_of(object()) is None + assert status_of(None) is None + + +# --------------------------------------------------------------------------- # +# translation +# --------------------------------------------------------------------------- # + + +def test_translate_object_absent(): + error = synthesized_client_error("NoSuchKey", 404) + replacement = translate_client_error( + error, operation="GetObject", bucket="b", key="k", endpoint="http://e" + ) + assert isinstance(replacement, ObjectNotFound) + for fragment in ("GetObject", "'b'", "'k'", "http://e"): + assert fragment in str(replacement) + + +def test_translate_ambiguous_head_with_resolver(): + error = synthesized_client_error("404", 404, operation="HeadObject") + bucket_reachable = translate_client_error( + error, operation="HeadObject", key="k", head_ambiguity_resolver=lambda: True + ) + assert isinstance(bucket_reachable, ObjectNotFound) + bucket_missing = translate_client_error( + error, operation="HeadObject", key="k", head_ambiguity_resolver=lambda: False + ) + assert isinstance(bucket_missing, BucketNotFound) + + +def test_translate_access_denied_names_the_policy_cause(): + error = synthesized_client_error("AccessDenied", 403) + replacement = translate_client_error(error, operation="HeadObject", key="k") + assert isinstance(replacement, AccessDenied) + assert "s3:ListBucket" in str(replacement) + assert "deny_means_absent" in str(replacement) + + +def test_deny_means_absent_opt_in(): + error = synthesized_client_error("AccessDenied", 403) + replacement = translate_client_error( + error, operation="HeadObject", key="k", deny_means_absent=True + ) + assert isinstance(replacement, ObjectNotFound) + + +def test_transient_and_unknown_propagate(): + assert ( + translate_client_error( + synthesized_client_error("SlowDown", 503), operation="GetObject" + ) + is None + ) + assert ( + translate_client_error( + synthesized_client_error("BrandNew", 418), operation="GetObject" + ) + is None + ) + + +# --------------------------------------------------------------------------- # +# the decorator seam +# --------------------------------------------------------------------------- # + + +class _FakeStore: + def __init__(self, error): + self._error = error + + def _error_context(self): + return dict(bucket="b", endpoint="http://e") + + @translate_s3_errors(operation="GetObject", key_arg="k") + def read(self, k): + raise self._error + + @translate_s3_errors(operation="GetObject", key_arg="k") + def ok(self, k): + return k.upper() + + +def test_decorator_translates_and_carries_the_key(): + store = _FakeStore(synthesized_client_error("NoSuchKey", 404)) + with pytest.raises(ObjectNotFound) as info: + store.read("my-key") + assert "my-key" in str(info.value) + assert "'b'" in str(info.value) + # the botocore original is chained, not swallowed + assert info.value.__cause__ is not None + + +def test_decorator_extracts_key_passed_by_keyword(): + store = _FakeStore(synthesized_client_error("NoSuchKey", 404)) + with pytest.raises(ObjectNotFound) as info: + store.read(k="kw-key") + assert "kw-key" in str(info.value) + + +def test_decorator_propagates_unknown_errors_untouched(): + original = synthesized_client_error("BrandNewCode", 418) + store = _FakeStore(original) + with pytest.raises(type(original)) as info: + store.read("k") + assert info.value is original + + +def test_decorator_translates_missing_credentials(): + from botocore.exceptions import NoCredentialsError + + store = _FakeStore(NoCredentialsError()) + with pytest.raises(CredentialsError) as info: + store.read("k") + assert "anon=True" in str(info.value) + + +def test_decorator_leaves_success_alone(): + store = _FakeStore(None) + assert store.ok("abc") == "ABC" + + +# --------------------------------------------------------------------------- # +# redaction +# --------------------------------------------------------------------------- # + + +@pytest.mark.parametrize( + "secret_param", + ["X-Amz-Signature", "X-Amz-Credential", "X-Amz-Security-Token", "AWSAccessKeyId"], +) +def test_redact_presigned_url_secrets(secret_param): + url = f"https://b.s3.amazonaws.com/k?{secret_param}=SECRETVALUE&keep=1" + redacted = redact(url) + assert "SECRETVALUE" not in redacted + assert "keep=1" in redacted + + +def test_redact_is_case_insensitive_and_leaves_plain_text_alone(): + assert redact("nothing secret here") == "nothing secret here" + assert "hunter2" not in redact("?x-amz-signature=hunter2") diff --git a/s3dol/tests/test_presets.py b/s3dol/tests/test_presets.py new file mode 100644 index 0000000..23199dc --- /dev/null +++ b/s3dol/tests/test_presets.py @@ -0,0 +1,226 @@ +"""Tier-1 tests for the preset registry (ADR-0003 as amended by ADR-0012 D4).""" + +import pytest + +from s3dol.errors import ConfigurationError, InvalidEndpoint, UnknownPresetParam +from s3dol.presets import ( + CONFIG_KWARGS_ALLOWLIST, + Capabilities, + Preset, + _REGISTRY, + available_presets, + detect_preset, + get_preset, + register, + unregister, + validate_endpoint_hygiene, +) + + +@pytest.fixture +def scratch_registry(): + """Snapshot/restore the registry so tests can register freely.""" + snapshot = dict(_REGISTRY) + yield + _REGISTRY.clear() + _REGISTRY.update(snapshot) + + +# --------------------------------------------------------------------------- # +# row validation +# --------------------------------------------------------------------------- # + + +def test_addressing_style_is_mandatory_and_auto_is_illegal(): + with pytest.raises(TypeError): + Preset(name="x") # no default: every row must state it + with pytest.raises(ConfigurationError, match="auto"): + Preset(name="x", addressing_style="auto") + + +@pytest.mark.parametrize( + "bad_url", + [ + "https://user:pw@s3.example.com", # userinfo: the credential carrier + "https://s3.example.com?token=abc", # query + "https://s3.example.com#frag", # fragment + ], +) +def test_endpoint_hygiene(bad_url): + with pytest.raises(InvalidEndpoint): + validate_endpoint_hygiene(bad_url) + with pytest.raises(InvalidEndpoint): + Preset(name="x", addressing_style="path", endpoint_url=bad_url) + + +def test_config_kwargs_allowlist_is_closed(): + with pytest.raises(ConfigurationError, match="allowlisted"): + Preset( + name="x", + addressing_style="path", + config_kwargs=(("proxies", "http://user:pw@proxy"),), + ) + with pytest.raises(ConfigurationError, match="scalar"): + Preset( + name="x", + addressing_style="path", + config_kwargs=(("connect_timeout", {"nested": 1}),), + ) + ok = Preset( + name="x", addressing_style="path", config_kwargs=(("connect_timeout", 5),) + ) + assert dict(ok.config_kwargs)["connect_timeout"] == 5 + assert "proxies" not in CONFIG_KWARGS_ALLOWLIST + + +def test_unknown_bound_param_raises(): + with pytest.raises(UnknownPresetParam): + Preset( + name="x", + addressing_style="path", + endpoint_url="https://{a}.example.com", + params=(("b", "1"),), + ) + + +# --------------------------------------------------------------------------- # +# template binding (C1's raw material) +# --------------------------------------------------------------------------- # + + +def test_bind_and_bound_endpoint(): + r2 = get_preset("r2") + assert r2.bound_endpoint() is None # unbound template -> no rung + assert r2.missing_endpoint_params() == {"account_id"} + bound = r2.bind(account_id="abc123") + assert bound.bound_endpoint() == "https://abc123.r2.cloudflarestorage.com" + assert bound.missing_endpoint_params() == set() + + +def test_bind_unknown_param_raises(): + with pytest.raises(UnknownPresetParam, match="account_id"): + get_preset("r2").bind(nonsense="x") + + +def test_region_kwarg_binds_region_template(): + scaleway = get_preset("scaleway") + assert scaleway.bound_endpoint() is None + assert scaleway.bound_endpoint(region_name="fr-par") == ( + "https://s3.fr-par.scw.cloud" + ) + + +# --------------------------------------------------------------------------- # +# pinned is derived, gated on verified (dormant in v1.0) +# --------------------------------------------------------------------------- # + + +def test_pinned_is_dormant_while_doc_sourced(): + row = Preset( + name="x", addressing_style="path", region_name="eu-west-1", verified=False + ) + assert row.pinned is False + verified = Preset( + name="x", addressing_style="path", region_name="eu-west-1", verified=True + ) + assert verified.pinned is True + + +def test_templated_region_never_pins(): + row = Preset( + name="x", addressing_style="path", region_name="{location}", verified=True + ) + assert row.pinned is False + + +# --------------------------------------------------------------------------- # +# registry +# --------------------------------------------------------------------------- # + + +def test_register_refuses_silent_overwrite(scratch_registry): + row = Preset(name="mine", addressing_style="path") + register(row) + with pytest.raises(ConfigurationError, match="already registered"): + register(row) + register(row, overwrite=True) # explicit is fine + unregister("mine") + assert "mine" not in available_presets() + + +def test_get_preset_unknown_names_the_registry(): + with pytest.raises(ConfigurationError) as info: + get_preset("no-such-provider") + assert "register" in str(info.value).lower() + assert "aws" in str(info.value) # the row list is in the message + + +# --------------------------------------------------------------------------- # +# detection (resolution pass 2) +# --------------------------------------------------------------------------- # + + +def test_detection_by_host_pattern(): + detected = detect_preset("https://abc123.r2.cloudflarestorage.com") + assert detected is not None and detected.name == "r2" + assert detect_preset("https://storage.googleapis.com").name == "gcs" + assert detect_preset("https://myref.storage.supabase.co/storage/v1/s3").name == ( + "supabase" + ) + + +def test_soft_rows_never_win_detection(): + # aws is a soft fallback; detection yielding None routes the caller to the + # generic-s3/aws fallback logic instead. + assert detect_preset("https://s3.eu-central-1.amazonaws.com") is None + assert detect_preset("https://unknown.example.com") is None + + +def test_ambiguous_detection_raises(scratch_registry): + register( + Preset( + name="clone-a", addressing_style="path", host_patterns=("*.clone.test",) + ) + ) + register( + Preset( + name="clone-b", addressing_style="path", host_patterns=("*.clone.test",) + ) + ) + with pytest.raises(ConfigurationError, match="more than one preset"): + detect_preset("https://x.clone.test") + + +# --------------------------------------------------------------------------- # +# builtin rows sanity +# --------------------------------------------------------------------------- # + + +def test_every_builtin_row_is_well_formed(): + for name in available_presets(): + row = get_preset(name) + assert row.addressing_style in ("path", "virtual") + assert row.verified is False # doc-sourced until proven live + + +def test_the_load_bearing_rows(): + assert get_preset("backblaze").checksum == "when_required" # mandatory + assert get_preset("supabase").checksum == "when_required" # the §3 fix + assert get_preset("supabase").error_overrides # 400-means-404 + assert get_preset("hetzner").payload_signing_enabled is False + assert get_preset("hetzner").presign_addressing_style == "virtual" + assert get_preset("scaleway").capabilities.max_multipart_parts == 1000 + gcs = get_preset("gcs").capabilities + assert gcs.list_objects_v2 is False and gcs.batch_delete is False + assert get_preset("minio").requires_endpoint is True + assert get_preset("ceph").requires_endpoint is True + assert get_preset("r2").region_name == "auto" # R2's literal region string + for soft_name in ("aws", "generic-s3"): + assert get_preset(soft_name).soft is True + + +def test_capabilities_defaults_are_aws_reference_semantics(): + caps = Capabilities() + assert caps.list_objects_v2 and caps.batch_delete and caps.multipart + assert caps.max_multipart_parts == 10_000 + assert caps.consistency == "strong" From bc095cf298236036b1267b582b00d35cdc031385 Mon Sep 17 00:00:00 2001 From: Thor Whalen <1906276+thorwhalen@users.noreply.github.com> Date: Tue, 11 Aug 2026 18:13:48 +0100 Subject: [PATCH 2/3] =?UTF-8?q?feat:=20s3dol=20v1=20=E2=80=94=20Layer=20B/?= =?UTF-8?q?C,=20sibling=20capability=20stores,=20compat=20shim,=20testing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements the v1 surface on top of Layer A (#19), per ADRs 0001-0012. Layer B (base.py): BucketCollection/Reader/Store and Endpoint* triads + ObjectHandle. The leaf owns a normalized prefix (pushed into ListObjectsV2, _key_of_id RAISES rather than slicing a non-matching id), key validity is checked before the wire, __len__ refuses with guidance, __delitem__ is idempotent (strict_delete opts into the probe), __contains__ resolves the body-less HEAD ambiguity with one cached HeadBucket, on_missing_bucket defaults to 'assume' (no probe, ever) and 'create' recovers rather than probes, and setdefault/pop re-raise ObjectArchived instead of overwriting it. Backend access goes through a module-internal _op_* protocol (ADR-0002's seam), which is what lets the shipped fake BE the real store. ADR-0011: no key-taking public methods. Keyed capabilities are sibling stores (BucketHandles/BucketUrls/BucketInfo, derived via s3dol.handles/urls/info, which refuse a wrapped store and name the parallel-wrap remedy); non-keyed operations are free functions (sub/prefixes/delete_many/delete_bucket) with the outward key mapping dol does not provide. A reflective conformance test (D5) enumerates every public attribute against an explicit inventory. **ADR-0011 D3b amended** (recorded in the ADR, not just here): the guarded url_for shim cannot serve both its call paths — dol.content_url hands over a key already mapped through the outer layers while a direct delegated call hands over the raw outer key, and applying _id_of_key is right for one and double-transforms the other (measured: p/x/x/g). It now applies dol's stated contract and REFUSES once the leaf has been wrapped, via a sticky probe flag that (unlike the wrapped_self weakref) survives the wrapper's death. This is the P1 evidence for dol#86/#83: a keyed capability method is unfixable in the general case, because the ambiguity is in the call, not the resolution. Also: values.py (the closed write domain, str rejected on decidability, as_fileobj registered on io.IOBase not typing.BinaryIO), writes/reads strategies as picklable frozen dataclasses, recipes.py (s3_store + codec facades built by composition only), store.py compat shim preserving the three behaviours dependents' passing tests rely on plus a corrected full-parse-or-passthrough de-chunker and the D7 migration warning, and s3dol.testing shipping an in-memory fake + the exported conformance suite. Adversarial review (4 lenses, 39 findings) fixed in-branch: diagnose echoed v1 credential kwargs verbatim; env-endpoint userinfo leaked through C2 messages, warnings, Resolution repr and diagnose; client_kwargs was an unpoliced credential carrier; hygiene refusals echoed the URL they refuse; (k, s, None) became the truthy token 'None'; signature_version=None silently re-enabled SigV2; the C2 guard exempted named soft rows; deny_means_absent swallowed clock-skew 403s; the divergence table both cried wolf and missed the real identity move; region capture hijacked AWS_DEFAULT_REGION; CallableCredentials double-fetched; preset error_overrides kinds were unvalidated; ambiguous HEAD guessed with no resolver bound; diagnose read the developer's real ~/.aws in "tier 1" tests; a module-level importorskip skipped 60 pure tests. Regression tests added for each. dol floor >= 0.3.63 (ADR-0006 §3 policy). 299 tests + doctests green; CI now runs 3.10 + 3.12 with --doctest-modules. Refs #11, closes #10. Claude-Session: https://claude.ai/code/session_01KqivR5Be21Wh7yRdEG4fiY --- README.md | 57 + conftest.py | 9 +- .../0011-keyed-capability-surface.md | 33 + misc/docs/state-of-play.md | 15 +- pyproject.toml | 16 +- s3dol/__init__.py | 119 +- s3dol/_diagnose.py | 143 ++- s3dol/base.py | 1144 +++++++++++++---- s3dol/capabilities.py | 396 ++++++ s3dol/connection.py | 243 +++- s3dol/errors.py | 98 +- s3dol/presets.py | 34 +- s3dol/reads.py | 62 + s3dol/recipes.py | 138 ++ s3dol/store.py | 459 ++++--- s3dol/testing.py | 426 ++++++ s3dol/tests/test_base.py | 523 +++++--- s3dol/tests/test_capabilities.py | 221 ++++ s3dol/tests/test_conformance.py | 114 ++ s3dol/tests/test_connection.py | 193 ++- s3dol/tests/test_diagnose.py | 87 +- s3dol/tests/test_keyed_surface.py | 153 +++ s3dol/tests/test_store.py | 47 - s3dol/tests/test_store_shim.py | 237 ++++ s3dol/tests/test_url_for.py | 61 - s3dol/tests/test_values.py | 90 ++ s3dol/utility.py | 127 -- s3dol/values.py | 204 +++ s3dol/writes.py | 99 ++ 29 files changed, 4617 insertions(+), 931 deletions(-) create mode 100644 s3dol/capabilities.py create mode 100644 s3dol/reads.py create mode 100644 s3dol/recipes.py create mode 100644 s3dol/testing.py create mode 100644 s3dol/tests/test_capabilities.py create mode 100644 s3dol/tests/test_conformance.py create mode 100644 s3dol/tests/test_keyed_surface.py delete mode 100644 s3dol/tests/test_store.py create mode 100644 s3dol/tests/test_store_shim.py delete mode 100644 s3dol/tests/test_url_for.py create mode 100644 s3dol/tests/test_values.py delete mode 100644 s3dol/utility.py create mode 100644 s3dol/values.py create mode 100644 s3dol/writes.py diff --git a/README.md b/README.md index e58eb99..72881ba 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,63 @@ To install: ```pip install s3dol``` [Documentation](https://i2mint.github.io/s3dol/) +## Quick start + +```python +import s3dol + +s = s3dol.s3_store('my-bucket') # MutableMapping[str, bytes] +s['hello.txt'] = b'world' +s['hello.txt'] # b'world' +list(s); 'hello.txt' in s; del s['hello.txt'] + +s = s3dol.s3_store('my-bucket', prefix='logs/') # scoped to a prefix +s = s3dol.s3_store('open-data', anon=True) # public bucket, no credentials +s = s3dol.s3_store('b', preset='minio', endpoint_url='http://localhost:9000') +``` + +Starting from nothing (the default never creates a bucket from a typo): + +```python +s = s3dol.s3_store('brand-new-bucket', on_missing_bucket='create') +``` + +**Big objects go through the same interface** — no `upload_multipart` method: + +```python +s['video.mp4'] = s3dol.Filepath('/tmp/video.mp4') # streamed, multipart above 8 MiB +s['stream.bin'] = s3dol.Chunks(chunk_iterator) # never fully in memory +``` + +**Keyed capabilities are stores you index**, not methods (a `dol` key wrapper +hands a method the *unmapped* key — so s3dol has none): + +```python +s3dol.handles(s)['video.mp4'].read(offset=0, length=1024) # ranged read +s3dol.handles(s)['video.mp4'].url(expires_in=3600) # presigned URL +s3dol.urls(s)['video.mp4'] # ...or directly +s3dol.info(s)['video.mp4'].size # one HeadObject +``` + +Everything else is a free function taking the store first: + +```python +s3dol.sub(s, 'folder/'); s3dol.prefixes(s) +s3dol.delete_many(s, ['a', 'b']); s3dol.delete_bucket(endpoint, 'name', force=True) +``` + +**Test without a cloud** — no network, no docker, no moto: + +```python +from s3dol.testing import mock_s3, run_conformance + +def test_my_service(): + assert MediaService(mock_s3(data={'a.mp4': b'...'})).play('a.mp4') +``` + +`run_conformance(make_store)` is exported too: run the same laws against your +own store (or a sibling `*dol` package's). + ## Heads-up: v1 is coming — run `s3dol.diagnose()` now A major, behaviour-clarifying v1 is in progress diff --git a/conftest.py b/conftest.py index 1b661d5..b231c0b 100644 --- a/conftest.py +++ b/conftest.py @@ -33,11 +33,18 @@ _S3_FREE_TEST_STEMS = frozenset( { "test_url_for", - # Layer A (v1) suites: tier-1 pure + moto-mocked tier-2 — no endpoint. + # v1 suites: tier-1 pure + moto-mocked tier-2 — no live endpoint. "test_errors", "test_presets", "test_connection", "test_diagnose", + "test_values", + "test_keyed_surface", + "test_conformance", + "test_base", + "test_capabilities", + "test_store_shim", + "test_recipes", } ) diff --git a/misc/docs/decisions/0011-keyed-capability-surface.md b/misc/docs/decisions/0011-keyed-capability-surface.md index c108770..7b21606 100644 --- a/misc/docs/decisions/0011-keyed-capability-surface.md +++ b/misc/docs/decisions/0011-keyed-capability-surface.md @@ -227,6 +227,39 @@ documented limitation, and the real fix is upstream: `content_url` must resolve chain **and** call the innermost `url_for` with the resolved key — resolving without that second half would double-transform. +#### D3b-amended (2026-08-11, during implementation) — the guarded method cannot serve both call paths + +The guard above assumed one call path. There are **two**, and they hand the *same signature* +**different key domains**: + +| caller | what `url_for` receives | correct action | +|---|---|---| +| `dol.content_url(wrapped, k)` | the key already mapped through the **outer** layers — because the walk stops at the layer owning `url_for` "*because that layer applies its own*" (`dol/content.py::_url_for_provider_and_key`) | apply `self._id_of_key` | +| `wrapped.url_for(k)` (delegation) | the raw **outer** key | resolve through the whole chain | + +Applying `_id_of_key` is right for the first and wrong for the second; re-resolving through the +chain is right for the second and **double-transforms** the first (measured: a store scoped to +`p/` under a `x/` key codec produced `p/x/x/g`). Nothing at the callee distinguishes them — +this is dol#83's defect reaching the one method the ADR kept. + +**Decision:** `url_for` applies `self._id_of_key(k)` (dol's stated contract), and **refuses** +— `NotSupported`, naming `s3dol.urls(store)[k]` — as soon as the leaf has ever been wrapped. +Consequences, stated plainly: + +- `dol.content_url` is correct for an **unwrapped** s3dol store (the common case, and the one + `dol/content.py`'s docstring shows) and **raises** for a wrapped one. Loud, never wrong. +- The refusal is driven by a *sticky flag* set when dol probes the leaf (`hasattr(store, + 'KeysView')` in `Store.__init__`), implemented as a descriptor because a `KvReader` leaf + would otherwise satisfy the probe from `MappingViewMixin` and never see it. Unlike the + `wrapped_self` weakref, the flag survives the wrapper's death — so the refusal is reliable, + not intermittent. The documented false positive stands: a once-wrapped leaf later used bare + also refuses. + +**This is the P1 evidence for dol#86/#83**: a keyed capability *method* is unfixable in the +general case — not merely fragile, and not fixable by a better resolution primitive, because +the ambiguity is in the *call*, not the resolution. Sibling stores (D2) route through +`__getitem__`, which has one key domain and one caller, and so remain correct by construction. + ### D4 — Destructive operations are free functions `delete_many(keys)` and `EndpointStore.delete(name, force=True)` both leave Layer B. diff --git a/misc/docs/state-of-play.md b/misc/docs/state-of-play.md index 99a70b6..b5782fa 100644 --- a/misc/docs/state-of-play.md +++ b/misc/docs/state-of-play.md @@ -67,7 +67,20 @@ consumer's requirements. | Resolved design questions | **§7** / **#14** → [ADR-0011](decisions/0011-keyed-capability-surface.md) · **§8** / **#15** → [ADR-0012](decisions/0012-credential-and-endpoint-resolution.md) | | Open design questions | none — implementation is unblocked; see §11 | -**Nothing has been implemented.** The repo still ships v0.1.9 unchanged. +**Implementation is under way** (2026-08-11). Layer A landed as an additive +0.1.x change (PR #19: `errors.py`, `presets.py`, `connection.py`, `_diagnose.py` +— ADR-0007 §5 step 0, so dependents can record what resolves *before* anything +moves), and the v1 surface followed on top (`values`/`writes`/`reads`/`base`/ +`capabilities`/`recipes`/`testing` + the `store.py` compat shim). Read §11 for +what remains. + +One design amendment came out of implementation and is recorded where it +belongs, in [ADR-0011 §D3b-amended](decisions/0011-keyed-capability-surface.md): +the guarded `url_for` shim **cannot** serve both of its call paths, because +`dol.content_url` and a direct delegated call hand the same signature different +key domains. It now refuses once the leaf has been wrapped. That is the P1 +evidence for dol#86/#83 — a keyed capability *method* is unfixable in the +general case, and not for want of a better resolution primitive. ## 3. The twelve ADRs, in one paragraph each diff --git a/pyproject.toml b/pyproject.toml index 6d64469..e763438 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,12 @@ classifiers = [ "Topic :: Software Development :: Libraries :: Python Modules", ] dependencies = [ - "dol", + # A hard floor, not a preference (ADR-0006 §3's upstream policy: raise the + # floor the day each fix lands, and delete the local workaround in the same + # commit). v1 needs dol.inner_most_key exported (0.3.62), content_url + # resolving keys through the wrapper chain (0.3.62), and filesys + # is_valid_key fixed — all shipped by 0.3.63. + "dol>=0.3.63", "boto3", # ADR-0012: the endpoint env vars (AWS_ENDPOINT_URL_S3 / AWS_ENDPOINT_URL) do not exist # before botocore 1.31, and ADR-0003 §3's checksum knobs @@ -49,12 +54,17 @@ test = [ "pytest>=7.0", "pytest-cov>=4.0", "moto[s3]>=5.0", + # conformance assertion 2b fetches a presigned URL; moto patches + # `requests` but NOT urllib (which would leave the machine). Declared + # explicitly rather than relied on transitively via moto[s3]. + "requests", ] dev = [ "pytest>=7.0", "pytest-cov>=4.0", "ruff>=0.1.0", "moto[s3]>=5.0", + "requests", ] docs = [ "sphinx>=6.0", @@ -144,10 +154,14 @@ enabled = false [tool.wads.ci.testing] python_versions = [ "3.10", + "3.12", ] pytest_args = [ "-v", "--tb=short", + # Every documented example is executable against the in-memory fake, so + # doctests run in the default gate (ADR-0008). + "--doctest-modules", ] coverage_enabled = true coverage_threshold = 0 diff --git a/s3dol/__init__.py b/s3dol/__init__.py index edae93b..d957e00 100644 --- a/s3dol/__init__.py +++ b/s3dol/__init__.py @@ -1,26 +1,117 @@ -""" -s3 (through boto3) with a simple (dict-like or list-like) interface -""" +"""s3dol — S3 and S3-compatible object storage behind a dict-like interface. + +The one-liner:: + + import s3dol + s = s3dol.s3_store('my-bucket') # MutableMapping[str, bytes] + s['k'] = b'v'; s['k']; list(s); del s['k'] + +Keyed capabilities are **sibling stores you index** (never methods — a dol key +wrapper would hand a method the unmapped key; ADR-0011):: + + s3dol.handles(s)['k'] # ObjectHandle (ranged reads, streams, metadata) + s3dol.urls(s)['k'] # presigned URL + s3dol.info(s)['k'] # ObjectInfo + +Non-keyed operations are free functions taking the store first:: + + s3dol.sub(s, 'folder/') s3dol.prefixes(s) + s3dol.delete_many(s, keys) s3dol.delete_bucket(endpoint, name, force=True) -from s3dol.base import S3Dol -from s3dol.store import S3Store +Test without a cloud: ``from s3dol.testing import mock_s3, run_conformance``. +Migrating from v0? ``s3dol.diagnose(**your_S3Store_kwargs)`` prints what +resolves, from where, and whether v1 moves it. The legacy ``S3Store`` keeps +working via ``s3dol.store`` (deprecated, removed in v2). -# New (v1) layer-A names, loaded lazily (PEP 562) so `import s3dol` does not -# pay for what a script does not use. `s3dol.diagnose(...)` is the ADR-0007 §5 -# step-0 migration safety mechanism — run it with your current S3Store kwargs -# to see what resolves, from where, and whether the upcoming v1 moves it. +Everything here loads lazily (PEP 562): ``import s3dol`` does not import boto3. +""" + +# name -> module path (attribute of the same name), or (module path, attribute) _LAZY_ATTRIBUTES = { - "diagnose": ("s3dol._diagnose", "diagnose"), + # Layer C — recipes + "s3_store": "s3dol.recipes", + "S3Texts": "s3dol.recipes", + "S3Jsons": "s3dol.recipes", + "S3Pickles": "s3dol.recipes", + # Layer B — stores and the handle + "BucketCollection": "s3dol.base", + "BucketReader": "s3dol.base", + "BucketStore": "s3dol.base", + "EndpointCollection": "s3dol.base", + "EndpointReader": "s3dol.base", + "EndpointStore": "s3dol.base", + "ObjectHandle": "s3dol.base", + "ObjectInfo": "s3dol.base", + "S3Profiles": "s3dol.base", + # sibling capability stores + accessors (ADR-0011 D2) + "BucketHandles": "s3dol.capabilities", + "BucketUrls": "s3dol.capabilities", + "BucketInfo": "s3dol.capabilities", + "handles": "s3dol.capabilities", + "urls": "s3dol.capabilities", + "info": "s3dol.capabilities", + # free functions (ADR-0011 D3/D4) + "sub": "s3dol.capabilities", + "prefixes": "s3dol.capabilities", + "delete_many": "s3dol.capabilities", + "delete_bucket": "s3dol.capabilities", + # value refs (ADR-0005) + "Filepath": "s3dol.values", + "Chunks": "s3dol.values", + "Streamable": "s3dol.values", + # Layer A — connection + presets "S3Connection": "s3dol.connection", "resolve": "s3dol.connection", "Resolution": "s3dol.connection", + "StaticCredentials": "s3dol.connection", + "ProfileCredentials": "s3dol.connection", + "CallableCredentials": "s3dol.connection", "Preset": "s3dol.presets", "Capabilities": "s3dol.presets", "register_preset": ("s3dol.presets", "register"), + # errors (the taxonomy — `except s3dol.S3Error` catches the package) + "S3Error": "s3dol.errors", + "ObjectNotFound": "s3dol.errors", + "BucketNotFound": "s3dol.errors", + "KeyNotValid": "s3dol.errors", + "ObjectArchived": "s3dol.errors", + "AccessDenied": "s3dol.errors", + "CredentialsError": "s3dol.errors", + "NotSupported": "s3dol.errors", + "BucketNotEmpty": "s3dol.errors", + "S3PartialFailure": "s3dol.errors", + "ConfigurationError": "s3dol.errors", + "S3DolWarning": "s3dol.errors", + # diagnostics + "diagnose": ("s3dol._diagnose", "diagnose"), + # legacy (deprecated; removed in v2) + "S3Store": "s3dol.store", } +__all__ = sorted(_LAZY_ATTRIBUTES) + ["S3Dol", "__version__"] + def __getattr__(name): + if name == "__version__": + import importlib.metadata + + try: + return importlib.metadata.version("s3dol") + except importlib.metadata.PackageNotFoundError: + return "unknown" + if name == "S3Dol": + # v0's profiles->buckets->keys explorer, renamed (ADR-0007 §1). + import warnings + + warnings.warn( + "s3dol.S3Dol is deprecated (removed in v2): it is now " + "s3dol.S3Profiles — its keys are AWS profile names.", + DeprecationWarning, + stacklevel=2, + ) + from s3dol.base import S3Profiles + + return S3Profiles try: target = _LAZY_ATTRIBUTES[name] except KeyError: @@ -33,12 +124,12 @@ def __getattr__(name): module_path, attribute = target, name value = getattr(importlib.import_module(module_path), attribute) # Cache the resolved attribute. This ALSO deliberately overrides the - # submodule binding that `import s3dol.diagnose` just created on this - # package — without it, `s3dol.diagnose` would resolve to the module on - # every access after the first, shadowing the function. + # submodule binding a same-named submodule import creates on this package + # (e.g. `info` vs a hypothetical info module) — without it, the second + # access would resolve to the module and shadow the function. globals()[name] = value return value def __dir__(): - return sorted(set(globals()) | set(_LAZY_ATTRIBUTES)) + return sorted(set(globals()) | set(__all__)) diff --git a/s3dol/_diagnose.py b/s3dol/_diagnose.py index 32f335e..fbd84a8 100644 --- a/s3dol/_diagnose.py +++ b/s3dol/_diagnose.py @@ -26,7 +26,7 @@ import io import re import sys -from typing import Mapping, Optional +from typing import Mapping, Optional, Union from s3dol.connection import S3Connection, Resolution, load_aws_config, resolve from s3dol.errors import redact @@ -49,13 +49,48 @@ } ) -_SECRET_ARG_NAMES = frozenset({"aws_secret_access_key", "aws_session_token"}) +#: Argument names whose VALUE is safe to echo. An **allowlist**, not a +#: denylist: a denylist of the two v0 secret names let every v1 shape through, +#: so ``credentials=('AK', 'SECRET')`` printed the secret verbatim into the one +#: artifact users are told to run and share (ADR-0012 D6: diagnose holds no +#: credential value, only a provenance label). +_ECHOABLE_ARG_NAMES = frozenset( + { + "bucket_name", + "path", + "prefix", + "region_name", + "profile_name", + "profile", + "preset", + "make_bucket", + "skip_bucket_check", + "is_supabase_endpoint", + "on_missing_bucket", + "anon", + "addressing_style", + "checksum", + "payload_signing_enabled", + "deny_means_absent", + "signature_version", + } +) + +#: Echoed, but redacted first (a URL is the residual credential carrier). +_REDACTED_ARG_NAMES = frozenset({"endpoint_url"}) #: Reasons a v0 branch is excluded from the differential (ADR-0012 D7): these #: two branches build clients through the module-level ``boto3.client(...)``, #: which resolves through the process-global cached DEFAULT_SESSION — they are #: not functions of their arguments. _EXCLUDED_BRANCHES = { + "env-profile (v0 RAISES: no env credentials)": ( + "v0 raises S3DolException('Missing AWS credentials in environment " + "variables') for this call. v1 instead falls through to botocore's " + "credential chain, so it may succeed — with whatever identity the " + "chain resolves (~/.aws, SSO, IMDS). If you relied on v0 failing " + "loudly here, add an explicit check." + ), "supabase": ( "v0's Supabase branch builds its client through boto3's process-global " "session, so its behaviour is not a function of these arguments. Note: " @@ -105,6 +140,12 @@ def _v0_mangle(args: Mapping, environ: Mapping) -> tuple[str, Optional[dict]]: ) return "env-credentials (endpoint dropped)", effective if profile == "environment variables": + if not env_creds: + # v0's get_client('environment variables') RAISES S3DolException + # here. v1 falls through to botocore's chain, which may resolve a + # DIFFERENT identity (~/.aws, SSO, IMDS) — a behaviour change worth + # naming, and not expressible as a resolution diff. + return "env-profile (v0 RAISES: no env credentials)", None effective.update( aws_access_key_id=environ.get("AWS_ACCESS_KEY_ID"), aws_secret_access_key=environ.get("AWS_SECRET_ACCESS_KEY"), @@ -141,6 +182,40 @@ def _spec_from_v0_args(args: Mapping) -> S3Connection: ) +def _credential_identity(spec, environ: Mapping) -> tuple: + """A comparable, non-secret credential *identity* (ADR-0012 D7: compared + by a boolean predicate, never by snapshot, and never by reading + ``.access_key`` — which performs network I/O on a refreshable credential). + + Provenance *labels* are the wrong comparator in both directions: the v0 + side labels env credentials "explicit static (AKIA…)" while the v1 side + labels the same keys "chain (env credentials present)" (a false alarm for + the most common configuration), and two different explicit keys both + truncate to "AKIA…" (a missed alarm for the very case step 0 exists to + catch — v0 letting env silently override explicit credentials). + """ + from s3dol.connection import ProfileCredentials, StaticCredentials + + if spec.anon: + return ("anonymous",) + credentials = spec.credentials + if isinstance(credentials, StaticCredentials): + return ("keys", credentials.access_key) # full id: compared, not printed + if isinstance(credentials, ProfileCredentials): + return ("profile", credentials.profile) + if credentials is not None: + return ("callable", credentials.provenance()) + if spec.profile: + return ("profile", spec.profile) + # The chain: its top rung is env credentials, which is what a v0-mangled + # side would have materialised as explicit keys — so identical inputs + # compare EQUAL here, which is the point. + env_key = environ.get("AWS_ACCESS_KEY_ID") + if env_key: + return ("keys", env_key) + return ("chain", environ.get("AWS_PROFILE") or "default") + + def _redact_url(url: Optional[str]) -> Optional[str]: """Redact query secrets and any userinfo from a URL for display.""" if url is None: @@ -186,21 +261,52 @@ def _divergence_rows(args: Mapping, environ: Mapping, aws_config) -> list[str]: f" {axis}: v0 -> {_redact_url(v0_value)!r}, " f"v1 -> {_redact_url(v1_value)!r} ** CHANGES ON UPGRADE **" ) - if v0_res.credential_provenance != v1_res.credential_provenance: + v0_identity = _credential_identity(_spec_from_v0_args(effective), environ) + v1_identity = _credential_identity(_spec_from_v0_args(args), environ) + if v0_identity != v1_identity: + moved = True + rows.append( + f" credential identity: v0 -> {_identity_label(v0_identity)}, " + f"v1 -> {_identity_label(v1_identity)} ** CHANGES ON UPGRADE **" + ) + # v1 introduces changes v0 never made, and they cannot show up above: + # BOTH sides run through v1's resolver, so pass-2 detection cancels out. + # Report them explicitly, or the announcement instrument cannot see the + # very path->virtual flip ADR-0012 D4 promises to announce. + if v1_res.preset_source == "detected": moved = True rows.append( - f" credentials: v0 -> {v0_res.credential_provenance}, " - f"v1 -> {v1_res.credential_provenance} ** CHANGES ON UPGRADE **" + f" provider detected from the endpoint: " + f"{v1_res.preset.name!r} — v1 now sets " + f"addressing_style={v1_res.addressing_style.value!r}" + + ( + f", checksum={v1_res.checksum.value!r}" + if v1_res.checksum.passed + else "" + ) + + f", region={v1_res.region_name.value!r}. v0 passed none of " + f"these (botocore defaults applied). ** NEW IN v1 **" ) if not moved: rows.append(" no divergence: v1 resolves this call exactly as v0 does.") return rows +def _identity_label(identity: tuple) -> str: + """A printable form of a credential identity — key ids truncated.""" + kind, *rest = identity + if kind == "keys": + return f"access key {str(rest[0])[:4]}…" + if kind == "anonymous": + return "anonymous" + return f"{kind}:{rest[0]}" + + def diagnose( connection: Optional[S3Connection] = None, *, environ: Optional[Mapping[str, str]] = None, + aws_config: Union[Mapping, None] = None, file=None, **kwargs, ) -> str: @@ -246,12 +352,18 @@ def emit(line=""): import os environ = dict(os.environ) - aws_config: object - try: - aws_config = load_aws_config() - except Exception as error: - aws_config = {} - emit(f"note: aws config unreadable ({type(error).__name__}: {error})") + if aws_config is None: + # The one impure read (the shared AWS config file), matching what + # a real construction resolves against. Injectable — without that, + # a "tier 1" test silently consults the developer's own + # ~/.aws/config and goes red on a machine whose profile sets an + # endpoint_url (ADR-0012 D2 made resolve() injectable exactly so + # this channel is controllable). + try: + aws_config = load_aws_config() + except Exception as error: + aws_config = {} + emit(f"note: aws config unreadable ({type(error).__name__}: {error})") v0_args = {k: v for k, v in kwargs.items() if k in _V0_ARG_NAMES} v1_kwargs = {k: v for k, v in kwargs.items() if k not in _V0_ARG_NAMES} @@ -261,7 +373,14 @@ def emit(line=""): emit() emit("arguments:") for key in sorted(kwargs): - shown = "" if key in _SECRET_ARG_NAMES else repr(kwargs[key]) + if key in _REDACTED_ARG_NAMES: + shown = repr(_redact_url(kwargs[key])) + elif key in _ECHOABLE_ARG_NAMES: + shown = repr(kwargs[key]) + elif key == "aws_access_key_id" and kwargs[key]: + shown = f"{str(kwargs[key])[:4]!r}… (prefix only)" + else: + shown = "" emit(f" {key} = {shown}") emit() diff --git a/s3dol/base.py b/s3dol/base.py index e8eda38..a818caa 100644 --- a/s3dol/base.py +++ b/s3dol/base.py @@ -1,150 +1,484 @@ -"""S3Dol with a dict-like interface for AWS S3 buckets. -Supports relative paths to access files under a prefix ending with slash. -Hints added to help setup credentials. - -bucket = S3Dol()[profile][bucket_name] -bucket['level1/level2/test-key'] == bucket['level1/']['level2/']['test-key'] +"""Layer B — close-to-metal stores over one bucket or one endpoint. + +Design record: `misc/docs/architecture.md` (the contract tables) on +ADR-0001 (layering; the prefix lives HERE, in the leaf), ADR-0004 (one error +seam), ADR-0005 (write domain / strategies), ADR-0006 (prefix arithmetic and +key validity), ADR-0008 (`__len__` absent by design), ADR-0010 (bucket policy), +ADR-0011 (**no key-taking public methods** — one guarded exception, `url_for`), +ADR-0012 (connection). + +The class triads mirror ``dol.filesys`` and ``azuredol``:: + + BucketCollection -> BucketReader -> BucketStore (keys: object keys) + EndpointCollection -> EndpointReader -> EndpointStore (keys: bucket names) + ObjectHandle (not a Mapping) + +Reader-only classes are *real classes* (a type checker catches +``reader[k] = v``; a read-scoped credential never even attempts a write) — +``dol.mk_read_only`` is non-functional on a dol store (verified, ADR-0001). + +**The rule for what may join a class here: nothing that takes a key** +(ADR-0011 D1). `dol` wrappers hand any non-dunder method the outer, unmapped +key, and the standard escape (``wrapped_self``) is itself silently wrong when +nothing references the wrapper. Keyed capabilities are sibling stores +(``s3dol.handles(store)[k]`` — `capabilities.py`); per-object richness lives on +:class:`ObjectHandle` (key bound at construction); everything else is a free +function taking the store first. The single exception is +:meth:`BucketReader.url_for`, kept for ``dol.SupportsUrlFor``, guarded to be +correct-or-loud (D3b). A reflective conformance test enforces the rule (D5). """ +from __future__ import annotations + from dataclasses import dataclass -import functools -import os -from typing import Union -from collections.abc import Iterable, Mapping +from datetime import datetime +from typing import Any, Iterator, Mapping, Optional, Union + +from dol.base import Collection as DolCollection, KvPersister, KvReader + +from s3dol.connection import S3Connection +from s3dol.errors import ( + BucketNotEmpty, + BucketNotFound, + ConfigurationError, + CredentialsError, + KeyNotValid, + NotSupported, + ObjectArchived, + S3Error, + classify_client_error, + code_of, + status_of, + translate_s3_errors, + translating_s3_errors, +) +from s3dol.reads import DEFAULT_READS, ReadStrategy +from s3dol.writes import DEFAULT_WRITES, WriteStrategy + +#: S3's real limit is 1024 **UTF-8 bytes** (enforced client-side so moto and +#: AWS agree — moto accepts longer keys; ADR-0006 §4). +MAX_KEY_BYTES = 1024 + +#: SigV4's presigned-URL lifetime cap (botocore does not enforce it). +MAX_PRESIGN_EXPIRES = 604_800 + + +def _validate_key(k, *, delimiter: str = "/") -> str: + """Normalise key-validity failures to :class:`KeyNotValid` **before the + wire** (ADR-0006 §4) — otherwise botocore types escape ``__getitem__`` + (``''`` -> ``ParamValidationError``; a lone surrogate -> + ``UnicodeEncodeError``; a 1025-byte key -> fine on moto, error on AWS).""" + if not isinstance(k, str): + raise KeyNotValid( + f"S3 keys are str; got {type(k).__name__}. " + f"(If you need structured keys, put a dol key codec on top.)" + ) + if k == "": + raise KeyNotValid("The empty string is not a valid S3 key.") + try: + encoded = k.encode("utf-8") + except UnicodeEncodeError as error: + raise KeyNotValid( + f"Key contains code points that cannot encode to UTF-8 " + f"(lone surrogate?): {k!r}" + ) from error + if len(encoded) > MAX_KEY_BYTES: + raise KeyNotValid( + f"Key is {len(encoded)} UTF-8 bytes; S3's limit is {MAX_KEY_BYTES}. " + f"(Checked client-side so moto and AWS agree.)" + ) + return k -from botocore.exceptions import ClientError -import boto3 -import dol -from s3dol.utility import KeyNotValidError, Resp, S3DolException, S3KeyError +@dataclass(frozen=True) +class ObjectInfo: + """Object metadata — from ``HeadObject`` or from a LIST row (which already + carries size/mtime/etag/storage class that v0 threw away; ADR-0009).""" + key: str + size: Optional[int] = None + last_modified: Optional[datetime] = None + etag: Optional[str] = None + content_type: Optional[str] = None + storage_class: Optional[str] = None + restore_status: Optional[str] = None + version_id: Optional[str] = None -noCredentialsFound = S3DolException( - "No AWS credentials found. Configure your AWS credentials as environment variables or in ~/.aws/credentials. " - "See https://github.com/i2mint/s3dol/#set-up-credentials for more information." -) +def _normalize_prefix(prefix: Optional[str], delimiter: str = "/") -> str: + """The mandatory normalisation, first (ADR-0006 §1): a prefix that does + not terminate in the delimiter is normalised, never accepted as-is — + ``prefix='logs'`` would otherwise expose ``logs2/…`` as readable *and + writable* keys one character away. -def get_aws_credentials(): - aws_access_key = os.environ.get("AWS_ACCESS_KEY_ID") - aws_secret_key = os.environ.get("AWS_SECRET_ACCESS_KEY") - aws_region = os.environ.get("AWS_DEFAULT_REGION") - aws_session_token = os.environ.get("AWS_SESSION_TOKEN") + >>> _normalize_prefix('logs'), _normalize_prefix('/logs/'), _normalize_prefix('') + ('logs/', 'logs/', '') + """ + if not prefix: + return "" + return f"{prefix.strip(delimiter)}{delimiter}" - return { - "aws_access_key_id": aws_access_key, - "aws_secret_access_key": aws_secret_key, - "region_name": aws_region, - "aws_session_token": aws_session_token, - } +class _WrapProbe: + """Descriptor recording dol's wrap-time ``hasattr(store, 'KeysView')`` + probe on the instance, then reporting the attribute absent. Class-level + access (``getattr(cls, 'KeysView')``) records nothing.""" -def list_profile_names(): - """ - Return a list of available AWS profiles - """ - session = boto3.session.Session() - available_profiles = session.available_profiles - # Check environment variables for default profile - if os.environ.get("AWS_ACCESS_KEY_ID") and os.environ.get("AWS_SECRET_ACCESS_KEY"): - available_profiles.append("environment variables") + def __get__(self, instance, owner=None): + if instance is not None: + instance.__dict__["_ever_wrapped"] = True + raise AttributeError("KeysView") - return available_profiles +class _BucketBase: + """Shared base of the bucket-level classes: owns the normalised prefix, + the connection, the key arithmetic, and the error context — so those exist + exactly once (ADR-0011 D2). Plain class; the dol ABCs are mixed in by the + concrete triad classes.""" -def get_client(profile_name=None, endpoint_url=None, **session_kwargs): - """ - Return a boto3 client for the specified profile - """ - if profile_name is None: - return _find_default_credentials(endpoint_url=endpoint_url, **session_kwargs) - - if profile_name == "environment variables": - aws_credentials = get_aws_credentials() - if ( - not aws_credentials["aws_access_key_id"] - or not aws_credentials["aws_secret_access_key"] - ): - raise S3DolException( - "Missing AWS credentials in environment variables: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY" + def __init__( + self, + bucket: str, + *, + connection: Union[S3Connection, None] = None, + prefix: str = "", + delimiter: str = "/", + reads: Optional[ReadStrategy] = None, + writes: Optional[WriteStrategy] = None, + on_missing_bucket: str = "assume", + strict_delete: bool = False, + ): + if not isinstance(bucket, str) or not bucket: + raise ConfigurationError(f"bucket must be a non-empty str; got {bucket!r}") + if on_missing_bucket not in ("assume", "create", "raise"): + raise ConfigurationError( + f"on_missing_bucket must be 'assume', 'create' or 'raise'; got " + f"{on_missing_bucket!r} (ADR-0010 §1)." ) - _skw = {**session_kwargs, **aws_credentials} - session = boto3.Session(**_skw) - else: - # Get credentials from profile - session = boto3.Session(profile_name=profile_name, **session_kwargs) - client = session.client("s3", endpoint_url=endpoint_url) - return client - - -def _find_default_credentials(endpoint_url=None, **session_kwargs): - try: - return get_client("environment variables", **session_kwargs) - except S3DolException: - pass - session = boto3.Session(**session_kwargs) - if session.get_credentials() is None: - raise noCredentialsFound - client = session.client("s3", endpoint_url=endpoint_url) - return client - - -@dataclass -class BaseS3BucketReader(dol.base.KvReader): - """Dict-like interface for AWS S3 buckets""" - - client: boto3.client - bucket_name: str - prefix: str = None - delimiter: str = "/" - - def __post_init__(self): - self.prefix = ( - f"{self.prefix.strip(self.delimiter)}{self.delimiter}" - if self.prefix - else "" + self.bucket = bucket + self.connection = connection if connection is not None else S3Connection() + self.delimiter = delimiter + self.prefix = _normalize_prefix(prefix, delimiter) + self.reads = reads if reads is not None else DEFAULT_READS + self.writes = writes if writes is not None else DEFAULT_WRITES + self.on_missing_bucket = on_missing_bucket + self.strict_delete = strict_delete + self._bucket_reachable = False + if on_missing_bucket == "raise": + # The explicit opt-in that performs ONE HeadBucket at construction + # (documented I/O — the only construction-time request; ADR-0010). + self._probe_bucket_or_raise() + + # -- key arithmetic (the leaf owns the prefix — ADR-0001/0006) --------- # + + def _id_of_key(self, k: str) -> str: + return f"{self.prefix}{_validate_key(k, delimiter=self.delimiter)}" + + def _key_of_id(self, _id: str) -> str: + # RAISE rather than slice a non-matching id: the server-side Prefix + # makes out-of-scope ids unreachable on the common path; this is the + # belt to that suspenders, because a provider that ignores Prefix must + # not silently produce corrupt keys (ADR-0006 §1). + if self.prefix and not _id.startswith(self.prefix): + raise KeyNotValid( + f"Backend returned id {_id!r} outside this store's prefix " + f"{self.prefix!r} — refusing to relativise it (the provider " + f"may be ignoring ListObjectsV2's Prefix)." + ) + return _id[len(self.prefix) :] + + # -- plumbing ---------------------------------------------------------- # + + @property + def client(self): + return self.connection.client + + def _error_context(self) -> dict: + row = self._resolution().preset + return dict( + bucket=self.bucket, + endpoint=self._endpoint_label(), + overrides=row.error_overrides if row is not None else (), + deny_means_absent=self.connection.deny_means_absent, + head_ambiguity_resolver=self._head_ambiguity_resolver, ) - def object_list_pages(self) -> Iterable[dict]: - if self._bucket_exists(): - yield from self.client.get_paginator("list_objects").paginate( - Bucket=self.bucket_name, Prefix=self.prefix + def _resolution(self): + cached = self.__dict__.get("_resolution_cache") + if cached is None: + cached = self.connection.resolution() + self.__dict__["_resolution_cache"] = cached + return cached + + def _endpoint_label(self) -> Optional[str]: + built = self.connection.__dict__.get("_client") + if built is not None: + return getattr(built.meta, "endpoint_url", None) + return self._resolution().endpoint_url.value + + def _mark_bucket_reachable(self): + self._bucket_reachable = True + + def _head_ambiguity_resolver(self) -> bool: + """Resolve the body-less ``HeadObject`` 404 (ADR-0004 §2): ``True`` + iff the bucket is proven reachable (so the miss is the object's). + Costs one ``HeadBucket``, once — cached. An auth failure here raises + ``AccessDenied`` rather than masquerading as a missing bucket.""" + if self._bucket_reachable: + return True + from botocore.exceptions import ClientError + + try: + self._op_head_bucket() + except ClientError as error: + classified = classify_client_error( + "HeadBucket", + code_of(error), + status_of(error), + overrides=self._error_context()["overrides"], + ) + if classified.kind == "bucket_absent": + return False + raise # auth/transport: never disguise as absence (ADR-0004 §1) + self._mark_bucket_reachable() + return True + + def _probe_bucket_or_raise(self): + if not self._head_ambiguity_resolver(): + raise BucketNotFound( + f"Bucket {self.bucket!r} not found (on_missing_bucket='raise'). " + f"Note only ('404', 404) means missing; a 403 would have raised " + f"AccessDenied naming s3:ListBucket." ) - def __iter__(self) -> Iterable[str]: - for resp in self.object_list_pages(): - Resp.ascertain_200_status_code(resp) - yield from filter( - lambda k: not k.endswith(self.delimiter), - map(Resp.key, Resp.contents(resp)), + def _refuse_anonymous_write(self, operation: str): + # botocore offers nothing here — moto even accepts an unsigned PUT — + # so the refusal is local (ADR-0012 D5). + if self.connection.anon: + raise CredentialsError( + f"{operation} refused: this store is anonymous (anon=True). " + f"Anonymous credentials cannot sign writes." ) - def __getitem__(self, k: str): - try: - return self.client.get_object(Bucket=self.bucket_name, Key=k)["Body"].read() - except self.client.exceptions.NoSuchKey as ex: - raise KeyError(f"Key {k} does not exist") from ex + def _sibling_kwargs(self) -> dict: + """Constructor kwargs for a sibling capability store over the same + key space (ADR-0011 D2) — the key arithmetic exists once, here.""" + return dict( + bucket=self.bucket, + connection=self.connection, + prefix=self.prefix, + delimiter=self.delimiter, + reads=self.reads, + writes=self.writes, + on_missing_bucket="assume", + strict_delete=self.strict_delete, + ) + + def _make_sibling(self, kind: str, **options): + """Build the sibling capability store of ``kind`` over the same key + space. Overridable so `s3dol.testing`'s fake hands out fake siblings.""" + from s3dol import capabilities as _capabilities + + sibling_class = { + "handles": _capabilities.BucketHandles, + "urls": _capabilities.BucketUrls, + "info": _capabilities.BucketInfo, + }[kind] + return sibling_class(**{**self._sibling_kwargs(), **options}) + + # -- the backend protocol (ADR-0002's module-internal seam) ------------ # + # Layer B talks to the backend ONLY through these. Implemented once by + # botocore here; s3dol.testing's in-memory fake overrides exactly these + # (raising synthesized botocore ClientErrors, so the error taxonomy and + # every guard above this line is exercised identically). Deliberately NOT + # a public extension point in v1: a protocol with one real implementer + # would be an abstraction shaped like its implementation. + + def _op_list_pages( + self, *, delimiter: Optional[str] = None, prefix: Optional[str] = None + ): + row = self._resolution().preset + use_v2 = row.capabilities.list_objects_v2 if row is not None else True + paginator = self.client.get_paginator( + "list_objects_v2" if use_v2 else "list_objects" + ) + kwargs: dict = dict( + Bucket=self.bucket, Prefix=self.prefix if prefix is None else prefix + ) + if delimiter: + kwargs["Delimiter"] = delimiter + return paginator.paginate(**kwargs) + + def _op_head_object(self, _id: str) -> dict: + return self.client.head_object(Bucket=self.bucket, Key=_id) + def _op_head_bucket(self) -> None: + self.client.head_bucket(Bucket=self.bucket) + + def _op_read(self, _id: str): + return self.reads(self.client, self.bucket, _id) + + def _op_write(self, _id: str, value) -> None: + self.writes(self.client, self.bucket, _id, value) + + def _op_delete(self, _id: str) -> None: + self.client.delete_object(Bucket=self.bucket, Key=_id) + + def _op_bulk_delete(self, _ids: list) -> tuple: + """One DeleteObjects call (≤1000 ids). Returns ``(deleted_ids, + errors)`` where errors is ``[(id, code, message), ...]`` — parsed out + of the HTTP **200** response (ADR-0010 §2).""" + response = self.client.delete_objects( + Bucket=self.bucket, + Delete={"Objects": [{"Key": k} for k in _ids], "Quiet": False}, + ) + deleted = [d["Key"] for d in response.get("Deleted", ())] + errors = [ + (e.get("Key"), e.get("Code"), e.get("Message")) + for e in response.get("Errors", ()) + ] + return deleted, errors + + def _op_create_bucket(self) -> None: + _create_bucket( + self.client, self.bucket, region=self._resolution().region_name.value + ) + + def _op_presign( + self, _id: str, *, expires_in: int, client_method: str, **params + ) -> Optional[str]: + return _presign( + self.connection, + bucket=self.bucket, + key=_id, + expires_in=expires_in, + client_method=client_method, + **params, + ) + + def _with(self, **overrides) -> "_BucketBase": + """An explicit-fields sub-store builder (never + ``type(self)(**self.__dict__)`` — ADR-0001). Overridden prefixes are + re-normalised by ``__init__``.""" + kwargs = self._sibling_kwargs() + kwargs["on_missing_bucket"] = self.on_missing_bucket + kwargs.update(overrides) + return type(self)(kwargs.pop("bucket"), **kwargs) + + #: The wrap probe (ADR-0011 D1a §3, adapted). dol.base.Store.__init__ + #: probes its wrapped store with ``hasattr(store, 'KeysView')``. On a + #: plain leaf that reaches ``__getattr__``; on a KvReader-derived leaf it + #: would be satisfied by MappingViewMixin's class attribute and the probe + #: would be undetectable — so we shadow it with a descriptor that records + #: the touch and raises AttributeError. keys()/values()/items() below + #: bypass it. (Documented false positive: a once-wrapped leaf later used + #: bare will refuse url_for — accepted, D1a.) + KeysView = _WrapProbe() + + def keys(self): + from dol.base import BaseKeysView + + return BaseKeysView(self) + + def values(self): + from dol.base import BaseValuesView + + return BaseValuesView(self) + + def items(self): + from dol.base import BaseItemsView + + return BaseItemsView(self) + + def __repr__(self): + # bucket, prefix, endpoint host, mode. No secrets. (architecture.md) + bits = [repr(self.bucket)] + if self.prefix: + bits.append(f"prefix={self.prefix!r}") + endpoint = self._endpoint_label() + if endpoint: + from urllib.parse import urlsplit + + bits.append(f"endpoint={urlsplit(endpoint).hostname!r}") + if self.connection.anon: + bits.append("anon=True") + return f"{type(self).__name__}({', '.join(bits)})" + + # -- pickling: every s3dol store pickles (ADR-0008 §6) ----------------- # + + def __getstate__(self): + state = dict(self.__dict__) + state.pop("_resolution_cache", None) + return state + + def __setstate__(self, state): + self.__dict__.update(state) + + +class BucketCollection(_BucketBase, DolCollection): + """Keys of one bucket (under the prefix). ``Collection`` only — + iteration and membership, no reads.""" + + def __iter__(self) -> Iterator[str]: + # Lazy, paginated, prefix pushed down to the server. RAISES if the + # bucket is missing or unlistable — never yields empty (goal 4). + with translating_s3_errors(self, operation="ListObjectsV2"): + for page in self._op_list_pages(): + self._mark_bucket_reachable() + for entry in page.get("Contents", ()): + _id = entry["Key"] + if _id == self.prefix: + # The exact-prefix marker object would relativise to + # the forbidden key '' — the scoped view filters it + # out; it stays addressable by absolute key on an + # unscoped store (ADR-0006 §4). + continue + yield self._key_of_id(_id) + + @translate_s3_errors(operation="HeadObject", key_arg="k") def __contains__(self, k) -> bool: + from botocore.exceptions import ClientError + try: - self.client.head_object(Bucket=self.bucket_name, Key=k) - return True # if all went well - except KeyNotValidError as e: - raise - except ClientError as e: - if e.response["Error"]["Code"] == "404": - # The object does not exist. + self._op_head_object(self._id_of_key(k)) + except ClientError as error: + classified = classify_client_error( + "HeadObject", + code_of(error), + status_of(error), + overrides=self._error_context()["overrides"], + ) + if classified.kind == "ambiguous_head": + if self._head_ambiguity_resolver(): + return False # bucket proven reachable: the object is absent + raise # bucket absent: let the seam raise BucketNotFound + if classified.kind == "object_absent": return False - else: - # Something else has gone wrong. - raise + if classified.kind == "access_denied" and self.connection.deny_means_absent: + return False + raise # auth/config/transport: never a silent False (ADR-0004) + self._mark_bucket_reachable() + return True + + def __len__(self): + # Explicit raise: dol's Collection.__len__ counts by iterating, so + # len(store) would be a full paginated listing and list(store) would + # cost TWO listings via the length hint (ADR-0008 §cost model). + raise TypeError( + f"{type(self).__name__} does not implement __len__ (a length is a " + f"full paginated listing, and list() would take it as a hint and " + f"pay twice). Use sum(1 for _ in store) if you accept the scan." + ) - def _bucket_exists(self) -> bool: - try: - self.client.head_bucket(Bucket=self.bucket_name) - return True - except ClientError: - return False + +class BucketReader(BucketCollection, KvReader): + """+ ``__getitem__`` -> bytes (or the read strategy's value type).""" + + @translate_s3_errors(operation="GetObject", key_arg="k") + def __getitem__(self, k): + value = self._op_read(self._id_of_key(k)) + self._mark_bucket_reachable() + return value def url_for( self, @@ -153,180 +487,496 @@ def url_for( expires_in: int = 3600, client_method: str = "get_object", **params, - ) -> str: - """Presigned URL for key ``k`` (default: a GET valid for ``expires_in`` - seconds). - - Lets a caller 302-redirect to the object store so it serves the bytes - (and HTTP ``Range``) directly, instead of proxy-streaming every byte - through the app. Works for any S3-compatible endpoint (AWS, R2, MinIO, - Supabase) — a pure client call, no object fetch. Extra ``params`` merge - into the presign ``Params`` (e.g. ``ResponseContentType='video/mp4'``). - See issue #7. + ) -> Optional[str]: + """Presigned URL for ``k`` — **the one keyed method** (ADR-0011 D3b), + kept solely because ``dol.SupportsUrlFor`` requires a *method* and + ``dol.content_url`` reaches it by ``getattr``. The canonical form is + ``s3dol.urls(store)[k]``. + + Guarded to be correct-or-loud, never silently wrong: + + - **unwrapped store**: ``k`` is a key in this store's own key space, + so the URL addresses ``self._id_of_key(k)``. Correct, and this is + also the contract ``dol.content_url`` calls under — it resolves the + key through the *outer* layers and stops at the layer owning + ``url_for``, "because that layer applies its own" + (``dol/content.py``); + - **this leaf has ever been wrapped**: **raises**. Two call paths then + reach this one signature with *different key domains* and nothing + distinguishes them: ``dol.content_url`` hands over a key already + mapped through the wrapper layers, while a direct + ``wrapped.url_for(k)`` (delegation) hands over the raw outer key. + Applying ``_id_of_key`` is right for the first and wrong for the + second; re-resolving through the chain is right for the second and + double-transforms the first. Since a wrong presigned URL addresses a + *different real object*, the only honest answer is refusal, naming + the sibling store — which is correct by construction because it goes + through ``__getitem__`` (ADR-0011 D2/D3b, and see the note below); + - **anonymous connection**: returns ``None`` (``SupportsUrlFor`` is + ``Optional[str]``; raising would turn a public-bucket streaming + fallback into a 500 — ADR-0012 D5). The check reads the built + client's signature version, never ``spec.anon``. + + .. note:: This refusal is a *finding*, not a limitation we chose. + ADR-0011 D3b expected a guarded method to serve both paths; it + cannot, on a leaf that owns its prefix. Reported upstream on + dol#86/#83 as evidence that a keyed capability *method* is + unfixable in the general case, and that the sibling-store form is + the only one correct by construction. """ - return self.client.generate_presigned_url( - client_method, - Params={"Bucket": self.bucket_name, "Key": k, **params}, - ExpiresIn=expires_in, + if self.__dict__.get("_ever_wrapped"): + raise NotSupported( + "url_for refused: this store has been wrapped by a dol " + "key/value wrapper, and the two call paths that reach this " + "method (dol.content_url, which pre-resolves the key, and a " + "direct delegated call, which does not) are indistinguishable " + "here — one of them would get a URL for the WRONG object. Use " + "the sibling store, which is correct by construction: " + "s3dol.urls(store)[key] (derive it from the unwrapped store " + "and apply the same wrapper to both)." + ) + return self._op_presign( + self._id_of_key(k), + expires_in=expires_in, + client_method=client_method, + **params, ) -class BaseS3BucketDol(BaseS3BucketReader, dol.base.KvPersister): - skip_bucket_exists_check: bool = False +class BucketStore(BucketReader, KvPersister): + """+ ``__setitem__`` / ``__delitem__``. The user-facing read-write store.""" + @translate_s3_errors(operation="PutObject", key_arg="k") def __setitem__(self, k, v): - if not self.skip_bucket_exists_check and not self._bucket_exists(): - # create_bucket will not be silent if permission to create bucket is denied - self.client.create_bucket(Bucket=self.bucket_name) - self.client.put_object(Bucket=self.bucket_name, Key=k, Body=v) + self._refuse_anonymous_write("__setitem__") + _id = self._id_of_key(k) + from botocore.exceptions import ClientError - def __delitem__(self, k): - self.client.delete_object(Bucket=self.bucket_name, Key=k) + try: + self._op_write(_id, v) + except ClientError as error: + # 'create' recovers, never probes: attempt, and on NoSuchBucket + # create-then-retry once (ADR-0010 §1). + if self.on_missing_bucket == "create" and code_of(error) == "NoSuchBucket": + self._op_create_bucket() + self._mark_bucket_reachable() + self._op_write(_id, v) + else: + raise + self._mark_bucket_reachable() + @translate_s3_errors(operation="DeleteObject", key_arg="k") + def __delitem__(self, k): + self._refuse_anonymous_write("__delitem__") + _id = self._id_of_key(k) + if self.strict_delete: + # The opt-in probe (one extra request, TOCTOU-racy — documented). + if k not in self: + raise ObjectNotFoundForDelete(k) + self._op_delete(_id) + # Idempotent by default: S3's DeleteObject returns 204 for an absent + # key; raising KeyError would require a HeadObject probe on EVERY + # delete (architecture.md contract table). + + # -- ObjectArchived must not be swallowed by the defaulted accessors --- # + # MutableMapping.setdefault is `try: self[k] except KeyError: self[k] = + # default` — a Glacier KeyError would let setdefault OVERWRITE the + # archived object with the default, silently (measured; ADR-0004 §4). + + def setdefault(self, k, default=None): + try: + return self[k] + except ObjectArchived: + raise + except KeyError: + self[k] = default + return default -# TODO: Use wrap_kvs and filt_iter from dol to handle key and id conversions -@dataclass -class S3BucketReader(BaseS3BucketReader): - """ - Enables path delimitations to access objects under a prefix ending with slash such as 'level1/level2/'. + def pop(self, k, *default): + try: + value = self[k] + except ObjectArchived: + raise + except KeyError: + if default: + return default[0] + raise + del self[k] + return value - s3_bucket['level1/']['level2/']['test-key'] == s3_bucket['level1/level2/test-key'] - """ + def update(self, other=(), /, **kwargs) -> None: + # Explicit override for the type: MutableMapping's value type is + # invariant, so `dst.update(src)` with refs (Filepath/Chunks) fails a + # type checker without this widened signature (ADR-0005 §3). + super().update(other, **kwargs) - skip_bucket_exists_check: bool = False - def _id_of_key(self, k): - return f"{self.prefix}{k}" +class ObjectNotFoundForDelete(S3Error, KeyError): + """Raised by ``strict_delete=True`` deletes of absent keys.""" - def _key_of_id(self, id): - return id[len(self.prefix) :] + def __str__(self): + return f"Key {self.args[0]!r} was absent (strict_delete=True)." - def __iter__(self): - return ( - self._key_of_id(_id) - for _id in super().__iter__() - if _id.startswith(self.prefix) - ) - def __getitem__(self, k: str): - _id = self._id_of_key(k) - if _id.endswith(self.delimiter): - _kw = {**self.__dict__, "prefix": _id} - return type(self)(**_kw) - return super().__getitem__(_id) +def _create_bucket(client, bucket: str, *, region: Optional[str]): + """CreateBucket with the region rules that bite (ADR-0010 §1): + ``CreateBucketConfiguration`` everywhere except us-east-1 (where passing + it is an error), and ``BucketAlreadyOwnedByYou`` tolerated (re-create is + only idempotent in us-east-1).""" + from botocore.exceptions import ClientError - def __contains__(self, k) -> bool: - _id = self._id_of_key(k) - return super().__contains__(_id) + kwargs: dict = {"Bucket": bucket} + if region and region not in ("us-east-1", "auto"): + kwargs["CreateBucketConfiguration"] = {"LocationConstraint": region} + try: + client.create_bucket(**kwargs) + except ClientError as error: + if code_of(error) not in ("BucketAlreadyOwnedByYou", "BucketAlreadyExists"): + raise - def url_for(self, k: str, **kwargs) -> str: - """Prefix-aware presigned URL: maps ``k`` through ``_id_of_key`` (as - ``__getitem__`` does) so the URL points at the same object.""" - return super().url_for(self._id_of_key(k), **kwargs) +def clamp_expires_in(expires_in: int) -> int: + """Clamp to SigV4's 604800 s cap, warning. botocore does not enforce it — + it just signs a URL that fails after 7 days.""" + import warnings -class S3BucketDol(S3BucketReader, BaseS3BucketDol): - def __setitem__(self, k, v): - _id = self._id_of_key(k) + if expires_in > MAX_PRESIGN_EXPIRES: + warnings.warn( + f"expires_in={expires_in} exceeds SigV4's hard cap of " + f"{MAX_PRESIGN_EXPIRES}s (botocore does not enforce it; the URL " + f"would simply fail after 7 days). Clamping.", + stacklevel=3, + ) + return MAX_PRESIGN_EXPIRES + return expires_in + + +def _presign( + connection: S3Connection, + *, + bucket: str, + key: str, + expires_in: int, + client_method: str = "get_object", + **params, +) -> Optional[str]: + """Shared presigning core: SigV4 (the connection guarantees it), the + anonymous ``None``, the 604800 s cap, and the credential-expiry clamp.""" + import warnings + + from botocore import UNSIGNED + + client = connection.presign_client + if client.meta.config.signature_version is UNSIGNED: + return None # no capability, not an error (ADR-0012 D5) + expires_in = clamp_expires_in(expires_in) + credentials = client._request_signer._credentials + expiry = getattr(credentials, "_expiry_time", None) + if expiry is not None: + from datetime import timezone + + remaining = (expiry - datetime.now(timezone.utc)).total_seconds() + if 0 < remaining < expires_in: + warnings.warn( + f"The signing credentials expire in ~{int(remaining)}s, " + f"before expires_in={expires_in}: a presigned URL cannot " + f"outlive the credential that signed it (STS/SSO/instance " + f"sessions). The URL will die with the session.", + stacklevel=3, + ) + return client.generate_presigned_url( + client_method, + Params={"Bucket": bucket, "Key": key, **params}, + ExpiresIn=expires_in, + ) - # Directly call BaseS3BucketDol.__setitem__ to ensure we respect skip_bucket_exists_check - if not self.skip_bucket_exists_check and not self._bucket_exists(): - self.client.create_bucket(Bucket=self.bucket_name) - self.client.put_object(Bucket=self.bucket_name, Key=_id, Body=v) - def __delitem__(self, k): - _id = self._id_of_key(k) - return super().__delitem__(_id) +# --------------------------------------------------------------------------- # +# ObjectHandle — the per-object escape hatch (NOT a Mapping) +# --------------------------------------------------------------------------- # -class S3ClientReader(dol.base.KvReader): - ignore_404_when_endpoint_has_substring = ".supabase." +class ObjectHandle: + """One object, key bound **at construction** — which is what makes it + immune to the delegation trap (a key codec over the store cannot corrupt a + key that was resolved before the handle existed; ADR-0011 D1, following + ``azuredol.BlobHandle``). This is where ranged reads, streaming, + download-to-disk, metadata and presigned URLs live.""" def __init__( - self, *, s3_bucket_dol=S3BucketDol, profile_name=None, **session_kwargs + self, bucket: str, key: str, *, connection: Optional[S3Connection] = None ): - self.client = get_client(profile_name=profile_name, **session_kwargs) - self.s3_bucket_dol = s3_bucket_dol + self.bucket = bucket + self.key = key # the ABSOLUTE (wire) key + self.connection = connection if connection is not None else S3Connection() + + @property + def client(self): + return self.connection.client + + def _error_context(self) -> dict: + return dict(bucket=self.bucket, endpoint=None) + + def __repr__(self): + return f"{type(self).__name__}({self.bucket!r}, {self.key!r})" + + def __eq__(self, other): + return isinstance(other, ObjectHandle) and ( + self.bucket, + self.key, + self.connection, + ) == (other.bucket, other.key, other.connection) + + def __hash__(self): + return hash((self.bucket, self.key, self.connection)) + + @translate_s3_errors(operation="GetObject") + def read( + self, *, offset: Optional[int] = None, length: Optional[int] = None + ) -> bytes: + """The object's bytes; a ranged read when ``offset``/``length`` given.""" + kwargs: dict = {"Bucket": self.bucket, "Key": self.key} + if offset is not None or length is not None: + start = offset or 0 + end = "" if length is None else start + length - 1 + kwargs["Range"] = f"bytes={start}-{end}" + return self.client.get_object(**kwargs)["Body"].read() + + @translate_s3_errors(operation="GetObject") + def stream(self, *, chunk_size: int = 8 * 2**20) -> Iterator[bytes]: + """Lazy chunk iterator over the body.""" + from functools import partial + + body = self.client.get_object(Bucket=self.bucket, Key=self.key)["Body"] + return iter(partial(body.read, chunk_size), b"") + + @translate_s3_errors(operation="GetObject") + def download_to(self, path) -> str: + """Download to disk (never fully in memory). Accepts a path or + :class:`~s3dol.values.Filepath`; returns the path written.""" + from s3dol.values import Filepath + + target = path.path if isinstance(path, Filepath) else str(path) + self.client.download_file(self.bucket, self.key, target) + return target + + @translate_s3_errors(operation="PutObject") + def write(self, value) -> None: + """Write the object (same value domain as ``store[k] = v``).""" + if self.connection.anon: + raise CredentialsError("Anonymous credentials cannot sign writes.") + DEFAULT_WRITES(self.client, self.bucket, self.key, value) + + @translate_s3_errors(operation="DeleteObject") + def delete(self) -> None: + if self.connection.anon: + raise CredentialsError("Anonymous credentials cannot sign writes.") + self.client.delete_object(Bucket=self.bucket, Key=self.key) + + def exists(self) -> bool: + from botocore.exceptions import ClientError + + try: + self.client.head_object(Bucket=self.bucket, Key=self.key) + except ClientError as error: + if classify_client_error( + "HeadObject", code_of(error), status_of(error) + ).kind in ("object_absent", "ambiguous_head"): + return False + raise + return True + + @translate_s3_errors(operation="HeadObject") + def info(self) -> ObjectInfo: + """One ``HeadObject`` -> :class:`ObjectInfo`.""" + head = self.client.head_object(Bucket=self.bucket, Key=self.key) + return ObjectInfo( + key=self.key, + size=head.get("ContentLength"), + last_modified=head.get("LastModified"), + etag=head.get("ETag"), + content_type=head.get("ContentType"), + storage_class=head.get("StorageClass") or "STANDARD", + restore_status=head.get("Restore"), + version_id=head.get("VersionId"), + ) + + def url( + self, + *, + expires_in: int = 3600, + client_method: str = "get_object", + **params, + ) -> Optional[str]: + """Presigned URL. Always SigV4; ``None`` when anonymous; capped at + SigV4's 604800 s; warns when the signing credential expires sooner + (the URL cannot outlive it).""" + return _presign( + self.connection, + bucket=self.bucket, + key=self.key, + expires_in=expires_in, + client_method=client_method, + **params, + ) + + @translate_s3_errors(operation="RestoreObject") + def restore(self, days: int = 1, *, tier: str = "Standard") -> None: + """Request restoration of an archived (Glacier) object.""" + self.client.restore_object( + Bucket=self.bucket, + Key=self.key, + RestoreRequest={"Days": days, "GlacierJobParameters": {"Tier": tier}}, + ) + + +# --------------------------------------------------------------------------- # +# Endpoint level (keys = bucket names) +# --------------------------------------------------------------------------- # - def __iter__(self): - return (b["Name"] for b in self.client.list_buckets().get("Buckets", [])) - def __contains__(self, k): +class _EndpointBase: + """Shared base of the endpoint-level classes.""" + + def __init__(self, connection: Optional[S3Connection] = None, **bucket_kwargs): + if connection is not None and not isinstance(connection, S3Connection): + raise ConfigurationError( + f"connection must be an S3Connection (got " + f"{type(connection).__name__}). Live boto3 objects are not " + f"accepted anywhere in v1 (ADR-0012 D1)." + ) + self.connection = connection if connection is not None else S3Connection() + #: kwargs forwarded to the bucket stores this endpoint hands out. + self.bucket_kwargs = bucket_kwargs + + @property + def client(self): + return self.connection.client + + def _error_context(self) -> dict: + row = self.connection.resolution().preset + return dict( + bucket=None, + endpoint=None, + overrides=row.error_overrides if row is not None else (), + deny_means_absent=self.connection.deny_means_absent, + ) + + def __repr__(self): + endpoint = self.connection.resolution().endpoint_url.value + inner = "" if not endpoint else f"endpoint={endpoint!r}" + return f"{type(self).__name__}({inner})" + + +class EndpointCollection(_EndpointBase, DolCollection): + """Bucket names at one endpoint.""" + + def __iter__(self) -> Iterator[str]: + with translating_s3_errors(self, operation="ListBuckets"): + for bucket in self.client.list_buckets().get("Buckets", ()): + yield bucket["Name"] + + @translate_s3_errors(operation="HeadBucket", key_arg="k") + def __contains__(self, k) -> bool: + if not isinstance(k, str) or not k: + return False + from botocore.exceptions import ClientError + try: self.client.head_bucket(Bucket=k) - return True - except Exception as e: - # Check for a 400 error code in the response attributes - if ( - hasattr(e, "response") - and e.response.get("Error", {}).get("Code") == "400" - ): - # Use the endpoint URL to decide if it's Supabase - endpoint = getattr(self.client.meta, "endpoint_url", "") - if "supabase" in endpoint: - return True - elif "404" in str(e): + except ClientError as error: + classified = classify_client_error( + "HeadBucket", + code_of(error), + status_of(error), + overrides=self._error_context()["overrides"], + ) + if classified.kind == "bucket_absent": return False - raise S3DolException(f"Error checking bucket existence: {e}") from e + raise # auth/config: never "the bucket doesn't exist" (ADR-0004) + return True + + def __len__(self) -> int: + # Bucket counts are small — an administrative resource (ADR-0008). + return sum(1 for _ in self) + + +class EndpointReader(EndpointCollection, KvReader): + """+ ``__getitem__`` -> :class:`BucketReader`.""" + + _bucket_class = BucketReader def __getitem__(self, k: str): - # if k not in self: - # raise S3KeyError(f'Bucket {k} does not exist') + if not isinstance(k, str) or not k: + raise KeyNotValid(f"Bucket names are non-empty str; got {k!r}") + return self._bucket_class(k, connection=self.connection, **self.bucket_kwargs) - return self.s3_bucket_dol(client=self.client, bucket_name=k) +class EndpointStore(EndpointReader, KvPersister): + """+ bucket creation/deletion. ``del endpoint[name]`` refuses a non-empty + bucket; the explicit cascading form is the free function + ``s3dol.delete_bucket(endpoint, name, force=True)`` (ADR-0010 §3 as + amended by ADR-0011 D4 — a public keyed destructive *method* is the shape + the family census found destroying wrong data).""" -class S3ClientDol(S3ClientReader, dol.base.KvPersister): - def __init__( - self, *, s3_bucket_dol=S3BucketDol, profile_name=None, **session_kwargs - ): - super().__init__( - s3_bucket_dol=s3_bucket_dol, profile_name=profile_name, **session_kwargs - ) + _bucket_class = BucketStore + @translate_s3_errors(operation="CreateBucket", key_arg="k") def __setitem__(self, k: str, v: Mapping): - """Populate a bucket with a mapping of keys to objects. Create bucket if it does not exist. - - :param k: bucket name - :type k: str - :param v: mapping of keys to objects to populate bucket with - :type v: Mapping - """ + """Create bucket ``k`` and populate it from mapping ``v`` (pass ``{}`` + to just create).""" + if self.connection.anon: + raise CredentialsError("Anonymous credentials cannot create buckets.") if not isinstance(v, Mapping): raise TypeError( - f"Value must be a mapping (dict-like) object. Got {type(v)}" + f"EndpointStore values are Mappings of objects to populate the " + f"bucket with (use {{}} to just create it); got {type(v).__name__}." ) - self.client.create_bucket(Bucket=k) + _create_bucket( + self.client, k, region=self.connection.resolution().region_name.value + ) bucket = self[k] - for bucket_item_key, bucket_item_value in v.items(): - bucket[bucket_item_key] = bucket_item_value + for key, value in v.items(): + bucket[key] = value - def __delitem__(self, k): - res = self.client.list_objects_v2(Bucket=k) - if "Contents" in res: - objects = [{"Key": obj["Key"]} for obj in res["Contents"]] - self.client.delete_objects(Bucket=k, Delete={"Objects": objects}) - self.client.delete_bucket(Bucket=k) + @translate_s3_errors(operation="DeleteBucket", key_arg="k") + def __delitem__(self, k: str): + if self.connection.anon: + raise CredentialsError("Anonymous credentials cannot delete buckets.") + from botocore.exceptions import ClientError + try: + self.client.delete_bucket(Bucket=k) + except ClientError as error: + if code_of(error) == "BucketNotEmpty": + raise BucketNotEmpty( + f"Bucket {k!r} is not empty. Deleting it would destroy " + f"its objects; that never happens implicitly. Use " + f"s3dol.delete_bucket(endpoint, {k!r}, force=True) for " + f"the explicit, paginated cascade." + ) from error + raise -class S3Dol(dol.base.KvReader): - """S3 profiles -> buckets -> keys -> objects""" - def __init__(self, s3_client_dol=S3ClientDol, s3_bucket_dol=S3BucketDol): - if len(list_profile_names()) == 0: - raise noCredentialsFound - self.s3_client_dol = s3_client_dol - self.s3_bucket_dol = s3_bucket_dol +class S3Profiles(KvReader): + """AWS **profile** names -> endpoint stores (the v0 ``S3Dol``, renamed — + its keys are verifiably profile names, never endpoints; ADR-0007 §1). - def __iter__(self): - return iter(list_profile_names()) + >>> # S3Profiles()['prod'] -> EndpointStore over the 'prod' profile + """ - def __getitem__(self, k: str | dict): - if isinstance(k, str): - return self.s3_client_dol(profile_name=k, s3_bucket_dol=self.s3_bucket_dol) - return self.s3_client_dol(s3_bucket_dol=self.s3_bucket_dol, **k) + def __init__(self, *, bucket_kwargs: Optional[dict] = None, readonly: bool = False): + self.bucket_kwargs = bucket_kwargs or {} + self.readonly = readonly + def __iter__(self) -> Iterator[str]: + import botocore.session -S3DolReadOnly = functools.partial( - S3Dol, s3_client_dol=S3ClientReader, s3_bucket_dol=S3BucketReader -) + yield from botocore.session.Session().available_profiles + + def __contains__(self, k) -> bool: + return isinstance(k, str) and k in set(self) + + def __getitem__(self, k: str): + endpoint_class = EndpointReader if self.readonly else EndpointStore + return endpoint_class(connection=S3Connection(profile=k), **self.bucket_kwargs) diff --git a/s3dol/capabilities.py b/s3dol/capabilities.py new file mode 100644 index 0000000..610077b --- /dev/null +++ b/s3dol/capabilities.py @@ -0,0 +1,396 @@ +"""Keyed capabilities as sibling stores; non-keyed operations as free +functions (ADR-0011 D2/D3/D4). + +A keyed capability is a Layer B ``KvReader`` over the **same key space** whose +``__getitem__`` returns the capability:: + + BucketStore k -> bytes + BucketHandles s3dol.handles(s) k -> ObjectHandle + BucketUrls s3dol.urls(s) k -> presigned URL (str) | None + BucketInfo s3dol.info(s) k -> ObjectInfo + +``__getitem__`` is the one thing `dol` maps correctly at every wrapper depth, +so sibling stores are correct *by construction*: no ``inner_most_key``, no +``wrapped_self``, no guard — immune to both the lost-reference hole (D1a) and +the non-``Store``-layer hole (D3a), the only form in ADR-0011 that is. + +**The cost, stated plainly** (D2): a user who wraps the data store must wrap +the sibling in parallel:: + + s = s3_store('bucket', prefix='p/') + h = s3dol.handles(s) + c = KeyCodecs.prefixed('x/') # a user-applied key codec + s2, h2 = c(s), c(h) # wrap BOTH + h2['f'] # -> handle on 'p/x/f'; correct at any depth + +The accessors therefore derive a sibling from an **unwrapped** store and +**raise** on a wrapped one, naming that remedy — s3dol will not guess a user's +codec chain (that is dol#10). + +Free functions (D3) resolve keys with the store held as an argument (alive, so +no weakref registry is consulted). Per D3a they are *more* reliable than +methods, not reliable: a hand-rolled non-``Store`` passthrough layer in the +chain can still defeat the resolution — no docstring here claims "correct at +any wrapper depth"; only the sibling stores may claim that. +""" + +from __future__ import annotations + +from typing import Iterable, Iterator, Optional + +from dol.base import KvReader, Store as DolStore + +from s3dol.base import ( + BucketCollection, + ObjectHandle, + ObjectInfo, + _BucketBase, + _EndpointBase, + _presign, +) +from s3dol.errors import ( + BucketNotEmpty, + ConfigurationError, + CredentialsError, + S3PartialFailure, + code_of, + translate_s3_errors, + translating_s3_errors, +) + +# --------------------------------------------------------------------------- # +# The sibling capability stores (D2) +# --------------------------------------------------------------------------- # + + +class BucketHandles(BucketCollection, KvReader): + """``k -> ObjectHandle`` (key bound at construction; zero round-trips).""" + + def __getitem__(self, k) -> ObjectHandle: + return ObjectHandle(self.bucket, self._id_of_key(k), connection=self.connection) + + +class BucketUrls(BucketCollection, KvReader): + """``k -> presigned URL`` (zero object requests; ``None`` when anonymous). + The canonical spelling of what ``url_for`` shims (ADR-0011 D3b).""" + + def __init__( + self, *args, expires_in: int = 3600, client_method: str = "get_object", **kwargs + ): + super().__init__(*args, **kwargs) + self.expires_in = expires_in + self.client_method = client_method + + def __getitem__(self, k) -> Optional[str]: + return _presign( + self.connection, + bucket=self.bucket, + key=self._id_of_key(k), + expires_in=self.expires_in, + client_method=self.client_method, + ) + + +class BucketInfo(BucketCollection, KvReader): + """``k -> ObjectInfo`` (one ``HeadObject``).""" + + @translate_s3_errors(operation="HeadObject", key_arg="k") + def __getitem__(self, k) -> ObjectInfo: + _id = self._id_of_key(k) + head = self.client.head_object(Bucket=self.bucket, Key=_id) + self._mark_bucket_reachable() + return ObjectInfo( + key=_id, + size=head.get("ContentLength"), + last_modified=head.get("LastModified"), + etag=head.get("ETag"), + content_type=head.get("ContentType"), + storage_class=head.get("StorageClass") or "STANDARD", + restore_status=head.get("Restore"), + version_id=head.get("VersionId"), + ) + + +# --------------------------------------------------------------------------- # +# The accessors: derive a sibling from an UNWRAPPED store; refuse a wrapped one +# --------------------------------------------------------------------------- # + +_PARALLEL_WRAP_REMEDY = ( + "derive the sibling from the *unwrapped* store and apply the same " + "wrapper to both: s2, sib2 = codec(s), codec(s3dol.{fn}(s)). " + "s3dol will not guess your codec chain (that is dol#10)." +) + + +def _leaf_of(store, *, fn: str) -> _BucketBase: + if isinstance(store, _BucketBase): + return store + if isinstance(store, DolStore): + raise ConfigurationError( + f"s3dol.{fn}() needs an unwrapped s3dol store, but got a dol " + f"wrapper ({type(store).__name__}). Returning an unwrapped " + f"sibling here would silently ignore your key/value transforms — " + f"instead, {_PARALLEL_WRAP_REMEDY.format(fn=fn)}" + ) + raise ConfigurationError( + f"s3dol.{fn}() takes an s3dol bucket store; got {type(store).__name__}." + ) + + +def handles(store) -> BucketHandles: + """The sibling store of :class:`ObjectHandle`\\ s: ``handles(s)[k]``.""" + return _leaf_of(store, fn="handles")._make_sibling("handles") + + +def urls( + store, *, expires_in: int = 3600, client_method: str = "get_object" +) -> BucketUrls: + """The sibling store of presigned URLs: ``urls(s)[k]``.""" + return _leaf_of(store, fn="urls")._make_sibling( + "urls", expires_in=expires_in, client_method=client_method + ) + + +def info(store) -> BucketInfo: + """The sibling store of :class:`ObjectInfo`: ``info(s)[k]``.""" + return _leaf_of(store, fn="info")._make_sibling("info") + + +# --------------------------------------------------------------------------- # +# Free functions (D3/D4) +# --------------------------------------------------------------------------- # + + +def _outward_key_chain(store) -> list: + """The wrapper layers outermost-first, ending at the s3dol leaf. + Refuses (rather than guesses) on a non-``Store`` middle layer — D3a's + documented hole.""" + chain = [store] + current = store + while not isinstance(current, _BucketBase): + if not isinstance(current, DolStore): + raise ConfigurationError( + f"Cannot resolve keys through {type(current).__name__}: it is " + f"neither an s3dol leaf nor a dol Store wrapper. Free " + f"functions refuse rather than guess on hand-rolled layers " + f"(ADR-0011 D3a)." + ) + inner = current.store + chain.append(inner) + current = inner + return chain + + +def _map_key_outward(store, wire_relative_key: str) -> str: + """Map a key from the LEAF's relative key space out to the caller's — + the inverse walk ``prefixes`` needs (each layer's ``_key_of_id``, + innermost first). Property-tested round trip: + ``inner_most_key(store, _map_key_outward(store, k_rel))`` recovers the + wire key for pure key-codec chains.""" + chain = _outward_key_chain(store) + key = wire_relative_key + for layer in reversed(chain[:-1]): # innermost wrapper -> outermost + key_of_id = getattr(layer, "_key_of_id", None) + if callable(key_of_id): + key = key_of_id(key) + return key + + +def sub(store, prefix: str): + """A store scoped to ``prefix``, **in the caller's key space** (D3). + + Two branches, deliberately different (each documented in the ADR): + + - **unwrapped s3dol store**: ``leaf._with(prefix=...)`` — cheap, keeps + server-side prefix pushdown, returns the same class; + - **dol-wrapped store**: composes ``Pipe(filt_iter.prefixes(p), + KeyCodecs.prefixed(p))`` over the *outer* store (filter FIRST — the only + safe composition, ADR-0006 §1), which preserves the user's codecs but + loses pushdown (a full scan filtered client-side; D8 closed this: the + cheap path is the unwrapped branch) and returns a different type. + """ + if isinstance(store, _BucketBase): + return store._with(prefix=f"{store.prefix}{_norm(prefix, store.delimiter)}") + if isinstance(store, DolStore): + from dol import KeyCodecs, Pipe, filt_iter + + normalized = _norm(prefix, "/") + return Pipe(filt_iter.prefixes(normalized), KeyCodecs.prefixed(normalized))( + store + ) + raise ConfigurationError( + f"s3dol.sub() takes an s3dol store or a dol wrapper over one; got " + f"{type(store).__name__}." + ) + + +def _norm(prefix: str, delimiter: str) -> str: + from s3dol.base import _normalize_prefix + + return _normalize_prefix(prefix, delimiter) + + +def _wire_scope_base(chain: list) -> str: + """The absolute (wire) prefix the *caller's* key space is rooted at: + the leaf's own prefix extended by each wrapper layer's contribution + (``layer._id_of_key('')``, outermost first). Only delimiter-shaped + contributions are accepted — a codec that maps ``''`` to anything else + (a suffix codec, say) has no well-defined directory structure and is + refused loudly rather than listed wrongly.""" + leaf: _BucketBase = chain[-1] + mapped = "" + for layer in chain[:-1]: # outermost wrapper -> innermost + id_of_key = getattr(layer, "_id_of_key", None) + if not callable(id_of_key): + continue + try: + mapped = id_of_key(mapped) + except Exception as error: + raise ConfigurationError( + f"s3dol.prefixes(): the wrapper layer {type(layer).__name__} " + f"cannot map the key-space root ({type(error).__name__}); its " + f"codec has no well-defined directory structure. List on the " + f"unwrapped store instead." + ) from error + if not isinstance(mapped, str) or ( + mapped and not mapped.endswith(leaf.delimiter) + ): + raise ConfigurationError( + f"s3dol.prefixes(): the wrapper layer {type(layer).__name__} " + f"maps the key-space root to {mapped!r}, which is not a " + f"delimiter-terminated scope — its codec has no well-defined " + f"directory structure. List on the unwrapped store instead." + ) + return f"{leaf.prefix}{mapped}" + + +def prefixes(store) -> list: + """The 'directories' one level under the store's scope, **relative to the + caller's key space** — one ``ListObjectsV2(Delimiter='/')`` reading + ``CommonPrefixes`` (which v0 parsed and then called from nowhere; + ADR-0009). Through a dol wrapper chain, the listing is rooted at the + caller's scope (each prefix layer's contribution) and results are mapped + back **outward** (``_key_of_id`` per layer, innermost first — the inverse + walk `dol` does not provide; ADR-0011 D3). A wrapper whose codec cannot + express the scope raises — loud, not wrong.""" + chain = _outward_key_chain(store) + leaf: _BucketBase = chain[-1] + base = _wire_scope_base(chain) + with translating_s3_errors(leaf, operation="ListObjectsV2"): + found = [] + for page in leaf._op_list_pages(delimiter=leaf.delimiter, prefix=base): + for row in page.get("CommonPrefixes", ()): + relative = leaf._key_of_id(row["Prefix"]) + found.append( + _map_key_outward(store, relative) if len(chain) > 1 else relative + ) + return found + + +#: AWS's DeleteObjects cap. moto accepts 1001, so tier 2 can never catch a +#: missing chunker (ADR-0010 §2) — hence the constant is also unit-tested. +DELETE_MANY_CHUNK = 1000 + + +def delete_many(store, keys: Iterable[str]) -> None: + """Bulk delete (ADR-0010 §2; a free function per ADR-0011 D4 — keyed + + destructive + delegated is the census's data-destroying shape). + + Chunks at 1000 (AWS's cap), parses the ``Errors`` list out of the HTTP + 200 response, and on partial failure raises one :class:`S3PartialFailure` + carrying ``.succeeded`` and ``.failures``. Absent keys are reported by S3 + as ``Deleted`` — this function does **not** distinguish them (same + idempotency as ``del store[k]``). + + Keys are the *caller's* keys: with the store held as an argument the + chain is resolved via ``dol.inner_most_key`` (alive store — no weakref + hole). Validate-the-target caveat (D4): the first argument must be an + s3dol store or a dol wrapper over one. + """ + from dol import inner_most_key + + chain = _outward_key_chain(store) # validates the target + leaf: _BucketBase = chain[-1] + if leaf.connection.anon: + raise CredentialsError("Anonymous credentials cannot sign deletes.") + if len(chain) > 1: + wire_keys = [inner_most_key(store, k) for k in list(keys)] + bad = [k for k in wire_keys if not isinstance(k, str)] + if bad: + raise ConfigurationError( + f"Key resolution through the wrapper chain produced non-str " + f"values ({bad[:3]!r}…): refusing to delete." + ) + else: + wire_keys = [leaf._id_of_key(k) for k in keys] + + row = leaf._resolution().preset + batch_ok = row.capabilities.batch_delete if row is not None else True + succeeded: list = [] + failures: dict = {} + with translating_s3_errors(leaf, operation="DeleteObjects"): + if batch_ok: + for start in range(0, len(wire_keys), DELETE_MANY_CHUNK): + chunk = wire_keys[start : start + DELETE_MANY_CHUNK] + deleted, errors = leaf._op_bulk_delete(chunk) + succeeded.extend(deleted) + for err_key, err_code, err_message in errors: # in an HTTP 200! + failures[err_key] = S3PartialFailure( + f"{err_code}: {err_message}", succeeded=[], failures={} + ) + else: + # Exact, cheap emulation (GCS & checksum-rejecting providers): + # identical observable result, only cost differs (ADR-0003 §2). + from botocore.exceptions import ClientError + + for key in wire_keys: + try: + leaf._op_delete(key) + succeeded.append(key) + except ClientError as error: + failures[key] = S3PartialFailure( + str(code_of(error)), succeeded=[], failures={} + ) + if failures: + raise S3PartialFailure( + f"delete_many: {len(failures)} of {len(wire_keys)} deletions " + f"failed ({len(succeeded)} succeeded).", + succeeded=succeeded, + failures=failures, + ) + + +def delete_bucket(endpoint, name: str, *, force: bool = False) -> None: + """Delete bucket ``name``. Refuses a non-empty bucket unless + ``force=True``, in which case the cascade **paginates** — v0 listed one + page, deleted ≤1000 objects, then failed: a partial, non-idempotent + destruction (ADR-0010 §3). A free function, not a method (ADR-0011 D4).""" + if not isinstance(endpoint, _EndpointBase): + raise ConfigurationError( + f"s3dol.delete_bucket() takes an EndpointStore/Reader first; got " + f"{type(endpoint).__name__}." + ) + if endpoint.connection.anon: + raise CredentialsError("Anonymous credentials cannot delete buckets.") + client = endpoint.client + with translating_s3_errors(endpoint, operation="DeleteBucket", key=name): + if force: + paginator = client.get_paginator("list_objects_v2") + for page in paginator.paginate(Bucket=name): + contents = page.get("Contents", ()) + if contents: + client.delete_objects( + Bucket=name, + Delete={"Objects": [{"Key": row["Key"]} for row in contents]}, + ) + from botocore.exceptions import ClientError + + try: + client.delete_bucket(Bucket=name) + except ClientError as error: + if code_of(error) == "BucketNotEmpty": + raise BucketNotEmpty( + f"Bucket {name!r} is not empty; pass force=True for the " + f"explicit, paginated cascade." + ) from error + raise diff --git a/s3dol/connection.py b/s3dol/connection.py index 799ff35..2e3227b 100644 --- a/s3dol/connection.py +++ b/s3dol/connection.py @@ -57,6 +57,7 @@ AmbiguousResolution, ConfigurationError, MissingEndpoint, + MissingPresetParam, PresetConflict, PresetHostMismatch, ) @@ -167,9 +168,18 @@ class CallableCredentials(CredentialProvider): fetch: Callable[[], Mapping] + def _name(self) -> str: + # NEVER repr(self.fetch): functools.partial's repr prints its bound + # arguments, and the rejection message below recommends a partial — + # a partial closing over a vault token would leak it (D1 §4). + target = getattr(self.fetch, "func", self.fetch) # unwrap a partial + return getattr(target, "__qualname__", None) or type(target).__name__ + + def __repr__(self): + return f"{type(self).__name__}(fetch={self._name()})" + def provenance(self) -> str: - name = getattr(self.fetch, "__qualname__", None) or repr(type(self.fetch)) - return f"callable:{name}" + return f"callable:{self._name()}" @staticmethod def _normalize(result: Mapping) -> dict: @@ -196,15 +206,6 @@ def pick(*names): def _load_botocore_credentials(self): from botocore.credentials import Credentials, RefreshableCredentials - first = self._normalize(self.fetch()) - if first["expiry_time"] is None: - return Credentials( - first["access_key"], - first["secret_key"], - first["token"], - method="s3dol-callable", - ) - def refresh(): fresh = self._normalize(self.fetch()) return { @@ -214,8 +215,18 @@ def refresh(): "expiry_time": fresh["expiry_time"], } + # ONE fetch on the first load — a token vendor may be rate-limited or + # single-use, so the initial credential is the one we keep. + first = refresh() + if first["expiry_time"] is None: + return Credentials( + first["access_key"], + first["secret_key"], + first["token"], + method="s3dol-callable", + ) return RefreshableCredentials.create_from_metadata( - metadata=refresh(), refresh_using=refresh, method="s3dol-callable" + metadata=first, refresh_using=refresh, method="s3dol-callable" ) @@ -269,7 +280,14 @@ def normalize_credentials(value: CredentialsInput) -> Optional[CredentialProvide f"credentials= tuple must be (key, secret) or " f"(key, secret, token); got {len(value)} items." ) - return StaticCredentials(*map(str, value)) + access, secret, *rest = value + token = rest[0] if rest else None + # str()-ing a None token would yield the truthy string 'None' and sign + # every request with X-Amz-Security-Token: None — the natural + # `credentials=(k, s, maybe_token)` shape must behave like the 2-tuple. + return StaticCredentials( + str(access), str(secret), None if token is None else str(token) + ) if isinstance(value, Mapping): unknown = set(value) - set(_AWS_CRED_KEYS) if unknown: @@ -292,11 +310,14 @@ def normalize_credentials(value: CredentialsInput) -> Optional[CredentialProvide try: pickle.dumps(value) except Exception as error: + target = getattr(value, "func", value) + name = getattr(target, "__qualname__", None) or type(target).__name__ raise ConfigurationError( - f"credentials= callable {value!r} is not picklable " + f"credentials= callable {name} is not picklable " f"({type(error).__name__}). Every s3dol store pickles " f"(ADR-0012 D1); use a module-level function or a " - f"functools.partial of one." + f"functools.partial of one. (Its repr is deliberately not " + f"shown — a bound argument may be a secret.)" ) from error return CallableCredentials(value) raise ConfigurationError( @@ -353,29 +374,42 @@ class Resolution: "AWS_RESPONSE_CHECKSUM_VALIDATION", ) -#: Region-looking tokens: ``us-east-1``, ``fr-par``, ``us-west-004``, hetzner's -#: ``fsn1``, and R2's literal ``auto``. -_REGION_TOKEN = re.compile(r"[a-z]{2}-[a-z]+-\d+|[a-z]{2}-[a-z]{3,}|[a-z]{3}\d|auto") +#: Region-shaped tokens, deliberately NARROW: ``us-east-1``, ``fr-par``, +#: ``us-west-004``. A looser pattern (an earlier draft also accepted any +#: ``xxxN`` first label, for hetzner's ``fsn1``) captures ordinary hostname +#: labels — ``srv1.corp``, ``eu-storage.corp`` — and because host capture +#: outranks ``AWS_DEFAULT_REGION`` in the ladder, that garbage would be handed +#: to botocore as the SigV4 credential scope and silently override the user's +#: exported region. Providers whose region is genuinely part of the host state +#: it in their preset row instead (hetzner binds ``{location}``). +_REGION_TOKEN = re.compile( + r"[a-z]{2}-[a-z]+-\d+|[a-z]{2}-[a-z]{3,}-\d{2,}|[a-z]{2}-[a-z]{3,}" +) def _region_from_host(endpoint_url: Optional[str]) -> Optional[str]: - """Best-effort region capture from a resolved endpoint host.""" + """Best-effort region capture — ONLY from the canonical ``s3..`` + (or ``s3-.``) host shape, never from an arbitrary label.""" if not endpoint_url: return None from urllib.parse import urlsplit host = urlsplit(endpoint_url).hostname or "" match = re.search(r"(?:^|\.)s3[.-]([a-z0-9-]+)\.", host) - candidates = [match.group(1)] if match else [] - labels = host.split(".") - if labels: - candidates.append(labels[0]) - for candidate in candidates: - if _REGION_TOKEN.fullmatch(candidate): - return candidate + if match and _REGION_TOKEN.fullmatch(match.group(1)): + return match.group(1) return None +def _looks_secret(name: str) -> bool: + """Whether an option name plausibly carries a secret (repr scrubbing).""" + lowered = str(name).lower() + return any( + marker in lowered + for marker in ("secret", "token", "password", "passwd", "credential") + ) + + _CREDENTIAL_KEY_MARKERS = ( "access_key", "secret", @@ -386,6 +420,15 @@ def _region_from_host(endpoint_url: Optional[str]) -> Optional[str]: ) +def _redact_env_value(key: str, value: Optional[str]) -> Optional[str]: + """A display-safe form of a consulted environment value.""" + from s3dol.errors import redact_url + + if value and "ENDPOINT" in key: + return redact_url(value) + return value + + def _scrub_credentials(config: Mapping) -> dict: """Recursively drop credential-shaped keys from an aws-config mapping.""" @@ -427,10 +470,16 @@ def _lower_rung_endpoint( """The endpoint the rungs *below* s3dol's two would supply — read purely, for diagnose() and the C1/C2 guards **only**, never to override (ADR-0012 D3; and never pass ``Config(ignore_configured_endpoint_urls=True)``).""" + from s3dol.errors import redact_url + for env_key in _ENV_ENDPOINT_KEYS: value = environ.get(env_key) if value: - return value, f"env:{env_key}" + # Redact at capture. This rung is never PASSED to botocore (which + # reads the env itself), only reported — so the Resolution, the C2 + # messages and diagnose all become leak-proof at once, while the + # host survives for detection and the guards. + return redact_url(value), f"env:{env_key}" if aws_config: try: from botocore.configprovider import ConfiguredEndpointProvider @@ -443,7 +492,7 @@ def _lower_rung_endpoint( ) value = provider.provide() if value: - return str(value), "aws-config" + return redact_url(str(value)), "aws-config" except Exception: # config parsing must never fail resolution return None return None @@ -508,28 +557,32 @@ def config() -> Mapping: endpoint = Sourced(lower[0], lower[1], passed=False) elif named is not None: missing = named.missing_endpoint_params(region_name=spec.region_name) - if named.requires_endpoint or missing: + if missing: + raise MissingPresetParam( + f"Preset {named.name!r} has unbound endpoint template " + f"parameter(s) {sorted(missing)}, so it supplies no " + f"endpoint — and nothing below supplied one either. Bind " + f"them (preset=get_preset({named.name!r}).bind(...)), or " + f"pass endpoint_url=. Falling back to AWS silently is " + f"structurally impossible (ADR-0012 C1)." + ) + if named.requires_endpoint: raise MissingEndpoint( - f"Preset {named.name!r} needs an endpoint and nothing " - f"supplied one" - + ( - f" (unbound template parameter(s): {sorted(missing)} — " - f"bind with get_preset({named.name!r}).bind(...))" - if missing - else "" - ) - + ". Pass endpoint_url=, or set AWS_ENDPOINT_URL_S3. " - f"Falling back to AWS silently is structurally impossible " - f"(ADR-0012 C1)." + f"Preset {named.name!r} cannot be reached without an " + f"explicit endpoint and nothing supplied one. Pass " + f"endpoint_url=, or set AWS_ENDPOINT_URL_S3 (ADR-0012 C1)." ) endpoint = Sourced(None, "SDK resolver", passed=False) else: endpoint = Sourced(None, "SDK resolver", passed=False) # -- C2 guard: a named, patterned row whose endpoint arrived from below + # ADR-0012 C2 exempts no *named* row: `soft` governs detection (pass 2), + # not this guard. Exempting 'aws' would silence it for exactly the + # population that declared "I want AWS" and was then retargeted elsewhere + # by an environment variable. if ( named is not None - and not named.soft and named.host_patterns and not endpoint.passed and endpoint.value @@ -680,7 +733,16 @@ def config() -> Mapping: preset_source=preset_source, credential_provenance=provenance, capabilities=row.capabilities, - environ_consulted=tuple((key, environ.get(key)) for key in _ENV_CONSULTED_KEYS), + # Redacted at capture: these are display values (diagnose prints them, + # and a Resolution is the natural thing to log), and an endpoint env + # var may carry userinfo we neither refuse nor pass on. + environ_consulted=tuple( + ( + key, + _redact_env_value(key, environ.get(key)), + ) + for key in _ENV_CONSULTED_KEYS + ), notes=tuple(notes), ) @@ -743,10 +805,44 @@ def __post_init__(self): f"checksum must be 'when_supported', 'when_required', or None; " f"got {self.checksum!r}." ) + if not isinstance(self.signature_version, str) or not self.signature_version: + # NOT a "None == do not override" field, unlike its neighbours: + # ADR-0003 §4 requires signature_version to be set explicitly on + # every client. Leaving it to botocore silently downgrades + # presigning to SigV2 in the regions endpoints.json still lists as + # v2-capable (us-east-1 among them — every conventional MinIO + # setup), and client.meta.config reports 's3v4' while doing it. + raise ConfigurationError( + f"signature_version must be a non-empty str (default 's3v4'); " + f"got {self.signature_version!r}. It is never left to botocore: " + f"that silently re-enables SigV2 presigning (ADR-0003 §4, " + f"issue #10). Pass anon=True for unsigned access." + ) if isinstance(self.client_kwargs, Mapping): object.__setattr__( self, "client_kwargs", tuple(sorted(self.client_kwargs.items())) ) + smuggled = sorted( + key + for key, _ in self.client_kwargs + if key + in ( + "aws_access_key_id", + "aws_secret_access_key", + "aws_session_token", + "config", + ) + ) + if smuggled: + raise ConfigurationError( + f"client_kwargs must not carry {smuggled}: boto3 would accept " + f"them, bypassing credential normalisation, the anon " + f"contradiction check, and repr redaction (the secret would " + f"then print in every log line that reprs this spec). Use " + f"credentials=/profile=, and the connection's own fields for " + f"Config (addressing_style, checksum, signature_version, " + f"payload_signing_enabled, verify)." + ) if self.anon: if self.credentials is not None or self.profile: raise ConfigurationError( @@ -789,7 +885,9 @@ def resolution( return resolve(self, environ if environ is not None else os.environ, aws_config) - def _build_client(self): + def _build_client( + self, *, _override_s3_settings: Optional[dict] = None, _override_endpoint=None + ): import warnings import boto3 @@ -809,6 +907,8 @@ def _build_client(self): s3_settings["payload_signing_enabled"] = ( resolution.payload_signing_enabled.value ) + if _override_s3_settings: + s3_settings.update(_override_s3_settings) config_kwargs: dict = dict(resolution.preset.config_kwargs or ()) if resolution.checksum.passed: config_kwargs["request_checksum_calculation"] = resolution.checksum.value @@ -837,18 +937,52 @@ def _build_client(self): client_kwargs = dict(self.client_kwargs) if self.verify is not None: client_kwargs["verify"] = self.verify + endpoint_to_pass = ( + resolution.endpoint_url.value if resolution.endpoint_url.passed else None + ) + if _override_endpoint is not None: + endpoint_to_pass = _override_endpoint return boto3.Session(botocore_session=botocore_session).client( "s3", - endpoint_url=( - resolution.endpoint_url.value - if resolution.endpoint_url.passed - else None - ), + endpoint_url=endpoint_to_pass, region_name=resolution.region_name.value, config=config, **client_kwargs, ) + @property + def presign_client(self): + """The client presigned URLs are generated with. + + For most providers this is :attr:`client`. Two providers genuinely + need different config for presigning than for the API (ADR-0003 §1): + Hetzner presigns virtual-hosted while the API is path-style; R2 + presigns only on the S3 API domain. When the resolved preset carries + ``presign_endpoint_url``/``presign_addressing_style``, a second client + is built (lazily, once, under the same lock) from the merged config. + """ + resolution = self.resolution() + row = resolution.preset + if row is None or ( + row.presign_endpoint_url is None and row.presign_addressing_style is None + ): + return self.client + built = self.__dict__.get("_presign_client") + if built is None: + with self._lock: + built = self.__dict__.get("_presign_client") + if built is None: + overrides = {} + if row.presign_addressing_style is not None: + overrides["addressing_style"] = row.presign_addressing_style + endpoint = row.presign_endpoint_url + built = self._build_client( + _override_s3_settings=overrides, + _override_endpoint=endpoint, + ) + object.__setattr__(self, "_presign_client", built) + return built + # -- pickling: mandatory for every connection (ADR-0012 D1 §3) --------- # def __getstate__(self): @@ -864,9 +998,18 @@ def __setstate__(self, state): # -- repr: non-default fields only; secrets never (D1 §4) -------------- # def __repr__(self): + from s3dol.errors import redact_url + shown = [] for f in fields(self): value = getattr(self, f.name) - if value != f.default: - shown.append(f"{f.name}={value!r}") + if value == f.default: + continue # only non-defaults, so the repr stays readable + if f.name == "client_kwargs": + value = tuple( + (key, "***" if _looks_secret(key) else item) for key, item in value + ) + elif f.name == "endpoint_url": + value = redact_url(value) + shown.append(f"{f.name}={value!r}") return f"{type(self).__name__}({', '.join(shown)})" diff --git a/s3dol/errors.py b/s3dol/errors.py index fcf1ccc..fe4c364 100644 --- a/s3dol/errors.py +++ b/s3dol/errors.py @@ -173,8 +173,10 @@ class MissingEndpoint(ConfigurationError): supplied (ADR-0012 C1).""" -class MissingPresetParam(ConfigurationError): - """A preset endpoint template has an unbound required parameter.""" +class MissingPresetParam(MissingEndpoint): + """A preset's endpoint template has an unbound required parameter, so the + row contributes no endpoint rung (ADR-0012 C1). A ``MissingEndpoint``, so + either ``except`` works.""" class UnknownPresetParam(ConfigurationError): @@ -231,6 +233,30 @@ class AnonymousFallback(S3DolWarning): ) +def redact_url(url: Optional[str]) -> Optional[str]: + """A URL safe to display: userinfo, query and fragment removed. + + A URL is the residual credential carrier (ADR-0012 D4). Endpoint hygiene + refuses those shapes for values *s3dol* is given, but an endpoint read + from ``AWS_ENDPOINT_URL_S3`` or an aws-config profile is not ours to + refuse — it must merely never be echoed intact. + + >>> redact_url('https://user:pw@minio.internal:9000/x?sig=abc#f') + 'https://REDACTED@minio.internal:9000/x?' + """ + if not url: + return url + from urllib.parse import urlsplit, urlunsplit + + parts = urlsplit(url) + netloc = parts.netloc + if "@" in netloc: + netloc = f"REDACTED@{netloc.rsplit('@', 1)[1]}" + return urlunsplit( + (parts.scheme, netloc, parts.path, "" if parts.query else "", "") + ) + + def redact(text: str) -> str: """Redact signed-URL query secrets from ``text``. @@ -318,6 +344,12 @@ class Classified: ((None, "AuthorizationHeaderMalformed", None), "wrong_region"), # -- archived objects ((None, "InvalidObjectState", None), "archived"), + # -- clock skew / request expiry: 403s that are NOT permission problems. + # They must sit ABOVE the wildcard 403 rows, or deny_means_absent + # would turn a skewed clock into "every key is absent". + ((None, "RequestTimeTooSkewed", None), "credentials"), + ((None, "RequestExpired", None), "credentials"), + ((None, "AuthorizationQueryParametersError", None), "credentials"), # -- credentials ((None, "ExpiredToken", None), "credentials"), ((None, "InvalidAccessKeyId", None), "credentials"), @@ -448,15 +480,21 @@ def translate_client_error( kind = classified.kind if kind == "ambiguous_head": - if head_ambiguity_resolver is not None and head_ambiguity_resolver(): - kind = "object_absent" - elif head_ambiguity_resolver is not None: - kind = "bucket_absent" - else: - kind = "object_absent" # documented default when no resolver bound + if head_ambiguity_resolver is None: + # ADR-0004 §2: the body-less HeadObject 404 must NOT be resolved + # by guessing — "the object is absent" and "the bucket is absent" + # are indistinguishable here on real AWS. With no disambiguator + # bound we propagate the backend error untouched (loud) rather + # than manufacture an absence a caller's .get() would swallow. + return None + kind = "object_absent" if head_ambiguity_resolver() else "bucket_absent" if kind == "access_denied": - if deny_means_absent and key is not None: + # Only a genuine permission code may be read as absence. A 403 that + # means something else (clock skew, expired request) is classified + # above; this guard is the second belt. + permission_code = code in (None, "AccessDenied", "403", "AllAccessDisabled") + if deny_means_absent and key is not None and permission_code: kind = "object_absent" else: return AccessDenied( @@ -497,6 +535,39 @@ def translate_client_error( return exc_type(message) +from contextlib import contextmanager + + +@contextmanager +def translating_s3_errors(store, *, operation: str, key=None, **translate_kwargs): + """Generator/inline form of the seam, for code the decorator cannot wrap + (``__iter__`` bodies: a decorated generator function returns its generator + before the body runs, so the decorator's ``except`` never fires). + + Same contract as :func:`translate_s3_errors`; the store supplies + ``_error_context()``. + """ + from botocore.exceptions import ClientError, NoCredentialsError + + try: + yield + except ClientError as error: + context = dict(getattr(store, "_error_context", dict)()) + context.update(translate_kwargs) + replacement = translate_client_error( + error, operation=operation, key=key, **context + ) + if replacement is None: + raise + raise replacement from error + except NoCredentialsError as error: + raise CredentialsError( + "No AWS credentials found. Configure credentials (env vars, " + "~/.aws/credentials, SSO, or S3Connection(credentials=...)), " + "or pass anon=True for public data." + ) from error + + def translate_s3_errors( *, operation: str, @@ -518,6 +589,14 @@ def decorator(func): import inspect sig = inspect.signature(func) + if key_arg is not None and key_arg not in sig.parameters: + raise TypeError( + f"translate_s3_errors(key_arg={key_arg!r}) on " + f"{func.__qualname__}: that function has no such parameter " + f"(it has {list(sig.parameters)[1:]}). A silently unresolved " + f"key_arg would drop the key from every error message and " + f"disable the deny_means_absent conversion." + ) @wraps(func) def wrapper(self, *args, **kwargs): @@ -529,6 +608,7 @@ def wrapper(self, *args, **kwargs): key = None if key_arg is not None: bound = sig.bind(self, *args, **kwargs) + bound.apply_defaults() # a defaulted key is still the key key = bound.arguments.get(key_arg) context = dict(getattr(self, "_error_context", dict)()) context.update(translate_kwargs) diff --git a/s3dol/presets.py b/s3dol/presets.py index 2a14a50..cee65f1 100644 --- a/s3dol/presets.py +++ b/s3dol/presets.py @@ -36,9 +36,8 @@ from s3dol.errors import ( ConfigurationError, InvalidEndpoint, - MissingPresetParam, - UnknownPresetParam, Rule, + UnknownPresetParam, ) # --------------------------------------------------------------------------- # @@ -115,10 +114,20 @@ def validate_endpoint_hygiene(url: str, *, what: str = "endpoint_url") -> None: f"with '@' in its netloc. Pass credentials via " f"S3Connection(credentials=...), never in a URL." ) + # The refusal messages deliberately do NOT echo the URL: its query/ + # fragment is exactly the secret carrier this check exists to stop, and an + # exception message lands in tracebacks, logs and diagnose reports. if parts.query: - raise InvalidEndpoint(f"{what} must not carry a query string: {url!r}") + raise InvalidEndpoint( + f"{what} must not carry a query string (host " + f"{parts.hostname!r}): the query is where signed-URL and SAS-style " + f"credentials live, and botocore would retain it verbatim in " + f"client.meta.endpoint_url. Pass the bare endpoint." + ) if parts.fragment: - raise InvalidEndpoint(f"{what} must not carry a fragment: {url!r}") + raise InvalidEndpoint( + f"{what} must not carry a fragment (host {parts.hostname!r})." + ) @dataclass(frozen=True) @@ -200,6 +209,23 @@ def __post_init__(self): f"Preset {self.name!r}: config_kwargs values must be " f"scalars; {key!r} is a {type(value).__name__}." ) + from s3dol.errors import KINDS + + for rule in self.error_overrides: + try: + (_operation, _code, _status), kind = rule + except (TypeError, ValueError): + raise ConfigurationError( + f"Preset {self.name!r}: each error_overrides row is " + f"((operation, code, status), kind); got {rule!r}." + ) from None + if kind not in KINDS: + raise ConfigurationError( + f"Preset {self.name!r}: error_overrides kind {kind!r} is " + f"not one of {list(KINDS)}. An unvalidated kind would " + f"detonate inside the error-translation seam at request " + f"time, masking the real backend error." + ) unknown = set(dict(self.params)) - self.template_params() if unknown: raise UnknownPresetParam( diff --git a/s3dol/reads.py b/s3dol/reads.py new file mode 100644 index 0000000..1ed0560 --- /dev/null +++ b/s3dol/reads.py @@ -0,0 +1,62 @@ +"""Read strategies (ADR-0005 §6): what ``s[k]`` returns. + +``s[k]`` returns ``bytes`` — always, under the default strategy, because the +N1 canonical form demands it. ``stream_reads`` is the documented *runtime* +variation (the store's values become chunk iterators); it does not change the +class's static value type — honest typing there would need a generic store, +which v1 does not do. + +Ranged reads and download-to-disk are per-object concerns and live on +``ObjectHandle`` (``handle.read(offset=, length=)``, ``handle.download_to``), +not here — a store-level "ranged" strategy would need a per-key range, which a +Mapping cannot express. + +Strategies are frozen dataclasses, not closures, because every s3dol store +must pickle (ADR-0008 §6) and a store carries its strategy. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from functools import partial +from typing import Any, Iterator, Protocol + + +class ReadStrategy(Protocol): + def __call__(self, client: Any, bucket: str, key: str) -> Any: ... + + +#: Default streaming chunk size (8 MiB, matching the write side). +DEFAULT_CHUNK_SIZE = 8 * 2**20 + + +@dataclass(frozen=True) +class BytesReads: + """The default: one ``GetObject``, fully read -> ``bytes``.""" + + def __call__(self, client, bucket: str, key: str) -> bytes: + return client.get_object(Bucket=bucket, Key=key)["Body"].read() + + +@dataclass(frozen=True) +class StreamReads: + """Values become lazy chunk iterators (the body streams; nothing is held + in memory). The ``GetObject`` itself — and its ``KeyError`` — still + happens eagerly at ``s[k]``; only the byte transfer is lazy.""" + + chunk_size: int = DEFAULT_CHUNK_SIZE + + def __call__(self, client, bucket: str, key: str) -> Iterator[bytes]: + body = client.get_object(Bucket=bucket, Key=key)["Body"] + return iter(partial(body.read, self.chunk_size), b"") + + +def bytes_reads() -> BytesReads: + return BytesReads() + + +def stream_reads(**kwargs) -> StreamReads: + return StreamReads(**kwargs) + + +DEFAULT_READS = BytesReads() diff --git a/s3dol/recipes.py b/s3dol/recipes.py new file mode 100644 index 0000000..b255c40 --- /dev/null +++ b/s3dol/recipes.py @@ -0,0 +1,138 @@ +"""Layer C — the one-liner factory and codec facades. Composition ONLY. + +If a recipe cannot be expressed as Layer B + `dol` wrappers, that is evidence +the capability belongs in Layer B as a *parameter* — this module never grows a +class statement (ADR-0001; the rule that keeps per-vendor classes from +reappearing). + +The one-liner:: + + s = s3dol.s3_store('my-bucket') # zero ceremony + s = s3dol.s3_store('my-bucket', prefix='logs/') # scoped + s = s3dol.s3_store('pub-bucket', anon=True) # public data + +With ``on_missing_bucket='assume'`` (the default; no probe, ever — ADR-0010) +the two-line quickstart raises on a bucket that does not exist yet; start from +nothing with ``on_missing_bucket='create'``. +""" + +from __future__ import annotations + +import json +from typing import Optional, Union + +from dol import wrap_kvs + +from s3dol.base import BucketReader, BucketStore +from s3dol.connection import S3Connection +from s3dol.errors import ConfigurationError +from s3dol.presets import Preset +from s3dol.reads import ReadStrategy +from s3dol.writes import WriteStrategy + + +def s3_store( + bucket: str, + *, + prefix: str = "", + preset: Union[str, Preset, None] = None, + connection: Optional[S3Connection] = None, + endpoint_url: Optional[str] = None, + region_name: Optional[str] = None, + profile: Optional[str] = None, + credentials=None, + anon: bool = False, + on_missing_bucket: str = "assume", + readonly: bool = False, + value_codec=None, + reads: Optional[ReadStrategy] = None, + writes: Optional[WriteStrategy] = None, +): + """One bucket as a ``MutableMapping[str, bytes]`` (a ``Mapping`` when + ``readonly=True``). + + Annotated (deliberately) as the abstract Mapping, **not** ``-> BucketStore``: + with ``value_codec`` the concrete type is a generated `dol` wrapper class + that is *not* a ``BucketStore`` subclass. (v0 annotated its factory + ``-> Store`` and returned something that was not one; inverting that lie + is not an improvement — architecture.md.) + + Connection axes (``preset``/``endpoint_url``/``region_name``/``profile``/ + ``credentials``/``anon``) are conveniences that build an + :class:`~s3dol.connection.S3Connection`; pass ``connection=`` instead for + full control (they are mutually exclusive with it). + """ + if connection is not None: + if ( + any( + x is not None + for x in (preset, endpoint_url, region_name, profile, credentials) + ) + or anon + ): + raise ConfigurationError( + "Pass either connection= or the individual connection axes " + "(preset/endpoint_url/region_name/profile/credentials/anon), " + "not both — one SSOT for where the bytes go." + ) + else: + connection = S3Connection( + preset=preset, + endpoint_url=endpoint_url, + region_name=region_name, + profile=profile, + credentials=credentials, + anon=anon, + ) + store_class = BucketReader if readonly else BucketStore + store = store_class( + bucket, + connection=connection, + prefix=prefix, + reads=reads, + writes=writes, + on_missing_bucket=on_missing_bucket, + ) + if value_codec is not None: + store = value_codec(store) + return store + + +# --------------------------------------------------------------------------- # +# Codec facades. Note S3Jsons uses value_encoder/decoder, NOT +# ValueCodecs.json(): that codec encodes to *str*, which Layer B rejects +# (ADR-0005 §2), and value_codec= does not compose via Pipe. A conformance +# assertion checks every shipped recipe's encoder output is a BytesSource. +# And never `bytes.decode` bare as a transform — dol's signature-inference +# would misread it (dol issue #9): wrap in a lambda. +# --------------------------------------------------------------------------- # + +S3Texts = wrap_kvs( + BucketStore, + obj_of_data=lambda b: b.decode(), + data_of_obj=lambda s: s.encode(), +) +S3Texts.__doc__ = """``BucketStore`` whose values are ``str`` (utf-8 on the wire).""" + +S3Jsons = wrap_kvs( + BucketStore, + obj_of_data=json.loads, + data_of_obj=lambda obj: json.dumps(obj).encode(), +) +S3Jsons.__doc__ = """``BucketStore`` whose values are JSON-able Python objects. + +Note the canonical-form law (ADR-0005 N3): ``s[k] = v`` then ``s[k]`` gives +``json.loads(json.dumps(v))`` — tuples come back as lists, int dict keys as str. +""" + + +def _pickle_codec_store(): + import pickle + + return wrap_kvs(BucketStore, obj_of_data=pickle.loads, data_of_obj=pickle.dumps) + + +S3Pickles = _pickle_codec_store() +S3Pickles.__doc__ = ( + """``BucketStore`` whose values are arbitrary picklable Python objects.""" +) diff --git a/s3dol/store.py b/s3dol/store.py index a63fe60..3a71c9f 100644 --- a/s3dol/store.py +++ b/s3dol/store.py @@ -1,207 +1,288 @@ -"""S3 Store Class""" +"""COMPAT SHIM — the v0 ``S3Store``, forwarded onto the v1 machinery. + +Removed in v2. This module **must survive as an importable module**: external +dependents do ``from s3dol.store import S3Store`` (fully qualified), not +``from s3dol import S3Store`` (ADR-0007 §3). + +The shim keeps v0's exact signature (``bucket_name`` positional-or-keyword, +``path=`` not renamed, the ``make_bucket`` tri-state, and +``is_supabase_endpoint=`` which an early draft forgot) and is **a fix delivery +mechanism**: callers get the corrected endpoint/credential resolution without +changing a line. Three v0 behaviours a dependent's *passing tests* rely on are +deliberately preserved here (and only here — ADR-0007 §4): + +- ``store[k] = 'a str'`` is accepted (encoded utf-8, with a + ``DeprecationWarning``); +- ``store['folder/']`` returns a sub-store; +- ``del store[absent]`` silently succeeds (v1 deletes are idempotent anyway). + +And one v0 behaviour is preserved on the Supabase branch only: the read-side +de-chunker for objects that v0's misconfigured writes stored with their +``aws-chunked`` HTTP framing verbatim (ADR-0003 §3). It is implemented +*correctly* here (multi-chunk; full-parse-or-passthrough) where v0's truncated +multi-chunk payloads. Per ADR-0003's sequencing rule the codec is not deleted +until a repair path ships — :func:`detect_chunked_framing` is that path's +diagnostic. + +When v1 resolves a call differently than v0 *effectively* did (v0 dropped an +explicit ``endpoint_url`` whenever env credentials were set, and let env +credentials override explicit ones), the shim warns with +:class:`~s3dol.errors.S3DolResolutionChanged` — the ADR-0012 D7 differential. +This module and that warning die together in v2. +""" + +from __future__ import annotations + +import os +import re +import warnings +from typing import MutableMapping, Optional + +from dol.base import KvPersister + +from s3dol._diagnose import _spec_from_v0_args, _v0_mangle +from s3dol.base import BucketStore +from s3dol.connection import resolve +from s3dol.errors import S3DolResolutionChanged + +# --------------------------------------------------------------------------- # +# The corrected aws-chunked de-framer (Supabase legacy objects) +# --------------------------------------------------------------------------- # + +_CHUNK_HEAD = re.compile(rb"^[0-9a-fA-F]{1,16}(;|\r\n)") + + +def dechunk_aws_chunked(data: bytes) -> bytes: + """Undo verbatim-persisted ``aws-chunked`` framing, if present. + + Full-parse-or-passthrough: either the ENTIRE payload parses as chunked + framing (every chunk header, every CRLF, through the terminal ``0`` chunk) + and the concatenated chunks are returned, or the payload is returned + untouched. v0's parser truncated any payload larger than one chunk and + misfired on payloads that merely *started* with hex digits; requiring a + complete valid parse removes both failure modes (a random payload that + happens to be a complete chunked document is astronomically unlikely). + + >>> dechunk_aws_chunked(b'5\\r\\nhello\\r\\n0\\r\\n\\r\\n') + b'hello' + >>> framed = b'3\\r\\nabc\\r\\n2\\r\\nde\\r\\n0\\r\\nx-amz-checksum-crc32:AAAA\\r\\n\\r\\n' + >>> dechunk_aws_chunked(framed) # multi-chunk (v0 truncated this) + b'abcde' + >>> dechunk_aws_chunked(b'cafe is hex-ish but not framed') + b'cafe is hex-ish but not framed' + """ + if not _CHUNK_HEAD.match(data): + return data + parts = [] + position = 0 + while True: + crlf = data.find(b"\r\n", position) + if crlf < 0: + return data + size_token = data[position:crlf].split(b";")[0] + try: + size = int(size_token, 16) + except ValueError: + return data + body_start = crlf + 2 + if size == 0: + # Trailer (e.g. x-amz-checksum-crc32:…) then blank line — accept + # anything through the final CRLFCRLF (or end of payload). + rest = data[body_start:] + if rest and not rest.endswith(b"\r\n"): + return data + return b"".join(parts) + chunk = data[body_start : body_start + size] + if ( + len(chunk) < size + or data[body_start + size : body_start + size + 2] != b"\r\n" + ): + return data + parts.append(chunk) + position = body_start + size + 2 + + +def detect_chunked_framing(store, *, sample: int = 20) -> list: + """Which of the first ``sample`` keys hold verbatim ``aws-chunked`` + framing (objects corrupted by v0 Supabase writes). The repair path: + ``store[k] = dechunk_aws_chunked(raw[k])`` per flagged key, reading raw + bytes through an un-codec'd store.""" + flagged = [] + for i, key in enumerate(store): + if i >= sample: + break + raw = store[key] + if isinstance(raw, bytes) and dechunk_aws_chunked(raw) != raw: + flagged.append(key) + return flagged + + +# --------------------------------------------------------------------------- # +# The shim store: v0 conveniences over a v1 BucketStore +# --------------------------------------------------------------------------- # + + +class _V0CompatStore(KvPersister): + """The v0 ergonomics (str writes, ``'folder/'`` sub-stores, counting + ``len()``) over a v1 :class:`BucketStore`. Shim-only; not public API.""" + + def __init__(self, inner: BucketStore, *, dechunk: bool = False): + self._inner = inner + self._dechunk = dechunk -from typing import Optional -from dol import Store + def __getitem__(self, k): + if isinstance(k, str) and k.endswith(self._inner.delimiter): + return type(self)( + self._inner._with(prefix=f"{self._inner.prefix}{k}"), + dechunk=self._dechunk, + ) + value = self._inner[k] + if self._dechunk and isinstance(value, bytes): + value = dechunk_aws_chunked(value) + return value -import boto3 -from s3dol.base import S3BucketDol, S3ClientDol -from s3dol.utility import S3DolException + def __setitem__(self, k, v): + if isinstance(v, str): + warnings.warn( + "Writing a str to an s3dol store is deprecated (removed in " + "v2): it will be stored as its utf-8 bytes. Be explicit: " + "s[k] = v.encode() — or s3dol.Filepath(v) if you meant a " + "file's contents.", + DeprecationWarning, + stacklevel=2, + ) + v = v.encode() + self._inner[k] = v + + def __delitem__(self, k): + # v1 deletes are idempotent (no probe), which is exactly v0's + # "del store[absent] silently succeeds" — nothing to shim. + del self._inner[k] + + def __iter__(self): + return iter(self._inner) + + def __contains__(self, k): + if isinstance(k, str) and k.endswith(self._inner.delimiter): + # v0: `'d/' in s` answered via HeadObject on the marker; keep it. + return self._inner.__contains__(k) if k else False + return k in self._inner + + def __len__(self): + # v0's len() worked (by counting — two listings via the length hint). + # The v1 store raises; the shim keeps v0's behaviour, cost and all. + return sum(1 for _ in self._inner) + + def url_for(self, k: str, **kwargs) -> Optional[str]: + # lacing calls store.url_for(k) on the v0 store; forward to the v1 + # guarded shim (prefix-aware, SigV4 — this is what closes issue #10). + return self._inner.url_for(k, **kwargs) + + def __repr__(self): + return f"S3Store({self._inner!r})" + + +def _warn_if_resolution_changed(v0_args: dict) -> None: + """The ADR-0012 D7 runtime differential: one v0 argument-mangler run + through v1's own resolver twice; warn iff the outcomes differ. Cost when + env credentials are absent: two environ lookups.""" + branch, effective = _v0_mangle(v0_args, os.environ) + if effective is None or effective == v0_args: + return # excluded branches, or v0 didn't mangle anything + try: + environ = dict(os.environ) + v1_now = resolve(_spec_from_v0_args(v0_args), environ) + v0_was = resolve(_spec_from_v0_args(effective), environ) + except Exception: + return # the differential must never break the construction + moved = [ + axis + for axis in ("endpoint_url", "region_name") + if getattr(v1_now, axis).value != getattr(v0_was, axis).value + ] + if v1_now.credential_provenance != v0_was.credential_provenance: + moved.append("credential identity") + if moved: + warnings.warn( + f"s3dol v1 resolves this S3Store(...) call differently than v0 " + f"did (v0 branch: {branch}; changed: {', '.join(moved)}). v0 was " + f"silently discarding an explicit argument; v1 honours it. Run " + f"s3dol.diagnose(**your_S3Store_kwargs) for the full table.", + S3DolResolutionChanged, + stacklevel=3, + ) -# TODO: Add support for key-based grouping (store['folder/subfolder/'] is the store with path = original_path + 'folder/subfolder/' -# TODO: Make profile_name handling work for SupabaseS3BucketDol and S3BucketDolWithouBucketCheck -# TODO: Review this S3Store (with SupabaseS3BucketDol etc.) setup and see if we can make it simpler/cleaner def S3Store( bucket_name: str, *, - make_bucket: bool | None = None, - path: str | None = None, - aws_access_key_id: str = None, - aws_secret_access_key: str = None, - aws_session_token: str = None, - endpoint_url: str = None, - region_name: str = None, - profile_name: str = None, - skip_bucket_check: bool | None = None, - is_supabase_endpoint: bool | None = None, -) -> Store: - """S3 Bucket Store - - :param bucket_name: name of bucket to store data in - :param make_bucket: if True, create bucket if it does not exist. - If None, skip bucket existence checks completely. - If False, check for existence but don't create. - :param path: prefix to use for bucket keys - :param aws_access_key_id: AWS access key ID - :param aws_secret_access_key: AWS secret access key - :param aws_session_token: AWS session token - :param endpoint_url: URL of S3 endpoint - :param region_name: AWS region name - :param profile_name: AWS profile name - :return: S3BucketDol + make_bucket: Optional[bool] = None, + path: Optional[str] = None, + aws_access_key_id: Optional[str] = None, + aws_secret_access_key: Optional[str] = None, + aws_session_token: Optional[str] = None, + endpoint_url: Optional[str] = None, + region_name: Optional[str] = None, + profile_name: Optional[str] = None, + skip_bucket_check: Optional[bool] = None, + is_supabase_endpoint: Optional[bool] = None, +) -> MutableMapping: + """DEPRECATED — the v0 entry point, kept working until v2. + + Use :func:`s3dol.s3_store` instead: ``s3_store(bucket, prefix=...)``. + + Signature is v0's exactly (``path=`` not renamed; ``make_bucket`` + tri-state maps ``True -> on_missing_bucket='create'``, ``False -> + 'raise'``, ``None -> 'assume'``). Behaviour differences vs v0 are bug + fixes only (explicit endpoint/credentials honoured; listing errors raise + instead of returning ``[]``; writes no longer create buckets unless asked) + and announce themselves via ``S3DolResolutionChanged`` when they change + this call's resolution. """ - - if is_supabase_endpoint is None: - is_supabase_endpoint = _is_supabase_endpoint(endpoint_url) - if skip_bucket_check is None and is_supabase_endpoint: - skip_bucket_check = True - # For Supabase endpoints, create a direct client without session token - - params = dict( + warnings.warn( + "s3dol.store.S3Store is deprecated and will be removed in s3dol 2.0: " + "use s3dol.s3_store(bucket, prefix=..., ...) (note: prefix=, not " + "path=).", + DeprecationWarning, + stacklevel=2, + ) + v0_args = dict( + bucket_name=bucket_name, + make_bucket=make_bucket, + path=path, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, aws_session_token=aws_session_token, endpoint_url=endpoint_url, region_name=region_name, - ) - - if is_supabase_endpoint: - # For Supabase endpoints, use a custom store that doesn't check for bucket existence - return SupabaseS3BucketDol.from_params( - **params, - bucket_name=bucket_name, - path=path, - ) - elif skip_bucket_check: - return S3BucketDolWithouBucketCheck.from_params( - **params, - bucket_name=bucket_name, - prefix=path, - ) - - # For standard AWS, use the regular flow - - s3cr = S3ClientDol( - **params, profile_name=profile_name, + skip_bucket_check=skip_bucket_check, + is_supabase_endpoint=is_supabase_endpoint, ) + _warn_if_resolution_changed(v0_args) - bucket = s3cr[bucket_name] - - if make_bucket is None: - bucket.skip_bucket_exists_check = True - elif make_bucket is True and bucket_name not in s3cr: - s3cr[bucket_name] = {} - - if path: - # Apply the key prefix in the standard flow too. It was documented - # (":param path: prefix to use for bucket keys") but silently dropped - # here — only the Supabase / skip-check branches honored it. Normalize - # exactly as S3BucketReader.__post_init__ does. #7 follow-up. - bucket.prefix = f"{path.strip(bucket.delimiter)}{bucket.delimiter}" - - return bucket - - -def _is_supabase_endpoint(endpoint_url: str) -> bool: - """Check if the endpoint URL is a Supabase endpoint""" - return endpoint_url and ".supabase." in endpoint_url - - -def validate_bucket(bucket_name: str, s3_client: S3ClientDol, make_bucket: bool | None): - """Validate bucket name and create if needed - - If make_bucket is None, skip existence check entirely. - If make_bucket is True, create bucket if it doesn't exist. - If make_bucket is False, check existence but don't create. - """ - if make_bucket is None: - # Skip validation - just return a bucket object without checking existence - return s3_client[bucket_name] - elif make_bucket is True and bucket_name not in s3_client: - s3_client[bucket_name] = {} - return s3_client[bucket_name] - - -class S3BucketDolWithouBucketCheck(S3BucketDol): - """A S3BucketDol that completely avoids the bucket exists check. - This is needed for Supabase endpoints where the bucket is not created - until the first object is uploaded, for example. - """ + if is_supabase_endpoint is None: + is_supabase_endpoint = bool(endpoint_url and ".supabase." in endpoint_url) - def __setitem__(self, k, v): - _id = self._id_of_key(k) # TODO: Smelly. use trans tools - self.client.put_object(Bucket=self.bucket_name, Key=_id, Body=v) - - def _bucket_exists(self): - return True - - @classmethod - def from_params( - cls, - bucket_name: str, - *, - path: str | None = None, - aws_access_key_id: str = None, - aws_secret_access_key: str = None, - aws_session_token: str = None, - region_name: str = None, - endpoint_url: str = None, - ): - """Create a S3BucketDol without session token""" - client = boto3.client( - "s3", - aws_access_key_id=aws_access_key_id, - aws_secret_access_key=aws_secret_access_key, - endpoint_url=endpoint_url, - aws_session_token=aws_session_token, - region_name=region_name, + connection = _spec_from_v0_args(v0_args) + if is_supabase_endpoint: + # Route through the preset row: path addressing + checksum + # when_required — the WRITE-side cure for what the read-side + # de-chunker treats (ADR-0003 §3). + from s3dol.connection import S3Connection + + connection = S3Connection( + preset="supabase", + endpoint_url=connection.endpoint_url, + region_name=connection.region_name, + profile=connection.profile, + credentials=connection.credentials, ) - - return cls(client=client, bucket_name=bucket_name, prefix=path) - - -# TODO: Messy. Should use wrap_kvs. -class SupabaseS3BucketDol(S3BucketDolWithouBucketCheck): - """ - S3BucketDol for Supabase endpoints. - Over the S3BucketDolWithouBucketCheck, it adds support to unravel the returned - content from the stuff (number of bytes, check sum, etc.) superbase adds to the - content. - """ - - def __getitem__(self, k): - _id = self._id_of_key(k) # TODO: Smelly. use trans tools - - # Handle nested key access for keys ending with delimiter - if _id.endswith(self.delimiter): - _kw = {**self.__dict__, "prefix": _id} - return type(self)(**_kw) - - try: - response = self.client.get_object(Bucket=self.bucket_name, Key=_id) - raw_data = response["Body"].read() - - # Process HTTP chunked encoding directly on bytes - # Look for the pattern: [chunk size in hex]\r\n[data]\r\n0\r\n[optional headers]\r\n\r\n - if ( - raw_data.startswith(b"0") - or raw_data[0:2].isdigit() - or ( - raw_data[0:1].isdigit() - and raw_data[1:2].isalpha() - and raw_data[1:2].lower() in b"abcdef" - ) - ): - # Find the first CRLF - first_crlf_pos = raw_data.find(b"\r\n") - if first_crlf_pos != -1: - # Extract what should be the hex chunk size - hex_size = raw_data[:first_crlf_pos] - try: - # Skip the chunk size and the CRLF - content_start = first_crlf_pos + 2 - # Find the end of the chunk (marked by another CRLF) - content_end = raw_data.find(b"\r\n", content_start) - if content_end != -1: - # Extract just the content between the CRLFs - return raw_data[content_start:content_end] - except ValueError: - pass # Not valid hex, continue to return raw data - - # If we couldn't parse it as chunked encoding or any step failed, - # return the raw data as a fallback - return raw_data - - except self.client.exceptions.NoSuchKey as ex: - raise KeyError(f"Key {k} does not exist") from ex + on_missing_bucket = {True: "create", False: "raise", None: "assume"}[make_bucket] + inner = BucketStore( + bucket_name, + connection=connection, + prefix=path or "", + on_missing_bucket=on_missing_bucket, + ) + return _V0CompatStore(inner, dechunk=is_supabase_endpoint) diff --git a/s3dol/testing.py b/s3dol/testing.py new file mode 100644 index 0000000..c651d62 --- /dev/null +++ b/s3dol/testing.py @@ -0,0 +1,426 @@ +"""``s3dol.testing`` — a shipped in-memory fake and the exported conformance +suite (ADR-0008). Downstream users can test services that take a store as a +dependency with **no network, no docker, no moto**:: + + from s3dol.testing import mock_s3 + + def test_my_service(): + assert MediaService(mock_s3()).url('a') + +The fake is not a re-implementation: :class:`FakeBucketStore` *is* the real +:class:`~s3dol.base.BucketStore` with only the backend protocol (the ``_op_*`` +seam of ADR-0002) swapped for a dict — so the prefix arithmetic, key validity, +error taxonomy (the fake raises *synthesized botocore ClientErrors* through +the same translation seam), guards and Mapping semantics are structurally +identical, and drift fails the same conformance suite that gates the real +store. + +The suite is parameterized by capability flags: ``mock_s3()`` runs the +structural presign assertion (2a) but not the fetch-it assertion (2b — there +is no HTTP server), so the honest claim is *"the same suite, parameterized by +an ``endpoint_is_fetchable`` capability"* (ADR-0008). +""" + +from __future__ import annotations + +import pickle +from datetime import datetime, timezone +from typing import Callable, Optional +from urllib.parse import quote + +from s3dol.base import BucketStore, ObjectInfo, clamp_expires_in +from s3dol.capabilities import BucketHandles, BucketInfo, BucketUrls +from s3dol.errors import KeyNotValid, ObjectNotFound, S3Error +from s3dol.values import as_fileobj, is_bytes_like, reject_str_value + + +def _client_error(code: str, status: int, operation: str): + """A synthesized botocore ClientError, exactly like the wire produces.""" + from botocore.exceptions import ClientError + + return ClientError( + { + "Error": {"Code": code, "Message": "synthesized by s3dol.testing"}, + "ResponseMetadata": {"HTTPStatusCode": status}, + }, + operation, + ) + + +class _FakeBackend: + """The shared in-memory 'bucket': absolute (wire) keys -> bytes.""" + + def __init__(self, data: Optional[dict] = None, *, bucket_exists: bool = True): + self.data: dict = dict(data or {}) + self.bucket_exists = bucket_exists + + +class _FakeOps: + """Mixin overriding exactly the ``_op_*`` backend protocol with dict + operations that raise synthesized ``ClientError``s — everything above the + seam (guards, taxonomy, key arithmetic) runs unchanged.""" + + def __init__(self, *args, backing: Optional[_FakeBackend] = None, **kwargs): + self._backing = backing if backing is not None else _FakeBackend() + super().__init__(*args, **kwargs) + + # the fake never builds a client + @property + def client(self): + raise S3Error( + "s3dol.testing fakes have no boto3 client — a code path reached " + "the backend outside the _op_* protocol seam (that is a bug worth " + "reporting)." + ) + + def _check_bucket(self, operation): + if not self._backing.bucket_exists: + raise _client_error("NoSuchBucket", 404, operation) + + def _op_list_pages( + self, *, delimiter: Optional[str] = None, prefix: Optional[str] = None + ): + self._check_bucket("ListObjectsV2") + base = self.prefix if prefix is None else prefix + keys = sorted(k for k in self._backing.data if k.startswith(base)) + if delimiter is None: + contents = [ + { + "Key": key, + "Size": len(self._backing.data[key]), + "LastModified": datetime.now(timezone.utc), + "ETag": '"fake"', + } + for key in keys + ] + yield {"Contents": contents} + return + seen, prefixes = set(), [] + for key in keys: + rest = key[len(base) :] + if delimiter in rest.rstrip(delimiter): + head = rest.split(delimiter, 1)[0] + delimiter + if head not in seen: + seen.add(head) + prefixes.append({"Prefix": f"{base}{head}"}) + yield {"CommonPrefixes": prefixes} + + def _op_head_object(self, _id): + self._check_bucket("HeadObject") + if _id not in self._backing.data: + # moto returns a body here; real AWS does not. The fake mimics + # REAL AWS (code synthesized from the status) so the ambiguity + # path of ADR-0004 §2 is exercised, which moto cannot do. + raise _client_error("404", 404, "HeadObject") + value = self._backing.data[_id] + return { + "ContentLength": len(value), + "LastModified": datetime.now(timezone.utc), + "ETag": '"fake"', + "ContentType": "binary/octet-stream", + } + + def _op_head_bucket(self): + if not self._backing.bucket_exists: + raise _client_error("404", 404, "HeadBucket") + + def _op_read(self, _id): + self._check_bucket("GetObject") + if _id not in self._backing.data: + raise _client_error("NoSuchKey", 404, "GetObject") + return self._backing.data[_id] + + def _op_write(self, _id, value): + self._check_bucket("PutObject") + if isinstance(value, str): + reject_str_value(value) + if is_bytes_like(value): + self._backing.data[_id] = bytes(value) + else: + with as_fileobj(value) as fileobj: + self._backing.data[_id] = fileobj.read() + + def _op_delete(self, _id): + self._check_bucket("DeleteObject") + self._backing.data.pop(_id, None) # idempotent, like the wire + + def _op_bulk_delete(self, _ids): + self._check_bucket("DeleteObjects") + deleted = [] + for _id in _ids: + self._backing.data.pop(_id, None) + deleted.append(_id) # absent keys report as Deleted, like S3 + return deleted, [] + + def _op_create_bucket(self): + self._backing.bucket_exists = True + + def _op_presign(self, _id, *, expires_in, client_method, **params): + if self.connection.anon: + return None + expires_in = clamp_expires_in(expires_in) # same guard as the real one + return ( + f"https://{self.bucket}.mock-s3.invalid/{quote(_id)}" + f"?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Expires={expires_in}" + f"&X-Amz-Signature=mock" + ) + + def _sibling_kwargs(self): + return super()._sibling_kwargs() + + def _make_sibling(self, kind, **options): + sibling_class = { + "handles": FakeBucketHandles, + "urls": FakeBucketUrls, + "info": FakeBucketInfo, + }[kind] + return sibling_class( + **{**super()._sibling_kwargs(), **options}, backing=self._backing + ) + + def _with(self, **overrides): + kwargs = super()._sibling_kwargs() + kwargs["on_missing_bucket"] = self.on_missing_bucket + kwargs.update(overrides) + return type(self)(kwargs.pop("bucket"), backing=self._backing, **kwargs) + + +class FakeObjectHandle: + """The fake's per-object escape hatch (mirrors ``ObjectHandle``).""" + + def __init__(self, bucket: str, key: str, *, backing: _FakeBackend, anon=False): + self.bucket = bucket + self.key = key + self._backing = backing + self._anon = anon + + def __repr__(self): + return f"{type(self).__name__}({self.bucket!r}, {self.key!r})" + + def _bytes(self) -> bytes: + if self.key not in self._backing.data: + raise ObjectNotFound(f"GetObject failed: key={self.key!r}") + return self._backing.data[self.key] + + def read(self, *, offset=None, length=None) -> bytes: + data = self._bytes() + if offset is None and length is None: + return data + start = offset or 0 + return data[start : None if length is None else start + length] + + def stream(self, *, chunk_size=8 * 2**20): + data = self._bytes() + for start in range(0, len(data), chunk_size): + yield data[start : start + chunk_size] + + def download_to(self, path) -> str: + from s3dol.values import Filepath + + target = path.path if isinstance(path, Filepath) else str(path) + with open(target, "wb") as f: + f.write(self._bytes()) + return target + + def write(self, value) -> None: + if isinstance(value, str): + reject_str_value(value) + if is_bytes_like(value): + self._backing.data[self.key] = bytes(value) + else: + with as_fileobj(value) as fileobj: + self._backing.data[self.key] = fileobj.read() + + def delete(self) -> None: + self._backing.data.pop(self.key, None) + + def exists(self) -> bool: + return self.key in self._backing.data + + def info(self) -> ObjectInfo: + return ObjectInfo( + key=self.key, + size=len(self._bytes()), + last_modified=datetime.now(timezone.utc), + etag='"fake"', + content_type="binary/octet-stream", + storage_class="STANDARD", + ) + + def url(self, *, expires_in=3600, client_method="get_object", **params): + if self._anon: + return None + return ( + f"https://{self.bucket}.mock-s3.invalid/{quote(self.key)}" + f"?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Expires={expires_in}" + f"&X-Amz-Signature=mock" + ) + + +class FakeBucketStore(_FakeOps, BucketStore): + """The real ``BucketStore`` over a dict backend.""" + + +class FakeBucketHandles(_FakeOps, BucketHandles): + def __getitem__(self, k): + return FakeObjectHandle( + self.bucket, + self._id_of_key(k), + backing=self._backing, + anon=self.connection.anon, + ) + + +class FakeBucketUrls(_FakeOps, BucketUrls): + def __getitem__(self, k): + return self._op_presign( + self._id_of_key(k), + expires_in=self.expires_in, + client_method=self.client_method, + ) + + +class FakeBucketInfo(_FakeOps, BucketInfo): + def __getitem__(self, k): + _id = self._id_of_key(k) + if _id not in self._backing.data: + raise ObjectNotFound(f"HeadObject failed: key={k!r}") + return ObjectInfo( + key=_id, + size=len(self._backing.data[_id]), + last_modified=datetime.now(timezone.utc), + etag='"fake"', + content_type="binary/octet-stream", + storage_class="STANDARD", + ) + + +def mock_s3( + bucket: str = "mock-bucket", + *, + prefix: str = "", + data: Optional[dict] = None, + bucket_exists: bool = True, + anon: bool = False, +) -> FakeBucketStore: + """An in-process fake ``BucketStore``: no network, no docker, no moto. + + ``data`` seeds the bucket with *absolute* (wire) keys -> bytes. + + >>> s = mock_s3(data={'logs/a.txt': b'hi', 'other/z': b'no'}, prefix='logs') + >>> list(s), s['a.txt'] + (['a.txt'], b'hi') + >>> s['b'] = b'new'; sorted(s) + ['a.txt', 'b'] + """ + from s3dol.connection import S3Connection + + return FakeBucketStore( + bucket, + backing=_FakeBackend(data, bucket_exists=bucket_exists), + connection=S3Connection(anon=anon) if anon else S3Connection(), + prefix=prefix, + ) + + +# --------------------------------------------------------------------------- # +# The exported conformance suite (ADR-0008) +# --------------------------------------------------------------------------- # + + +def run_conformance( + make_store: Callable[[], object], + *, + endpoint_is_fetchable: bool = False, + normalize: Callable = lambda v: v, + check_pickle: bool = True, + check_urls: bool = True, + secret: Optional[str] = None, +) -> None: + """Assert the s3dol store laws against a fresh store from ``make_store``. + + ``make_store`` must return an EMPTY, writable store (each call may return + a new one over the same empty backend). ``normalize`` is the per-layer + canonical form of ADR-0005 N3 (identity for Layer B; ``json.loads ∘ + json.dumps`` for a json recipe, …). ``secret``, when given, is asserted + absent from every repr this suite touches. + + Sibling packages and user code are welcome to run this against their own + stores — that is why it ships in the package rather than in ``tests/``. + """ + from urllib.parse import urlsplit + + store = make_store() + + # -- Mapping laws + the value law (N3) + store["a"] = b"alpha" + store["b/c"] = b"beta" + assert store["a"] == normalize(b"alpha") + assert sorted(store) == ["a", "b/c"] + assert "a" in store and "b/c" in store + assert "missing" not in store + # iter/contains agreement — the law EncodingType breaks (ADR-0006 §4) + assert all(k in store for k in store) + try: + store["missing"] + raise AssertionError("reading an absent key must raise KeyError") + except KeyError: + pass + assert store.get("missing") is None # the defaulted accessor degrades + del store["b/c"] + assert "b/c" not in store + del store["b/c"] # idempotent (documented; costs no probe) + + # -- len() raises with guidance (ADR-0008 cost model) + try: + len(store) + raise AssertionError("len(store) must raise TypeError") + except TypeError: + pass + + # -- keys that must round-trip (and KeyNotValid before the wire) + for key in ["plain", "a b", "café", "a+b", "a\rb", "p/x y", "a%20b"]: + store[key] = b"v" + assert all(k in store for k in store) + for bad in ["", 123, "x" * 2000]: + try: + store[bad] = b"v" + raise AssertionError(f"key {bad!r} must be rejected") + except (KeyNotValid, KeyError, ValueError, TypeError): + pass + + # -- str values are rejected, loudly (ADR-0005 §2) + try: + store["s"] = "a string" + raise AssertionError("str values must be rejected") + except TypeError: + pass + + # -- url assertions (2a: structural, never substring — ADR-0008) + if check_urls: + url_for = getattr(store, "url_for", None) + if callable(url_for): + url = url_for("a") + if url is not None: + parts = urlsplit(url) + assert "X-Amz-Algorithm=AWS4-HMAC-SHA256" in (parts.query or ""), url + assert "AWSAccessKeyId" not in url, f"SigV2 tell in {url!r}" + assert parts.path.split("/")[-1] in ("a", quote("a")), ( + f"URL path must address the key: {url!r}" + ) + if endpoint_is_fetchable: + import requests + + assert requests.get(url).content == normalize(b"alpha") + + # -- pickling (ADR-0008 §6): a store must cross a process boundary + if check_pickle: + clone = pickle.loads(pickle.dumps(store)) + assert sorted(clone) == sorted(store) + + # -- no secret in repr (ADR-0008 §7) + if secret is not None: + assert secret not in repr(store) + connection = getattr(store, "connection", None) + if connection is not None: + assert secret not in repr(connection) diff --git a/s3dol/tests/test_base.py b/s3dol/tests/test_base.py index f6a61c9..9ca279b 100644 --- a/s3dol/tests/test_base.py +++ b/s3dol/tests/test_base.py @@ -1,190 +1,391 @@ -import os -import pytest +"""Layer B: prefix arithmetic, key validity, contracts, guards. + +Mostly tier 1 against the in-memory fake (which *is* the real store with only +the backend protocol swapped), plus a moto section for what only a real client +can show. +""" + +import pickle +import warnings -from s3dol.base import S3BucketDol, S3ClientReader, S3Dol, S3ClientDol, S3DolReadOnly +import pytest +from dol import KeyCodecs, Pipe, filt_iter + +from s3dol.base import ( + MAX_KEY_BYTES, + BucketStore, + ObjectHandle, + _normalize_prefix, + _validate_key, +) +from s3dol.errors import ( + BucketNotFound, + ConfigurationError, + CredentialsError, + KeyNotValid, + NotSupported, + ObjectNotFound, +) +from s3dol.testing import FakeBucketStore, _FakeBackend, mock_s3 -def overwrite_aws_environment_variables( - aws_access_key_id, aws_secret_access_key, endpoint_url -): - os.environ["AWS_ACCESS_KEY_ID"] = aws_access_key_id - os.environ["AWS_SECRET_ACCESS_KEY"] = aws_secret_access_key - os.environ["AWS_ENDPOINT_URL"] = endpoint_url +# --------------------------------------------------------------------------- # +# prefix arithmetic (ADR-0006 §1) — the boundary the whole scoping story rests on +# --------------------------------------------------------------------------- # @pytest.mark.parametrize( - "aws_access_key_id, aws_secret_access_key, endpoint_url", - [("localstack", "localstack", "http://localhost:4566")], + "given, expected", + [("logs", "logs/"), ("logs/", "logs/"), ("/logs/", "logs/"), ("", ""), (None, "")], ) -def test_s3_dol_crud(aws_access_key_id, aws_secret_access_key, endpoint_url): - overwrite_aws_environment_variables( - aws_access_key_id, aws_secret_access_key, endpoint_url - ) +def test_prefix_normalisation_is_mandatory_and_first(given, expected): + assert _normalize_prefix(given) == expected + + +def test_unnormalised_prefix_cannot_leak_a_sibling_tenant(): + # The one-character boundary violation ADR-0006 §1 exists to prevent: + # prefix='logs' must NOT expose 'logs2/2026.txt' as key '2/2026.txt'. + data = {"logs/2026.txt": b"mine", "logs2/2026.txt": b"NEIGHBOUR", "logsX": b"x"} + store = mock_s3(data=data, prefix="logs") + assert sorted(store) == ["2026.txt"] + assert store["2026.txt"] == b"mine" + assert "2/2026.txt" not in store + with pytest.raises(KeyError): + store["2/2026.txt"] + + +def test_scoping_round_trip_with_siblings_markers_and_non_matching_keys(): + data = { + "a/b": b"1", + "a/c": b"2", + "z": b"3", # non-matching + "ab/x": b"4", # sibling prefix + "a/": b"", # the exact-prefix marker + } + store = mock_s3(data=data, prefix="a/") + assert sorted(store) == ["b", "c"] # marker filtered; siblings excluded + assert all(k in store for k in store) + for key in store: + assert store._key_of_id(store._id_of_key(key)) == key - test_bucket_name = "test-bucket" - test_key = "test-key" - test_value = b"test-value" - - s3 = S3Dol() - assert "environment variables" in s3 - profile_names = list(s3) - assert "environment variables" in profile_names - - # create client from profile - s3_client = s3["environment variables"] - # create bucket - if test_bucket_name in s3_client: - del s3_client[test_bucket_name] - assert test_bucket_name not in s3_client - - s3_client[test_bucket_name] = {} - s3_bucket = s3_client[test_bucket_name] - assert test_bucket_name in s3_client - bucket_names = list(s3_client) - assert test_bucket_name in bucket_names - n_obj = len(list(s3_bucket)) - - # create and delete object - assert test_key not in s3_bucket - s3_bucket[test_key] = test_value - assert test_key in s3_bucket - assert test_key in list(s3_bucket) - assert s3_bucket[test_key] == test_value - assert s3["environment variables"][test_bucket_name][test_key] == test_value - assert len(list(s3_bucket)) == n_obj + 1 - del s3_bucket[test_key] - assert test_key not in s3_bucket - assert list(s3_bucket) == [] - assert len(list(s3_bucket)) == n_obj - - # delete bucket - del s3_client[test_bucket_name] - assert test_bucket_name not in s3_client - assert test_bucket_name not in list(s3_client) - - # delete bucket with object - s3_client[test_bucket_name] = {} - s3_bucket = s3_client[test_bucket_name] - s3_bucket[test_key] = test_value - del s3_client[test_bucket_name] - assert test_bucket_name not in s3_client - - # access bucket object with path delimiations - s3_client[test_bucket_name] = {} - s3_bucket = s3_client[test_bucket_name] - s3_bucket["level1/level2/test-key"] = test_value - - assert_bucket_key_value(s3_bucket, "level1/level2/test-key", test_value) - assert_bucket_key_value(s3_bucket["level1/level2/"], "test-key", test_value) - assert_bucket_key_value(s3_bucket["level1/"]["level2/"], "test-key", test_value) - assert_bucket_key_value(s3_bucket["level1/"], "level2/test-key", test_value) - - # delete object with path delimiations - del s3_bucket["level1/"]["level2/"]["test-key"] - assert "level1/level2/test-key" not in s3_bucket - - -def assert_bucket_key_value(s3_bucket, key, value): - assert key in s3_bucket, f"failed contains test with key: {key}" - assert key in list(s3_bucket), f"failed iter test with key: {key}" - assert s3_bucket[key] == value, f"failed get test with key: {key}" - - -def mk_s3_client_from_env( - aws_access_key_id, aws_secret_access_key, endpoint_url, s3_client_class -): - overwrite_aws_environment_variables( - aws_access_key_id, aws_secret_access_key, endpoint_url - ) - return s3_client_class() +def test_key_of_id_raises_rather_than_slicing_a_non_matching_id(): + # The belt to the server-side Prefix's suspenders: a provider that ignores + # Prefix must not silently produce corrupt keys. + store = mock_s3(prefix="a/") + with pytest.raises(KeyNotValid, match="outside"): + store._key_of_id("zzz/other") -def mk_s3_client_from_params( - aws_access_key_id, aws_secret_access_key, endpoint_url, s3_client_class -): - return s3_client_class( - aws_access_key_id=aws_access_key_id, - aws_secret_access_key=aws_secret_access_key, - endpoint_url=endpoint_url, - ) +def test_exact_prefix_marker_is_hidden_from_its_own_scoped_view(): + # The documented wart: it would relativise to '', which is not a valid key. + store = mock_s3(data={"a/": b"", "a/b": b"1"}, prefix="a/") + assert sorted(store) == ["b"] + # ...and stays addressable by absolute key on an unscoped store. + assert sorted(mock_s3(data={"a/": b"", "a/b": b"1"})) == ["a/", "a/b"] -@pytest.mark.parametrize( - "aws_access_key_id, aws_secret_access_key, endpoint_url", - [("localstack", "localstack", "http://localhost:4566")], -) -@pytest.mark.parametrize( - "mk_s3_client", [mk_s3_client_from_env, mk_s3_client_from_params] -) -def test_s3_client( - aws_access_key_id, aws_secret_access_key, endpoint_url, mk_s3_client -): - test_bucket_name = "test-bucket" - - s3_client = mk_s3_client( - aws_access_key_id=aws_access_key_id, - aws_secret_access_key=aws_secret_access_key, - endpoint_url=endpoint_url, - s3_client_class=S3ClientDol, + +# --------------------------------------------------------------------------- # +# key validity, before the wire (ADR-0006 §4) +# --------------------------------------------------------------------------- # + + +@pytest.mark.parametrize("bad", ["", 123, None, b"bytes"]) +def test_invalid_keys_are_keynotvalid_not_backend_types(bad): + with pytest.raises(KeyNotValid): + _validate_key(bad) + + +def test_key_length_limit_is_utf8_bytes_not_characters(): + _validate_key("é" * (MAX_KEY_BYTES // 2)) # 1024 bytes exactly: fine + with pytest.raises(KeyNotValid, match="UTF-8 bytes"): + _validate_key("é" * (MAX_KEY_BYTES // 2 + 1)) + + +def test_lone_surrogate_key_is_rejected_locally(): + with pytest.raises(KeyNotValid, match="UTF-8"): + _validate_key("bad\ud800key") + + +def test_keynotvalid_is_both_keyerror_and_valueerror(): + store = mock_s3() + with pytest.raises(KeyError): + store[""] + with pytest.raises(ValueError): + store[""] + + +# --------------------------------------------------------------------------- # +# Mapping contracts (architecture.md table) +# --------------------------------------------------------------------------- # + + +def test_len_raises_with_guidance_rather_than_counting(): + with pytest.raises(TypeError, match="sum\\(1 for _ in store\\)"): + len(mock_s3()) + + +def test_delete_is_idempotent_by_default_and_probes_only_when_asked(): + store = mock_s3(data={"a": b"1"}) + del store["a"] + del store["a"] # idempotent: S3 returns 204 for an absent key + strict = FakeBucketStore( + "mock-bucket", backing=_FakeBackend({"a": b"1"}), strict_delete=True ) + del strict["a"] + with pytest.raises(KeyError): + del strict["a"] + + +def test_missing_bucket_raises_on_iteration_never_yields_empty(): + store = FakeBucketStore("mock-bucket", backing=_FakeBackend(bucket_exists=False)) + with pytest.raises(BucketNotFound): + list(store) + + +def test_on_missing_bucket_assume_performs_no_probe(): + # 'assume' must never call HeadBucket at construction (ADR-0010 §1). + backend = _FakeBackend(bucket_exists=False) + FakeBucketStore("mock-bucket", backing=backend) # constructs fine + with pytest.raises(BucketNotFound): + FakeBucketStore("mock-bucket", backing=backend, on_missing_bucket="raise") + + +def test_on_missing_bucket_create_recovers_without_probing(): + backend = _FakeBackend(bucket_exists=False) + store = FakeBucketStore("mock-bucket", backing=backend, on_missing_bucket="create") + store["k"] = b"v" # attempt -> NoSuchBucket -> create -> retry once + assert store["k"] == b"v" and backend.bucket_exists + + +def test_setdefault_never_overwrites_an_archived_object(): + # MutableMapping.setdefault is try/except KeyError -> assign, so an + # ObjectArchived KeyError would silently overwrite the archived object. + from s3dol.errors import ObjectArchived + + class Archived(FakeBucketStore): + def _op_read(self, _id): + raise ObjectArchived("archived") + + store = Archived("mock-bucket", backing=_FakeBackend({"a": b"COLD"})) + with pytest.raises(ObjectArchived): + store.setdefault("a", b"REPLACEMENT") + assert store._backing.data["a"] == b"COLD" # untouched + with pytest.raises(ObjectArchived): + store.pop("a", b"default") + + +def test_stores_pickle(): + store = mock_s3(data={"p/a": b"1"}, prefix="p") + clone = pickle.loads(pickle.dumps(store)) + assert sorted(clone) == ["a"] and clone["a"] == b"1" + + +def test_repr_shows_scope_and_no_secrets(): + store = mock_s3(prefix="tenant-a") + text = repr(store) + assert "mock-bucket" in text and "tenant-a/" in text + + +# --------------------------------------------------------------------------- # +# writes: the value domain +# --------------------------------------------------------------------------- # - if test_bucket_name in s3_client: - del s3_client[test_bucket_name] - assert test_bucket_name not in s3_client - s3_client[test_bucket_name] = {} - s3_bucket = s3_client[test_bucket_name] - assert isinstance(s3_bucket, S3BucketDol) +def test_str_values_are_refused_at_the_store_boundary(): + with pytest.raises(TypeError, match="Filepath"): + mock_s3()["k"] = "a string" - s3_client_reader = mk_s3_client( - aws_access_key_id=aws_access_key_id, - aws_secret_access_key=aws_secret_access_key, - endpoint_url=endpoint_url, - s3_client_class=S3ClientReader, + +def test_filepath_ref_writes_file_content(tmp_path): + from s3dol.values import Filepath + + payload = tmp_path / "big.bin" + payload.write_bytes(b"CONTENT") + store = mock_s3() + store["k"] = Filepath(payload) + assert store["k"] == b"CONTENT" + + +def test_chunks_ref_concatenates(): + from s3dol.values import Chunks + + store = mock_s3() + store["k"] = Chunks(iter([b"ab", b"cd"])) + assert store["k"] == b"abcd" + + +# --------------------------------------------------------------------------- # +# anonymous stores refuse writes locally (ADR-0012 D5) +# --------------------------------------------------------------------------- # + + +def test_anonymous_store_refuses_writes_locally(): + store = mock_s3(anon=True) + with pytest.raises(CredentialsError): + store["k"] = b"v" + with pytest.raises(CredentialsError): + del store["k"] + + +def test_anonymous_url_for_returns_none_not_raises(): + # SupportsUrlFor is Optional[str]; raising would turn a public-bucket + # streaming fallback into a 500 (ADR-0012 D5). + assert mock_s3(data={"k": b"v"}, anon=True).url_for("k") is None + + +# --------------------------------------------------------------------------- # +# url_for: the single guarded keyed method (ADR-0011 D3b) +# --------------------------------------------------------------------------- # + + +def test_url_for_unwrapped_addresses_the_prefixed_key(): + store = mock_s3(data={"p/f": b"v"}, prefix="p") + assert "p/f" in store.url_for("f") + + +def test_url_for_refuses_once_the_leaf_has_been_wrapped(): + # The finding behind ADR-0011 D3b's revision: TWO call paths reach this + # one signature with different key domains — dol.content_url pre-resolves + # the key through the outer layers and stops at the provider "because that + # layer applies its own" (dol/content.py), while a direct delegated call + # hands over the raw outer key. Indistinguishable here, and a wrong + # presigned URL addresses a different REAL object, so: refuse. + for make_call in ( + lambda s: KeyCodecs.prefixed("x/")(s).url_for("g"), # temporary wrapper + lambda s: _wrap_then_call(s), # named wrapper + ): + store = mock_s3(data={"p/x/g": b"v"}, prefix="p") + with pytest.raises(NotSupported, match="s3dol.urls"): + make_call(store) + + +def _wrap_then_call(store): + wrapper = KeyCodecs.prefixed("x/")(store) + return wrapper.url_for("g") + + +def test_url_for_refuses_even_after_the_wrapper_is_collected(): + # A delegated bound method holds no reference to its wrapper, so the + # weakref-based wrapped_self degrades to "never wrapped" here (ADR-0011 + # D1a). The probe-descriptor flag survives that, which is what makes the + # refusal reliable rather than intermittent. + store = mock_s3(data={"p/f": b"v"}, prefix="p") + wrapper = KeyCodecs.prefixed("x/")(store) + del wrapper + with pytest.raises(NotSupported, match="s3dol.urls"): + store.url_for("f") + + +def test_dol_content_url_is_correct_for_an_unwrapped_store(): + # The reason url_for exists at all (dol.SupportsUrlFor is a *method* and + # content_url reaches it by getattr). + from dol import content_url + + store = mock_s3(data={"p/g": b"v"}, prefix="p") + assert "p/g" in content_url(store, "g") + + +def test_the_sibling_store_is_correct_where_url_for_must_refuse(): + # ...and the remedy the refusal names is correct at any depth, because it + # goes through __getitem__ (ADR-0011 D2). + import s3dol + + store = mock_s3(data={"p/x/g": b"v"}, prefix="p") + wrapped_data, wrapped_urls = ( + KeyCodecs.prefixed("x/")(store), + KeyCodecs.prefixed("x/")(s3dol.urls(store)), ) - s3_bucket = s3_client_reader[test_bucket_name] - assert isinstance(s3_bucket, S3BucketDol) + assert wrapped_data["g"] == b"v" + assert "p/x/g" in wrapped_urls["g"] - if test_bucket_name in s3_client: - del s3_client[test_bucket_name] - assert test_bucket_name not in s3_client_reader +def test_url_for_clamps_and_warns_past_sigv4s_cap(): + store = mock_s3(data={"k": b"v"}) + with pytest.warns(UserWarning, match="604800"): + url = store.url_for("k", expires_in=999_999) + assert "X-Amz-Expires=604800" in url -@pytest.mark.parametrize( - "aws_access_key_id, aws_secret_access_key, endpoint_url", - [("localstack", "localstack", "http://localhost:4566")], -) -def test_s3_dol_readonly(aws_access_key_id, aws_secret_access_key, endpoint_url): - overwrite_aws_environment_variables( - aws_access_key_id, aws_secret_access_key, endpoint_url + +# --------------------------------------------------------------------------- # +# ObjectHandle binds its key at construction +# --------------------------------------------------------------------------- # + + +def test_handle_reads_ranges_streams_and_downloads(tmp_path): + from s3dol.testing import FakeObjectHandle + + handle = FakeObjectHandle( + "b", "k", backing=_FakeBackend({"k": b"the quick brown fox"}) ) + assert handle.read() == b"the quick brown fox" + assert handle.read(offset=4, length=5) == b"quick" + assert b"".join(handle.stream(chunk_size=4)) == b"the quick brown fox" + target = handle.download_to(tmp_path / "out.bin") + assert open(target, "rb").read() == b"the quick brown fox" + assert handle.exists() and handle.info().size == 19 - test_bucket_name = "test-bucket" - test_key = "test-key" - test_key2 = "test-key2" - test_value = b"test-value" - s3 = S3Dol() - s3["environment variables"][test_bucket_name] = { - test_key: test_value, - test_key2: test_value, - } +def test_handle_missing_object_raises_keyerror(): + from s3dol.testing import FakeObjectHandle + + handle = FakeObjectHandle("b", "nope", backing=_FakeBackend()) + with pytest.raises(ObjectNotFound): + handle.read() + assert handle.exists() is False + + +# --------------------------------------------------------------------------- # +# construction validation +# --------------------------------------------------------------------------- # + + +@pytest.mark.parametrize("bad_bucket", ["", None, 42]) +def test_bucket_must_be_a_non_empty_str(bad_bucket): + with pytest.raises(ConfigurationError): + mock_s3(bad_bucket) - s3_readonly = S3DolReadOnly()["environment variables"] - assert test_bucket_name in s3_readonly - assert test_key in s3_readonly[test_bucket_name] - assert test_key2 in s3_readonly[test_bucket_name] - assert s3_readonly[test_bucket_name][test_key] == test_value - with pytest.raises(TypeError): - s3_readonly[test_bucket_name][test_key] = test_value +def test_unknown_on_missing_bucket_value_rejected(): + with pytest.raises(ConfigurationError, match="on_missing_bucket"): + FakeBucketStore("b", backing=_FakeBackend(), on_missing_bucket="probe") - with pytest.raises(TypeError): - del s3_readonly[test_bucket_name][test_key] - del s3["environment variables"][test_bucket_name][test_key] - assert test_key not in s3_readonly[test_bucket_name] +# --------------------------------------------------------------------------- # +# moto tier 2: what only a real client shows +# --------------------------------------------------------------------------- # -if __name__ == "__main__": - pytest.main([__file__]) +def test_real_store_round_trip_and_presign_is_sigv4(monkeypatch): + moto = pytest.importorskip("moto") + for var in ("AWS_ENDPOINT_URL", "AWS_ENDPOINT_URL_S3", "AWS_PROFILE"): + monkeypatch.delenv(var, raising=False) + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "testing") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "testing") + monkeypatch.setenv("AWS_DEFAULT_REGION", "us-east-1") + from s3dol.connection import S3Connection + + with moto.mock_aws(): + connection = S3Connection(region_name="us-east-1") + connection.client.create_bucket(Bucket="real-bucket") + store = BucketStore("real-bucket", connection=connection, prefix="p") + store["a"] = b"alpha" + assert store["a"] == b"alpha" and sorted(store) == ["a"] + url = store.url_for("a") + assert "X-Amz-Algorithm=AWS4-HMAC-SHA256" in url and "/p/a" in url + assert "AWSAccessKeyId" not in url + # the object really is at the prefixed key + raw = connection.client.get_object(Bucket="real-bucket", Key="p/a") + assert raw["Body"].read() == b"alpha" + + +def test_real_store_pickles_across_a_process_boundary(monkeypatch): + pytest.importorskip("moto") + from s3dol.connection import S3Connection + + store = BucketStore( + "real-bucket", connection=S3Connection(region_name="us-east-1"), prefix="p" + ) + clone = pickle.loads(pickle.dumps(store)) + assert clone.bucket == store.bucket and clone.prefix == store.prefix + assert clone.connection == store.connection diff --git a/s3dol/tests/test_capabilities.py b/s3dol/tests/test_capabilities.py new file mode 100644 index 0000000..060cefe --- /dev/null +++ b/s3dol/tests/test_capabilities.py @@ -0,0 +1,221 @@ +"""Sibling capability stores and free functions (ADR-0011 D2/D3/D4).""" + +import pytest +from dol import KeyCodecs, Pipe, cached_keys, filt_iter + +import s3dol +from s3dol.errors import ConfigurationError, CredentialsError, S3PartialFailure +from s3dol.testing import mock_s3 + +# --------------------------------------------------------------------------- # +# D2 — sibling stores are correct BY CONSTRUCTION (they route through +# __getitem__, the one thing dol maps correctly at every depth) +# --------------------------------------------------------------------------- # + + +def test_siblings_share_the_stores_key_space(): + store = mock_s3(data={"p/f": b"value"}, prefix="p") + assert s3dol.handles(store)["f"].key == "p/f" + assert "p/f" in s3dol.urls(store)["f"] + assert s3dol.info(store)["f"].size == 5 + + +@pytest.mark.parametrize( + "wrap", + [ + lambda s: KeyCodecs.prefixed("x/")(s), + lambda s: Pipe(filt_iter.prefixes("x/"), KeyCodecs.prefixed("x/"))(s), + lambda s: cached_keys(KeyCodecs.prefixed("x/")(s)), + ], + ids=["key-codec", "pipe", "cached_keys-over-codec"], +) +def test_parallel_wrapping_is_correct_at_any_depth_and_lifetime(wrap): + # The property no resolution primitive can offer: no inner_most_key, no + # wrapped_self, no live reference needed. + store = mock_s3(data={"p/x/f": b"D"}, prefix="p") + assert wrap(store)["f"] == b"D" + assert wrap(s3dol.handles(store))["f"].key == "p/x/f" + assert "p/x/f" in wrap(s3dol.urls(store))["f"] + # ...including when nothing holds the wrapper (a temporary in one expr): + assert KeyCodecs.prefixed("x/")(s3dol.handles(store))["f"].key == "p/x/f" + + +def test_accessors_refuse_a_wrapped_store_and_name_the_remedy(): + store = mock_s3(data={"p/f": b"v"}, prefix="p") + wrapped = KeyCodecs.prefixed("x/")(store) + for accessor in (s3dol.handles, s3dol.urls, s3dol.info): + with pytest.raises(ConfigurationError) as info: + accessor(wrapped) + message = str(info.value) + assert "unwrapped" in message and "codec(" in message + + +def test_accessors_reject_a_non_store(): + with pytest.raises(ConfigurationError, match="s3dol.handles"): + s3dol.handles({"a": b"1"}) + + +def test_url_sibling_takes_presign_options(): + store = mock_s3(data={"k": b"v"}) + assert "X-Amz-Expires=60" in s3dol.urls(store, expires_in=60)["k"] + + +def test_siblings_iterate_the_same_keys_as_the_data_store(): + store = mock_s3(data={"p/a": b"1", "p/b": b"2", "other": b"3"}, prefix="p") + assert sorted(s3dol.handles(store)) == sorted(store) == ["a", "b"] + + +def test_anonymous_urls_sibling_yields_none(): + assert s3dol.urls(mock_s3(data={"k": b"v"}, anon=True))["k"] is None + + +# --------------------------------------------------------------------------- # +# D3 — free functions +# --------------------------------------------------------------------------- # + + +def test_sub_on_an_unwrapped_store_extends_the_prefix_and_keeps_pushdown(): + store = mock_s3(data={"a/b/c": b"1", "a/d": b"2"}, prefix="a") + child = s3dol.sub(store, "b") + assert list(child) == ["c"] and child["c"] == b"1" + assert type(child) is type(store) # same class; the prefix moved + assert child.prefix == "a/b/" + + +def test_sub_on_a_wrapped_store_composes_filter_first(): + # Filter FIRST is the only safe composition (ADR-0006 §1): reversed, the + # store is silently empty — and bare prefixed() corrupts out-of-band keys. + store = mock_s3(data={"p/x/b/c": b"1", "p/x/d": b"2"}, prefix="p") + wrapped = KeyCodecs.prefixed("x/")(store) + child = s3dol.sub(wrapped, "b") + assert list(child) == ["c"] and child["c"] == b"1" + + +def test_sub_rejects_a_foreign_object(): + with pytest.raises(ConfigurationError, match="s3dol.sub"): + s3dol.sub({"a": 1}, "p") + + +def test_prefixes_lists_one_level_in_the_callers_key_space(): + store = mock_s3( + data={"a/b/c": b"1", "a/d": b"2", "a/e/f": b"3"}, prefix="a" + ) + assert sorted(s3dol.prefixes(store)) == ["b/", "e/"] + + +def test_prefixes_maps_outward_through_a_wrapper(): + store = mock_s3(data={"p/x/dir/a": b"1", "p/x/e": b"2", "p/other/z": b"3"}, prefix="p") + wrapped = KeyCodecs.prefixed("x/")(store) + # scoped to the caller's key space: 'other/' is NOT theirs to see + assert s3dol.prefixes(wrapped) == ["dir/"] + + +def test_prefixes_refuses_a_wrapper_whose_codec_has_no_directory_structure(): + store = mock_s3(data={"p/a/b": b"1"}, prefix="p") + wrapped = KeyCodecs.suffixed(".json")(store) # maps '' -> '.json' + with pytest.raises(ConfigurationError, match="directory structure"): + s3dol.prefixes(wrapped) + + +def test_free_functions_refuse_a_hand_rolled_non_store_layer(): + # D3a's documented hole: a passthrough __getattr__ layer breaks the walk. + # Refuse rather than return a plausible wrong key. + class Passthrough: + def __init__(self, store): + self.store = store + + def __getattr__(self, name): + return getattr(self.store, name) + + layered = Passthrough(mock_s3(data={"p/a": b"1"}, prefix="p")) + with pytest.raises(ConfigurationError, match="refuse rather than guess"): + s3dol.prefixes(layered) + + +# --------------------------------------------------------------------------- # +# D4 — destructive operations are free functions +# --------------------------------------------------------------------------- # + + +def test_delete_many_resolves_the_callers_keys_through_the_chain(): + store = mock_s3(data={"p/x/a": b"1", "p/x/b": b"2", "p/other": b"3"}, prefix="p") + wrapped = KeyCodecs.prefixed("x/")(store) + s3dol.delete_many(wrapped, ["a", "b"]) + assert sorted(store._backing.data) == ["p/other"] # nothing outside scope + + +def test_delete_many_on_an_unwrapped_store(): + store = mock_s3(data={"p/a": b"1", "p/b": b"2"}, prefix="p") + s3dol.delete_many(store, ["a"]) + assert sorted(store) == ["b"] + + +def test_delete_many_is_idempotent_for_absent_keys(): + # S3 reports absent keys as Deleted; delete_many does not pretend otherwise. + store = mock_s3(data={"a": b"1"}) + s3dol.delete_many(store, ["a", "never-existed"]) + assert list(store) == [] + + +def test_delete_many_chunks_at_a_thousand(): + from s3dol.capabilities import DELETE_MANY_CHUNK + + assert DELETE_MANY_CHUNK == 1000 # AWS's cap; moto accepts 1001 + calls = [] + + class Counting(type(mock_s3())): + def _op_bulk_delete(self, ids): + calls.append(len(ids)) + return super()._op_bulk_delete(ids) + + from s3dol.testing import _FakeBackend + + data = {f"k{i}": b"v" for i in range(2500)} + store = Counting("mock-bucket", backing=_FakeBackend(data)) + s3dol.delete_many(store, list(data)) + assert calls == [1000, 1000, 500] + + +def test_delete_many_partial_failure_carries_the_split(): + from s3dol.testing import FakeBucketStore, _FakeBackend + + class Failing(FakeBucketStore): + def _op_bulk_delete(self, ids): + return [ids[0]], [(ids[1], "AccessDenied", "nope")] + + store = Failing("b", backing=_FakeBackend({"a": b"1", "b": b"2"})) + with pytest.raises(S3PartialFailure) as info: + s3dol.delete_many(store, ["a", "b"]) + assert info.value.succeeded == ["a"] and set(info.value.failures) == {"b"} + + +def test_delete_many_emulates_when_batch_delete_is_unsupported(): + # GCS and any provider that rejects the mandatory DeleteObjects checksum: + # loop DeleteObject; identical observable result, only cost differs. + from s3dol.connection import S3Connection + from s3dol.testing import FakeBucketStore, _FakeBackend + + store = FakeBucketStore( + "b", + backing=_FakeBackend({"a": b"1", "b": b"2"}), + connection=S3Connection(endpoint_url="https://storage.googleapis.com"), + ) + assert store._resolution().capabilities.batch_delete is False + s3dol.delete_many(store, ["a", "b"]) + assert list(store) == [] + + +def test_delete_many_refuses_a_foreign_first_argument(): + # The honest caveat: a free function accepts anything — so validate. + with pytest.raises(ConfigurationError): + s3dol.delete_many({"a": b"1"}, ["a"]) + + +def test_delete_many_refuses_anonymous_writes(): + with pytest.raises(CredentialsError): + s3dol.delete_many(mock_s3(data={"a": b"1"}, anon=True), ["a"]) + + +def test_delete_bucket_requires_an_endpoint_store(): + with pytest.raises(ConfigurationError, match="delete_bucket"): + s3dol.delete_bucket(mock_s3(), "b", force=True) diff --git a/s3dol/tests/test_conformance.py b/s3dol/tests/test_conformance.py new file mode 100644 index 0000000..3dab992 --- /dev/null +++ b/s3dol/tests/test_conformance.py @@ -0,0 +1,114 @@ +"""The exported conformance suite, run against the fake, the real store +(moto), and a shipped recipe — the ADR-0008 drift gate: the fake and the real +store pass the SAME laws, so divergence fails here.""" + +import json + +import pytest + +from s3dol.testing import FakeBucketStore, _FakeBackend, mock_s3, run_conformance + +moto = pytest.importorskip("moto") + + +@pytest.fixture +def _mock_aws_env(monkeypatch): + for var in ("AWS_ENDPOINT_URL", "AWS_ENDPOINT_URL_S3", "AWS_PROFILE"): + monkeypatch.delenv(var, raising=False) + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "testing") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "testing") + monkeypatch.setenv("AWS_DEFAULT_REGION", "us-east-1") + + +def test_conformance_fake(): + run_conformance(lambda: FakeBucketStore("mock-bucket", backing=_FakeBackend())) + + +def test_conformance_fake_with_prefix(): + run_conformance( + lambda: FakeBucketStore( + "mock-bucket", backing=_FakeBackend(), prefix="tenant-a" + ) + ) + + +def test_conformance_real_store(_mock_aws_env): + from s3dol.base import BucketStore + from s3dol.connection import S3Connection + + with moto.mock_aws(): + connection = S3Connection(region_name="us-east-1") + connection.client.create_bucket(Bucket="conf-bucket") + run_conformance( + lambda: BucketStore("conf-bucket", connection=connection), + check_pickle=False, # moto's patch does not survive a real + # process hop; pickling itself is covered in test_base_moto + ) + + +def test_conformance_real_store_with_prefix(_mock_aws_env): + from s3dol.base import BucketStore + from s3dol.connection import S3Connection + + with moto.mock_aws(): + connection = S3Connection(region_name="us-east-1") + connection.client.create_bucket(Bucket="conf-bucket") + run_conformance( + lambda: BucketStore("conf-bucket", connection=connection, prefix="p"), + check_pickle=False, + ) + + +def test_conformance_json_recipe(_mock_aws_env): + # Layer C: the value law's normalize is json.loads∘json.dumps (N3 — + # (1,2) -> [1,2]); values are py objects so the str-rejection and url + # sections do not apply verbatim. + from s3dol.connection import S3Connection + from s3dol.recipes import S3Jsons + + with moto.mock_aws(): + connection = S3Connection(region_name="us-east-1") + connection.client.create_bucket(Bucket="json-bucket") + store = S3Jsons("json-bucket", connection=connection) + store["a"] = {"x": 1, "t": (1, 2)} + assert store["a"] == json.loads(json.dumps({"x": 1, "t": (1, 2)})) + assert store["a"]["t"] == [1, 2] # the honest N3 invariant + + +def test_shipped_recipe_encoders_produce_bytes(): + # ADR-0008 assertion 9: every shipped recipe's encoder output is a + # BytesSource — this is what catches ValueCodecs.json() returning str. + import pickle as pickle_module + + encoders = { + "S3Texts": lambda s: s.encode(), + "S3Jsons": lambda obj: json.dumps(obj).encode(), + "S3Pickles": pickle_module.dumps, + } + samples = {"S3Texts": "text", "S3Jsons": {"a": 1}, "S3Pickles": {"a": 1}} + for name, encode in encoders.items(): + assert isinstance(encode(samples[name]), bytes), name + + +def test_recipe_wire_format_is_plain_bytes(_mock_aws_env): + # A codec must not change the wire format: re-read through a raw store. + from s3dol.base import BucketStore + from s3dol.connection import S3Connection + from s3dol.recipes import S3Jsons + + with moto.mock_aws(): + connection = S3Connection(region_name="us-east-1") + connection.client.create_bucket(Bucket="wire-bucket") + S3Jsons("wire-bucket", connection=connection)["k"] = {"n": 5} + raw = BucketStore("wire-bucket", connection=connection)["k"] + assert json.loads(raw) == {"n": 5} + + +def test_conformance_catches_a_broken_store(): + # The suite must actually bite: a store that lies about membership fails. + class Lying(FakeBucketStore): + def __contains__(self, k): + return False + + with pytest.raises(AssertionError): + run_conformance(lambda: Lying("mock-bucket", backing=_FakeBackend())) diff --git a/s3dol/tests/test_connection.py b/s3dol/tests/test_connection.py index 34fc75c..2f10262 100644 --- a/s3dol/tests/test_connection.py +++ b/s3dol/tests/test_connection.py @@ -25,6 +25,8 @@ AmbiguousResolution, ConfigurationError, MissingEndpoint, + MissingPresetParam, + PresetConflict, PresetHostMismatch, ) from s3dol.presets import get_preset @@ -307,9 +309,14 @@ def test_c1_requires_endpoint_raises_missing_endpoint(): def test_c1_unbound_template_raises_and_names_the_params(): - with pytest.raises(MissingEndpoint) as info: + # MissingPresetParam names this case exactly, and IS a MissingEndpoint — + # so either `except` works and neither class is dead (ADR-0004's own + # indictment of v0 was five never-raised exception classes). + with pytest.raises(MissingPresetParam) as info: resolve(S3Connection(preset="r2"), {}) assert "account_id" in str(info.value) + with pytest.raises(MissingEndpoint): + resolve(S3Connection(preset="r2"), {}) def test_c1_env_endpoint_satisfies_a_requiring_preset(): @@ -469,7 +476,12 @@ def test_load_aws_config_is_credential_scrubbed(): # tier 2 (moto): the built client — SigV4 presigning is issue #10's regression # --------------------------------------------------------------------------- # -moto = pytest.importorskip("moto") +@pytest.fixture +def moto(): + """moto, per-test — a module-level importorskip would skip this file's + ~60 pure tier-1 tests (credential shapes, pickling, the lock race, every + resolve() ladder) along with the two that actually need it.""" + return pytest.importorskip("moto") @pytest.fixture @@ -481,7 +493,7 @@ def _mock_aws_env(monkeypatch): monkeypatch.setenv("AWS_DEFAULT_REGION", "us-east-1") -def test_client_presigns_sigv4_even_in_us_east_1(_mock_aws_env): +def test_client_presigns_sigv4_even_in_us_east_1(moto, _mock_aws_env): # us-east-1 is precisely a region where botocore, left to itself, silently # downgrades presigning to SigV2 (ADR-0003 §4). Assert on the URL, never on # client.meta.config.signature_version — it reports 's3v4' while emitting @@ -498,7 +510,7 @@ def test_client_presigns_sigv4_even_in_us_east_1(_mock_aws_env): assert "AWSAccessKeyId" not in url # the SigV2 tell -def test_anon_client_is_unsigned_and_still_builds(_mock_aws_env): +def test_anon_client_is_unsigned_and_still_builds(moto, _mock_aws_env): from botocore import UNSIGNED with moto.mock_aws(): @@ -506,7 +518,7 @@ def test_anon_client_is_unsigned_and_still_builds(_mock_aws_env): assert spec.client.meta.config.signature_version is UNSIGNED -def test_static_credentials_reach_the_request(_mock_aws_env): +def test_static_credentials_reach_the_request(moto, _mock_aws_env): with moto.mock_aws(): spec = S3Connection( credentials=("AKIAINJECTED", "sEkR3t"), region_name="us-east-1" @@ -517,3 +529,174 @@ def test_static_credentials_reach_the_request(_mock_aws_env): "get_object", Params={"Bucket": "cred-bucket", "Key": "k"}, ExpiresIn=60 ) assert "AKIAINJECTED" in url # the injected identity signed, not the env one + + +# --------------------------------------------------------------------------- # +# branches the first pass left untested (found by adversarial review) +# --------------------------------------------------------------------------- # + + +def test_pinned_preset_conflict_raises(): + # The ratchet of ADR-0012 D4: pinning is dormant while rows are + # doc-sourced, and turns on with evidence. Exercise the branch that + # consumes it, or it can be broken before any row is ever verified. + from s3dol.presets import Preset, register, unregister + + register( + Preset( + name="verified-row", + addressing_style="path", + endpoint_url="https://verified.example", + region_name="eu-west-9", + verified=True, + ) + ) + try: + with pytest.raises(PresetConflict, match="eu-west-9"): + resolve(S3Connection(preset="verified-row", region_name="us-east-1"), {}) + # ...and agreeing is fine + assert ( + resolve( + S3Connection(preset="verified-row", region_name="eu-west-9"), {} + ).region_name.value + == "eu-west-9" + ) + finally: + unregister("verified-row") + + +def test_notes_are_actually_emitted_as_warnings_at_client_build(monkeypatch): + # resolve() only COLLECTS notes; _build_client is the sole user-visible + # channel, and filterwarnings over it is the documented strict mode. + # Without this test, deleting the warn loop leaves the suite green. + monkeypatch.setenv("AWS_ENDPOINT_URL_S3", "https://proxy.mycorp.internal") + with pytest.warns(AmbiguousResolution, match="host patterns"): + S3Connection(preset="r2").client + + +def test_filterwarnings_is_strict_mode(monkeypatch): + import warnings + + from s3dol.errors import S3DolWarning + + monkeypatch.setenv("AWS_ENDPOINT_URL_S3", "https://proxy.mycorp.internal") + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + warnings.filterwarnings("error", category=S3DolWarning) + with pytest.raises(AmbiguousResolution): + S3Connection(preset="r2").client + + +def test_verify_and_client_kwargs_reach_the_built_client(monkeypatch): + # verify= is a connection-security field: dropping it would silently lose + # a self-signed-MinIO user's CA setting, failing OPEN. + captured = {} + + class FakeSession: + def __init__(self, botocore_session=None): + pass + + def client(self, service, **kwargs): + captured.update(kwargs) + return object() + + import boto3 + + monkeypatch.setattr(boto3, "Session", FakeSession) + S3Connection( + endpoint_url="https://minio.internal:9000", + verify="/etc/ssl/corp-ca.pem", + client_kwargs={"use_ssl": True}, + ).client + assert captured["verify"] == "/etc/ssl/corp-ca.pem" + assert captured["use_ssl"] is True + assert captured["endpoint_url"] == "https://minio.internal:9000" + + +def test_signature_version_none_is_refused_not_silently_sigv2(): + # 'None == do not override' is the convention of its NEIGHBOURS; for this + # field it would re-enable the SigV2 downgrade (issue #10) silently. + with pytest.raises(ConfigurationError, match="SigV2"): + S3Connection(signature_version=None) + + +def test_client_kwargs_cannot_smuggle_credentials(): + with pytest.raises(ConfigurationError, match="credentials="): + S3Connection(client_kwargs={"aws_secret_access_key": "sEkR3t"}) + with pytest.raises(ConfigurationError): + S3Connection(client_kwargs={"config": object()}) + + +@pytest.mark.parametrize( + "shape", + [ + lambda: S3Connection(client_kwargs={"aws_secret_access_key": "sEkR3tVALUE"}), + lambda: S3Connection(credentials=("AKIAX", "sEkR3tVALUE")), + ], +) +def test_no_credential_shape_can_leak_through_repr(shape): + try: + spec = shape() + except ConfigurationError: + return # refused outright: even better + assert "sEkR3tVALUE" not in repr(spec) + + +def test_env_endpoint_userinfo_never_survives_into_messages_or_repr(): + secret_url = "https://user:sEkR3tVALUE@minio.internal:9000" + resolution = resolve(S3Connection(), {"AWS_ENDPOINT_URL_S3": secret_url}) + assert "sEkR3tVALUE" not in repr(resolution) + assert "minio.internal" in repr(resolution) # the host survives + with_note = resolve( + S3Connection(preset="wasabi"), {"AWS_ENDPOINT_URL_S3": secret_url} + ) + assert all("sEkR3tVALUE" not in message for _, message in with_note.notes) + + +def test_tuple_credentials_with_a_none_token_behave_like_the_pair(): + assert normalize_credentials(("AK", "SK", None)) == normalize_credentials( + ("AK", "SK") + ) + + +def test_callable_credentials_fetch_exactly_once_on_first_load(): + # A token vendor may be rate-limited or single-use. + calls = [] + + def counting_fetch(): + calls.append(1) + return { + "access_key": "a", + "secret_key": "b", + "expires_at": "2030-01-01T00:00:00Z", + } + + CallableCredentials(counting_fetch)._load_botocore_credentials() + assert len(calls) == 1 + + +def test_region_capture_does_not_hijack_an_explicit_env_region(): + for host in ( + "https://eu-storage.mycorp.com", + "https://srv1.corp.example", + "https://my-storage.example.com", + ): + resolution = resolve( + S3Connection(endpoint_url=host), {"AWS_DEFAULT_REGION": "eu-west-1"} + ) + assert resolution.region_name.value == "eu-west-1", host + # ...while the canonical shape is still captured + assert ( + resolve( + S3Connection(endpoint_url="https://s3.eu-central-2.amazonaws.com"), {} + ).region_name.value + == "eu-central-2" + ) + + +def test_c2_guard_applies_to_a_named_soft_row_too(): + with pytest.raises(PresetHostMismatch): + resolve( + S3Connection(preset="aws"), + {"AWS_ENDPOINT_URL_S3": "https://s3.eu-central-1.wasabisys.com"}, + ) diff --git a/s3dol/tests/test_diagnose.py b/s3dol/tests/test_diagnose.py index 402a156..ce57c2a 100644 --- a/s3dol/tests/test_diagnose.py +++ b/s3dol/tests/test_diagnose.py @@ -6,8 +6,12 @@ def run_diagnose(**kwargs): - """diagnose() against a controlled environment, capturing the report.""" + """diagnose() against a controlled environment AND a controlled aws + config, capturing the report. Injecting both is what makes these tier 1: + otherwise every test silently consults the developer's own ~/.aws/config + and goes red on a machine whose profile sets an endpoint_url.""" kwargs.setdefault("environ", {}) + kwargs.setdefault("aws_config", {}) return diagnose(file=io.StringIO(), **kwargs) @@ -53,10 +57,19 @@ def test_secret_values_never_echo(): def test_prints_to_the_given_file_and_returns_the_text(): sink = io.StringIO() - text = diagnose(endpoint_url="http://x:1", environ={}, file=sink) + text = diagnose(endpoint_url="http://x:1", environ={}, aws_config={}, file=sink) assert sink.getvalue() == text +def test_a_configured_aws_endpoint_does_not_bleed_into_a_what_if_run(): + # The injected config is the ONLY config consulted. + report = run_diagnose( + preset="minio", aws_config={"profiles": {"default": {"endpoint_url": "http://cfg:1"}}} + ) + assert "http://cfg:1" in report and "RESOLUTION FAILS" not in report + assert "RESOLUTION FAILS" in run_diagnose(preset="minio") # nothing supplies one + + # --------------------------------------------------------------------------- # # the v0 mangler (D7): ~20 frozen lines mapping 1:1 onto v0 # --------------------------------------------------------------------------- # @@ -129,6 +142,76 @@ def test_excluded_branch_prints_the_reason(): assert "process-global" in report +def test_explicit_credentials_overridden_by_env_is_reported_as_a_move(): + # THE case step 0 exists for: v0 let env credentials silently override + # explicit ones. Comparing provenance LABELS missed it (both truncate to + # 'AKIA…'); the identity predicate catches it. + report = run_diagnose( + bucket_name="b", + aws_access_key_id="AKIAEXPLICIT", + aws_secret_access_key="explicit-secret", + environ={ + "AWS_ACCESS_KEY_ID": "AKIAFROMENV", + "AWS_SECRET_ACCESS_KEY": "env-secret", + }, + ) + assert "credential identity" in report + assert "CHANGES ON UPGRADE" in report + + +def test_env_only_credentials_are_not_a_false_alarm(): + # The most common v0 configuration: env credentials, no explicit kwargs. + # Both sides resolve to the same keys, so the table must say so — a + # migration signal that cries wolf gets filtered away. + report = run_diagnose( + bucket_name="b", + endpoint_url="http://minio:9000", + environ={"AWS_ACCESS_KEY_ID": "AKIAENV", "AWS_SECRET_ACCESS_KEY": "s"}, + ) + assert "credential identity" not in report + + +def test_detection_supplied_settings_are_announced_as_new_in_v1(): + # Both sides run through v1's resolver, so pass-2 detection cancels out of + # any diff — it has to be reported explicitly or the announcement + # instrument cannot see the path->virtual flip it exists to announce. + report = run_diagnose( + bucket_name="b", endpoint_url="https://s3.us-west-004.backblazeb2.com" + ) + assert "NEW IN v1" in report + assert "backblaze" in report and "when_required" in report + + +def test_env_profile_without_env_credentials_names_the_v0_raise(): + report = run_diagnose(bucket_name="b", profile_name="environment variables") + assert "v0 RAISES" in report + assert "EXCLUDED" in report + + +def test_v1_credential_kwargs_are_never_echoed(): + # The allowlist, not a denylist of two v0 names: every v1 shape must be + # covered, in the one artifact users are told to run and paste into issues. + for kwargs in ( + {"credentials": ("AKIAX", "sEkR3tVALUE")}, + {"credentials": {"aws_secret_access_key": "sEkR3tVALUE"}}, + {"client_kwargs": {"aws_secret_access_key": "sEkR3tVALUE"}}, + ): + assert "sEkR3tVALUE" not in run_diagnose(**kwargs) + + +def test_the_access_key_id_is_shown_by_prefix_only(): + report = run_diagnose(bucket_name="b", aws_access_key_id="AKIAFULLKEYVALUE") + assert "AKIAFULLKEYVALUE" not in report + assert "AKIA" in report + + +def test_an_endpoint_with_userinfo_is_redacted_in_the_echo(): + report = run_diagnose( + bucket_name="b", endpoint_url="https://user:sEkR3tVALUE@host:9000" + ) + assert "sEkR3tVALUE" not in report + + def test_credential_identity_divergence_is_a_predicate_not_a_value(): report = run_diagnose( bucket_name="b", diff --git a/s3dol/tests/test_keyed_surface.py b/s3dol/tests/test_keyed_surface.py new file mode 100644 index 0000000..5ae1e94 --- /dev/null +++ b/s3dol/tests/test_keyed_surface.py @@ -0,0 +1,153 @@ +"""The ADR-0011 D5 reflective conformance test: no keyed methods, enforced. + +Not a registry — a registry fails silently when an author forgets to declare a +method. This test enumerates **every public attribute of every Layer B class** +against an explicit inventory and fails on any *new* one, forcing the +"does this take anything that reaches the wire as a key?" judgement at review +time. (A signature predicate cannot make that judgement: ``delete(name)``, +``delete_many(keys)``, ``batch(operations)`` and ``sync_to(target)`` all hide +keys in innocently-named parameters.) + +Allowlist of keyed methods: ``url_for`` (the guarded ``dol.SupportsUrlFor`` +shim, ADR-0011 D3b) — and NOTHING else. To add a capability: +sibling store (D2), ``ObjectHandle`` (key bound at construction), or a free +function taking the store first (D3). Do not grow this inventory with a keyed +method; the comment you are reading is the review gate. +""" + +import pytest + +from s3dol.base import ( + BucketCollection, + BucketReader, + BucketStore, + EndpointCollection, + EndpointReader, + EndpointStore, + S3Profiles, +) +from s3dol.capabilities import BucketHandles, BucketInfo, BucketUrls + +#: The ONLY method allowed to take a key (guarded; see ADR-0011 D3b). +KEYED_METHOD_ALLOWLIST = {"url_for"} + +#: Mapping-protocol members `dol` maps correctly at every wrapper depth — +#: the sanctioned keyed surface. +MAPPING_PROTOCOL = { + "__getitem__", + "__setitem__", + "__delitem__", + "__contains__", + "__iter__", + "__len__", + "__eq__", + "__ne__", + "get", + "keys", + "values", + "items", + "pop", + "popitem", + "clear", + "update", + "setdefault", + "head", # dol.KvReader convenience (first item; not key-taking) +} + +#: The full public (non-dunder, non-Mapping) inventory, per class. A test +#: failure here means the public surface changed: review the new attribute +#: against ADR-0011 D1 before extending the inventory. +PUBLIC_INVENTORY = { + BucketCollection: {"bucket", "connection", "delimiter", "prefix", "reads", + "writes", "on_missing_bucket", "strict_delete", "client", + "KeysView", "ValuesView", "ItemsView"}, + BucketReader: {"url_for"}, + BucketStore: set(), + BucketHandles: set(), + BucketUrls: {"expires_in", "client_method"}, + BucketInfo: set(), + EndpointCollection: {"connection", "bucket_kwargs", "client", + "KeysView", "ValuesView", "ItemsView"}, + EndpointReader: set(), + EndpointStore: set(), + S3Profiles: {"bucket_kwargs", "readonly", "KeysView", "ValuesView", + "ItemsView"}, +} + +#: Methods among the inventory that take a key-like argument. Must stay equal +#: to the allowlist. +KEYED_IN_INVENTORY = {"url_for"} + + +def _public_surface(cls) -> set: + """Every public attribute the s3dol classes in ``cls``'s MRO define, + minus the Mapping protocol.""" + names = set() + for klass in cls.__mro__: + if klass.__module__.startswith("s3dol"): + names |= set(vars(klass)) + return { + name + for name in names + if not name.startswith("_") and name not in MAPPING_PROTOCOL + } + + +@pytest.mark.parametrize("cls", list(PUBLIC_INVENTORY), ids=lambda c: c.__name__) +def test_public_surface_matches_the_inventory(cls): + surface = _public_surface(cls) + all_declared = set() + for klass, names in PUBLIC_INVENTORY.items(): + if klass in cls.__mro__: + all_declared |= names + unexpected = surface - all_declared + assert not unexpected, ( + f"{cls.__name__} grew public attribute(s) {sorted(unexpected)} not in " + f"the ADR-0011 D5 inventory. If any takes something that reaches the " + f"wire as a key — at any argument position, or inside a structure — " + f"it must become a sibling store, an ObjectHandle method, or a free " + f"function. Otherwise, add it to PUBLIC_INVENTORY with a reviewer's " + f"eyes on this message." + ) + + +def test_the_keyed_allowlist_is_exactly_url_for(): + assert KEYED_IN_INVENTORY == KEYED_METHOD_ALLOWLIST == {"url_for"} + + +def test_url_for_exists_only_where_declared(): + # BucketReader (and its subclasses) only — never the endpoint level. + assert "url_for" in vars(BucketReader) + for cls in (BucketCollection, EndpointCollection, EndpointReader, EndpointStore): + assert "url_for" not in vars(cls) + + +def test_object_handle_binds_its_key_at_construction(): + # The handle may be method-rich precisely because no method takes a key. + import inspect + + from s3dol.base import ObjectHandle + + for name, member in vars(ObjectHandle).items(): + if name.startswith("_") or not callable(member): + continue + params = list(inspect.signature(member).parameters)[1:] # drop self + assert "k" not in params and "key" not in params, ( + f"ObjectHandle.{name} takes a key parameter — the handle's " + f"immunity to the delegation trap rests on the key being bound " + f"at construction (ADR-0011 D1)." + ) + + +def test_free_functions_take_the_store_first(): + import inspect + + import s3dol.capabilities as capabilities + + for name in ("sub", "prefixes", "delete_many", "handles", "urls", "info"): + first = next(iter(inspect.signature(getattr(capabilities, name)).parameters)) + assert first in ("store", "endpoint"), (name, first) + first = next( + iter(inspect.signature(capabilities.delete_bucket).parameters) + ) + assert first == "endpoint" diff --git a/s3dol/tests/test_store.py b/s3dol/tests/test_store.py deleted file mode 100644 index ad3ac65..0000000 --- a/s3dol/tests/test_store.py +++ /dev/null @@ -1,47 +0,0 @@ -import pytest -from s3dol.base import S3ClientDol - -from s3dol.store import S3Store -from s3dol.tests.test_base import assert_bucket_key_value -from s3dol.utility import S3DolException - - -def setup_test_bucket(aws_access_key_id, aws_secret_access_key, endpoint_url): - s3_client = S3ClientDol( - aws_access_key_id=aws_access_key_id, - aws_secret_access_key=aws_secret_access_key, - endpoint_url=endpoint_url, - ) - if "test-bucket" not in s3_client: - s3_client["test-bucket"] = {} - - if "level1/level2/test-key" in s3_client["test-bucket"]: - del s3_client["test-bucket"]["level1/level2/test-key"] - - -@pytest.mark.parametrize( - "aws_access_key_id, aws_secret_access_key, endpoint_url", - [("localstack", "localstack", "http://localhost:4566")], -) -def test_s3_store_crud(aws_access_key_id, aws_secret_access_key, endpoint_url): - setup_test_bucket(aws_access_key_id, aws_secret_access_key, endpoint_url) - - s3_store = S3Store( - bucket_name="test-bucket", - aws_access_key_id=aws_access_key_id, - aws_secret_access_key=aws_secret_access_key, - endpoint_url=endpoint_url, - ) - test_value = b"test-store-value" - s3_store["level1/level2/test-key"] = test_value - assert_bucket_key_value(s3_store, "level1/level2/test-key", test_value) - assert_bucket_key_value(s3_store["level1/level2/"], "test-key", test_value) - assert_bucket_key_value(s3_store["level1/"]["level2/"], "test-key", test_value) - assert_bucket_key_value(s3_store["level1/"], "level2/test-key", test_value) - - del s3_store["level1/"]["level2/"]["test-key"] - assert "level1/level2/test-key" not in s3_store - - -if __name__ == "__main__": - pytest.main([__file__]) diff --git a/s3dol/tests/test_store_shim.py b/s3dol/tests/test_store_shim.py new file mode 100644 index 0000000..41c251f --- /dev/null +++ b/s3dol/tests/test_store_shim.py @@ -0,0 +1,237 @@ +"""The v0 ``S3Store`` compat shim (ADR-0007 §3/§4, ADR-0012 D7). + +The shim's justification is "existing users keep working", so these tests are +written from the dependents' *actual* usage: `lacing` calls +``S3Store(bucket, path=prefix, **kw)`` positionally and passes +``make_bucket=True``; `http_cosmo_prep` passes the bucket by keyword with an +explicit ``endpoint_url``, writes ``str`` values, uses ``store['folder/']`` as +a sub-store, and relies on ``del`` being a silent no-op. +""" + +import inspect +import warnings + +import pytest + +from s3dol.errors import S3DolResolutionChanged +from s3dol.store import ( + S3Store, + _V0CompatStore, + dechunk_aws_chunked, + detect_chunked_framing, +) +from s3dol.testing import _FakeBackend, mock_s3 + + +def compat(data=None, prefix="", dechunk=False): + """A shim store over the in-memory fake (no network).""" + return _V0CompatStore(mock_s3(data=data, prefix=prefix), dechunk=dechunk) + + +# --------------------------------------------------------------------------- # +# the signature must not move (both dependents import s3dol.store.S3Store) +# --------------------------------------------------------------------------- # + + +def test_module_survives_as_an_importable_path(): + import s3dol.store # noqa: F401 — the fully-qualified import both use + + assert callable(S3Store) + + +def test_signature_is_v0s_exactly(): + parameters = inspect.signature(S3Store).parameters + assert list(parameters)[0] == "bucket_name" + assert parameters["bucket_name"].kind is inspect.Parameter.POSITIONAL_OR_KEYWORD + for name in ( + "make_bucket", + "path", # NOT renamed to prefix on the shim + "aws_access_key_id", + "aws_secret_access_key", + "aws_session_token", + "endpoint_url", + "region_name", + "profile_name", + "skip_bucket_check", + "is_supabase_endpoint", # v0 accepts it; omitting it = TypeError on upgrade + ): + assert name in parameters, name + + +def test_test_utils_still_importable_for_py2store(): + # py2store imports these under suppress(ImportError) — breakage is SILENT. + from s3dol.tests.util import ( # noqa: F401 + extract_s3_access_info, + get_s3_test_access_info_from_env_vars, + ) + + +def test_calling_it_warns_about_the_v2_removal(): + with pytest.warns(DeprecationWarning, match="s3_store"): + with pytest.raises(Exception): # no credentials/bucket here; the warning is the point + S3Store("some-bucket", make_bucket=False) + + +# --------------------------------------------------------------------------- # +# the three behaviours a dependent's PASSING tests rely on (ADR-0007 §4) +# --------------------------------------------------------------------------- # + + +def test_str_values_are_accepted_with_a_deprecation_warning(): + store = compat() + with pytest.warns(DeprecationWarning, match="utf-8"): + store["k"] = "a string" + assert store["k"] == b"a string" + + +def test_trailing_slash_returns_a_sub_store(): + store = compat(data={"level1/level2/k": b"v"}) + assert store["level1/level2/"]["k"] == b"v" + assert store["level1/"]["level2/"]["k"] == b"v" + assert store["level1/"]["level2/k"] == b"v" + + +def test_delete_of_an_absent_key_is_a_silent_no_op(): + store = compat(data={"a": b"1"}) + del store["a"] + del store["a"] # v0 relied on this; v1 deletes are idempotent anyway + + +def test_len_still_counts_on_the_shim(): + # The v1 store raises TypeError; the shim keeps v0's behaviour (and cost). + assert len(compat(data={"a": b"1", "b": b"2"})) == 2 + + +def test_url_for_is_forwarded_and_prefix_aware(): + store = compat(data={"blobs/abc": b"v"}, prefix="blobs") + url = store.url_for("abc") + assert "blobs/abc" in url + assert "X-Amz-Algorithm=AWS4-HMAC-SHA256" in url # the #10 fix + assert "AWSAccessKeyId" not in url # the SigV2 tell + + +def test_shim_iterates_and_contains_like_a_mapping(): + store = compat(data={"a": b"1", "b": b"2"}) + assert sorted(store) == ["a", "b"] + assert "a" in store and "zz" not in store + + +# --------------------------------------------------------------------------- # +# the corrected de-chunker (ADR-0003 §3) — v0's truncated multi-chunk payloads +# --------------------------------------------------------------------------- # + + +def test_dechunk_single_chunk(): + assert dechunk_aws_chunked(b"5\r\nhello\r\n0\r\n\r\n") == b"hello" + + +def test_dechunk_multi_chunk_where_v0_truncated(): + framed = b"3\r\nabc\r\n2\r\nde\r\n0\r\nx-amz-checksum-crc32:AAAA\r\n\r\n" + assert dechunk_aws_chunked(framed) == b"abcde" + + +@pytest.mark.parametrize( + "payload", + [ + b"cafe is hex-ish but not framed", # v0 misfired on this shape + b"0123456789", + b"", + b"deadbeef", + b"5\r\nhello", # truncated framing: passthrough, never a partial parse + b"5\r\ntoo-long-for-its-header\r\n0\r\n\r\n", + ], +) +def test_dechunk_is_full_parse_or_passthrough(payload): + assert dechunk_aws_chunked(payload) == payload + + +def test_detect_chunked_framing_finds_the_corrupted_keys(): + store = mock_s3( + data={"good": b"plain bytes", "bad": b"5\r\nhello\r\n0\r\n\r\n"} + ) + assert detect_chunked_framing(store) == ["bad"] + + +def test_dechunking_store_repairs_reads_only_when_enabled(): + framed = b"5\r\nhello\r\n0\r\n\r\n" + assert compat(data={"k": framed}, dechunk=True)["k"] == b"hello" + assert compat(data={"k": framed})["k"] == framed # never a default + + +# --------------------------------------------------------------------------- # +# the D7 migration warning +# --------------------------------------------------------------------------- # + + +def test_warns_when_v1_resolves_differently_than_v0(monkeypatch): + # v0 DROPPED an explicit endpoint_url whenever env credentials existed — + # the store then talked to AWS. v1 honours it; that moves the data target, + # so it must announce itself. + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "AKIAENV") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "envsecret") + monkeypatch.delenv("AWS_ENDPOINT_URL", raising=False) + monkeypatch.delenv("AWS_ENDPOINT_URL_S3", raising=False) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + S3Store( + "b", + endpoint_url="https://minio.example:9000", + aws_access_key_id="AKIAEXPLICIT", + aws_secret_access_key="explicitsecret", + ) + messages = [str(w.message) for w in caught if w.category is S3DolResolutionChanged] + assert messages and "diagnose" in messages[0] + + +def test_no_warning_when_nothing_moves(monkeypatch): + for var in ("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_ENDPOINT_URL", + "AWS_ENDPOINT_URL_S3"): + monkeypatch.delenv(var, raising=False) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + S3Store("b", endpoint_url="https://minio.example:9000") + assert not [w for w in caught if w.category is S3DolResolutionChanged] + + +def test_strict_mode_is_filterwarnings_not_a_kwarg(monkeypatch): + # ADR-0012 D1: no strict= kwarg; filterwarnings('error', S3DolWarning) is + # strict mode, with zero API surface. + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "AKIAENV") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "envsecret") + monkeypatch.delenv("AWS_ENDPOINT_URL", raising=False) + monkeypatch.delenv("AWS_ENDPOINT_URL_S3", raising=False) + from s3dol.errors import S3DolWarning + + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + warnings.filterwarnings("error", category=S3DolWarning) + with pytest.raises(S3DolResolutionChanged): + S3Store( + "b", + endpoint_url="https://minio.example:9000", + aws_access_key_id="AKIAEXPLICIT", + aws_secret_access_key="explicitsecret", + ) + + +def test_make_bucket_tristate_maps_to_on_missing_bucket(): + from s3dol.base import BucketStore + + captured = {} + original = BucketStore.__init__ + + def spy(self, *args, **kwargs): + captured["on_missing_bucket"] = kwargs.get("on_missing_bucket") + captured["prefix"] = kwargs.get("prefix") + original(self, *args, **kwargs) + + BucketStore.__init__ = spy + try: + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + S3Store("b", make_bucket=True, path="logs") + assert captured == {"on_missing_bucket": "create", "prefix": "logs"} + S3Store("b", make_bucket=None) + assert captured["on_missing_bucket"] == "assume" + finally: + BucketStore.__init__ = original diff --git a/s3dol/tests/test_url_for.py b/s3dol/tests/test_url_for.py deleted file mode 100644 index 422f953..0000000 --- a/s3dol/tests/test_url_for.py +++ /dev/null @@ -1,61 +0,0 @@ -"""Presigned-URL (``url_for``) tests, mocked with moto. See issue #7. - -Unlike the env-gated real-S3 tests, these run anywhere — moto fully mocks the -S3 API including ``generate_presigned_url``. -""" - -import pytest - -pytest.importorskip("moto") -from moto import mock_aws - -from s3dol.store import S3Store - -_CREDS = dict( - aws_access_key_id="testing", - aws_secret_access_key="testing", - region_name="us-east-1", -) - - -@pytest.fixture(autouse=True) -def _isolate_aws_env(monkeypatch): - """Make these moto tests hermetic. The localstack tests (test_base) set - ``AWS_ENDPOINT_URL=http://localhost:4566`` via ``os.environ`` with no - cleanup, which would route boto3 past moto to a (down) localstack. Clear any - leaked endpoint + pin fake creds so ``@mock_aws`` always intercepts.""" - for var in ("AWS_ENDPOINT_URL", "AWS_ENDPOINT_URL_S3", "AWS_PROFILE"): - monkeypatch.delenv(var, raising=False) - monkeypatch.setenv("AWS_ACCESS_KEY_ID", "testing") - monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "testing") - monkeypatch.setenv("AWS_DEFAULT_REGION", "us-east-1") - - -@mock_aws -def test_url_for_returns_presigned_get_url(): - store = S3Store("test-bucket", make_bucket=True, **_CREDS) - store["hello.txt"] = b"hi" - url = store.url_for("hello.txt") - assert isinstance(url, str) - assert "test-bucket" in url - assert "hello.txt" in url - # Presigned marker (SigV4 X-Amz-Signature / SigV2 Signature). - assert "Signature" in url - - -@mock_aws -def test_url_for_is_prefix_aware(): - # path= sets a key prefix; url_for must point at the prefixed id, matching - # __getitem__ (else the URL 404s). - store = S3Store("test-bucket", make_bucket=True, path="blobs", **_CREDS) - store["abc"] = b"x" - url = store.url_for("abc") - assert "blobs" in url and "abc" in url - - -@mock_aws -def test_url_for_honors_expires_in(): - store = S3Store("test-bucket", make_bucket=True, **_CREDS) - store["k"] = b"v" - url = store.url_for("k", expires_in=60) - assert "X-Amz-Expires=60" in url or "Expires=" in url diff --git a/s3dol/tests/test_values.py b/s3dol/tests/test_values.py new file mode 100644 index 0000000..84a599d --- /dev/null +++ b/s3dol/tests/test_values.py @@ -0,0 +1,90 @@ +"""Tier-1 tests for the value refs and write domain (ADR-0005).""" + +import io +import pickle + +import pytest + +from s3dol.values import ( + Chunks, + Filepath, + Streamable, + as_fileobj, + is_bytes_like, + reject_str_value, +) + + +def test_str_is_rejected_with_the_two_option_message(): + with pytest.raises(TypeError) as info: + reject_str_value("/tmp/big.mp4") + message = str(info.value) + assert "encode()" in message and "Filepath" in message + + +def test_bare_pathlike_is_rejected_pointing_at_filepath(): + from pathlib import Path + + with pytest.raises(TypeError, match="Filepath"): + as_fileobj(Path("/tmp/x")) + + +def test_alien_type_names_the_extension_point(): + with pytest.raises(TypeError, match="register"): + as_fileobj(12345) + + +@pytest.mark.parametrize( + "value", [b"bytes", bytearray(b"ba"), memoryview(b"mv")] +) +def test_bytes_like_members(value): + assert is_bytes_like(value) + assert as_fileobj(value).read() == bytes(value) + + +def test_open_binary_files_dispatch_via_io_iobase(): + # The trap: @register(typing.BinaryIO) is accepted and never fires + # (io.BytesIO is not in its MRO — ADR-0005 §1). io.IOBase is the base + # that actually covers real file objects. + assert as_fileobj(io.BytesIO(b"data")).read() == b"data" + + +def test_filepath_normalises_pathlike_and_reads(tmp_path): + target = tmp_path / "payload.bin" + target.write_bytes(b"CONTENT") + ref = Filepath(target) # a Path is fine; normalised to str + assert isinstance(ref.path, str) + with as_fileobj(ref) as fileobj: + assert fileobj.read() == b"CONTENT" + # re-readable (N1-total): a second consumption works + with as_fileobj(ref) as fileobj: + assert fileobj.read() == b"CONTENT" + assert pickle.loads(pickle.dumps(ref)) == ref + + +def test_chunks_concatenates_and_is_one_shot(): + ref = Chunks(iter([b"ab", b"cd", b"e"])) + assert as_fileobj(ref).read() == b"abcde" + with pytest.raises(ValueError, match="one-shot"): + as_fileobj(ref) # the documented error, not a silent b'' write + + +def test_streamable_reopens_per_write(): + ref = Streamable(lambda: iter([b"xy", b"z"])) + assert as_fileobj(ref).read() == b"xyz" + assert as_fileobj(ref).read() == b"xyz" # fresh stream each time + + +def test_iterable_fileobj_partial_reads(): + fileobj = as_fileobj(Chunks(iter([b"abc", b"defg"]))) + assert fileobj.read(2) == b"ab" + assert fileobj.read(3) == b"cde" + assert fileobj.read() == b"fg" + + +def test_write_strategies_are_picklable_values(): + from s3dol.reads import BytesReads, StreamReads + from s3dol.writes import SimpleWrites, TransferWrites + + for strategy in (TransferWrites(), SimpleWrites(), BytesReads(), StreamReads()): + assert pickle.loads(pickle.dumps(strategy)) == strategy diff --git a/s3dol/utility.py b/s3dol/utility.py deleted file mode 100644 index dad5362..0000000 --- a/s3dol/utility.py +++ /dev/null @@ -1,127 +0,0 @@ -""" -This module provides utility functions and custom exceptions for the `s3dol` package. - -Classes: -- S3DolException: Base exception class for `s3dol` package. -- S3KeyError: Exception raised when a key error occurs. -- NoSuchKeyError: Exception raised when a key is not found. -- KeyNotValidError: Exception raised when a key is not valid. -- GetItemForKeyError: Exception raised when an error occurs while getting an item for a key. -- S3HttpError: Exception raised when an HTTP error occurs. -- S3Not200StatusCodeError: Exception raised when an HTTP status code other than 200 is returned. - -Functions: -- raise_on_error: Function that raises an error. -- return_none_on_error: Function that returns None. -- return_empty_tuple_on_error: Function that returns an empty tuple. -- Resp.status_code: Function that returns the HTTP status code from a dictionary. -- Resp.contents: Function that returns the contents from a dictionary. -- Resp.key: Function that returns the key from a dictionary. -- Resp.common_prefixes: Function that returns the common prefixes from a dictionary. -- Resp.prefix: Function that returns the prefix from a dictionary. -- Resp.buckets: Function that returns the buckets from a dictionary. -- Resp.body: Function that returns the body from a dictionary. -- Resp.ascertain_status_code: Function that raises an error if the HTTP status code is not 200. -- Resp.ascertain_200_status_code: Function that raises an error if the HTTP status code is not 200 and the response is not a dictionary with a 'ResponseMetadata' key. -""" - -"""""" -from dol.paths import path_get, OnErrorType - - -class S3DolException(Exception): ... - - -class S3KeyError(KeyError): ... - - -class NoSuchKeyError(S3KeyError): ... - - -class KeyNotValidError(S3KeyError): ... - - -class GetItemForKeyError(S3KeyError): ... - - -class S3HttpError(RuntimeError): ... - - -class S3Not200StatusCodeError(S3HttpError): ... - - -raise_on_error: OnErrorType -return_none_on_error: OnErrorType -return_empty_tuple_on_error: OnErrorType - - -def raise_on_error(d: dict): - raise - - -def return_none_on_error(d: dict): - return None - - -def return_empty_tuple_on_error(d: dict): - return () - - -class Resp: - @staticmethod - def status_code(d, on_error: OnErrorType = raise_on_error): - return path_get(d, ["ResponseMetadata", "HTTPStatusCode"], on_error) - - @staticmethod - def contents(d, on_error: OnErrorType = return_empty_tuple_on_error): - return path_get(d, ["Contents"], on_error) - - @staticmethod - def key(d, on_error: OnErrorType = raise_on_error): - return path_get(d, ["Key"], on_error) - - @staticmethod - def common_prefixes(d, on_error: OnErrorType = return_empty_tuple_on_error): - return path_get(d, ["CommonPrefixes"], on_error) - - @staticmethod - def prefix(d, on_error: OnErrorType = raise_on_error): - return path_get(d, ["Prefix"], on_error) - - @staticmethod - def buckets(d, on_error: OnErrorType = return_empty_tuple_on_error): - return path_get(d, ["Buckets"], on_error) - - @staticmethod - def body(d, on_error: OnErrorType = return_none_on_error): - return path_get(d, ["Body"], on_error) - - @staticmethod - def ascertain_status_code( - d, - status_code=200, - raise_error=S3HttpError, - *error_args, - **error_kwargs, - ): - if Resp.status_code(d) != status_code: - raise raise_error(*error_args, **error_kwargs) - - @staticmethod - def ascertain_200_status_code(d): - status_code = Resp.status_code(d, return_none_on_error) - if status_code != 200: - if not isinstance(d, dict): - raise S3Not200StatusCodeError( - "Yeah, that's not even a dict, so doubt it's even a response." - f"I'm expecting a response over here. Instead I got a {type(d)}" - ) - elif "ResponseMetadata" in d: - raise S3Not200StatusCodeError( - f"Status code was not 200. Was {d['ResponseMetadata']}. " - f"ResponseMetadata is {d['ResponseMetadata']}" - ) - else: - raise S3Not200StatusCodeError( - f"Status code was not 200. In fact, the response dict didn't even have a ResponseMetadata key" - ) diff --git a/s3dol/values.py b/s3dol/values.py new file mode 100644 index 0000000..7a7a45d --- /dev/null +++ b/s3dol/values.py @@ -0,0 +1,204 @@ +"""Value refs and the write domain (ADR-0005). + +The write domain is a small, closed, explicitly-named union:: + + BytesSource = bytes | bytearray | memoryview | BinaryIO | Filepath | Chunks | Streamable + +``Filepath``/``Chunks``/``Streamable`` are tiny frozen dataclasses — value +*refs*, not values (h5py's ``SoftLink``/``ExternalLink`` are the precedent: +purpose-built objects meaning "the value is a reference to content elsewhere", +assigned through an ordinary ``__setitem__``). + +The law that makes the asymmetric read/write types safe (ADR-0005 §3): + +- **N1 (canonical form)**: ``normalize: WriteDomain -> bytes`` is total on + ``bytes | bytearray | memoryview | Filepath`` and **one-shot** on + ``BinaryIO | Chunks | Streamable`` — a stream ref is consumed by its first + write; assigning the same ref twice is a documented error, not a second copy. +- **N3 (the honest invariant)**: ``s[k] = v => s[k] == normalize(v)`` — for + the re-readable half. + +``str`` is rejected, loudly, on **decidability** (§2): ``s['config'] = '{"a":1}'`` +and ``s['video'] = '/tmp/big.mp4'`` are both overwhelmingly plausible, and any +rule distinguishing them is a latent data-corruption bug. One word — +``Filepath(p)`` — removes the ambiguity. Bare ``os.PathLike`` is rejected too: +accepting it while rejecting ``str`` would be a confusing half-rule. + +>>> reject_str_value('{"a": 1}') +Traceback (most recent call last): +... +TypeError: ... +""" + +from __future__ import annotations + +import io +import os +from dataclasses import dataclass, field +from functools import singledispatch +from typing import BinaryIO, Iterable, Union + +# --------------------------------------------------------------------------- # +# The refs +# --------------------------------------------------------------------------- # + + +@dataclass(frozen=True) +class Filepath: + """The value *is the content of this file*. Re-readable (N1-total).""" + + path: str + + def __post_init__(self): + # A Path is fine; normalise it to str so the ref stays picklable/hashable. + object.__setattr__(self, "path", os.fspath(self.path)) + + +@dataclass(frozen=True) +class Chunks: + """The value is the concatenation of an iterable of ``bytes`` chunks. + **One-shot** unless the iterable is re-iterable (a list is; a generator is + not — and a consumed generator writes ``b''``, which is why assigning the + same one-shot ref twice is a documented error).""" + + chunks: Iterable[bytes] + consumed: list = field(default_factory=list, compare=False, repr=False) + + def _consume_once(self): + if self.consumed: + raise ValueError( + "This Chunks ref was already written once. A stream ref is " + "one-shot (ADR-0005 N1): re-create the ref (or pass a list of " + "chunks, which is re-readable)." + ) + self.consumed.append(True) + return self.chunks + + +@dataclass(frozen=True) +class Streamable: + """The value is whatever a zero-arg factory's stream yields — for sources + that must be (re)opened at write time, e.g. an HTTP download. The factory + is called once per write, so unlike :class:`Chunks` this ref IS safely + re-writable (each write gets a fresh stream).""" + + open_stream: object # zero-arg callable -> BinaryIO | Iterable[bytes] + + +#: The static write domain (``BinaryIO`` is for mypy; runtime dispatch uses +#: ``io.IOBase`` — see :func:`as_fileobj`). +BytesSource = Union[ + bytes, bytearray, memoryview, BinaryIO, Filepath, Chunks, Streamable +] + + +def reject_str_value(v): + """The one error message for the ``str`` ambiguity (ADR-0005 §2).""" + raise TypeError( + "A str value is ambiguous here: did you mean its utf-8 bytes, or a " + "filepath?\n" + " content : s[k] = v.encode() (or wrap the store: " + "dol.wrap_kvs(store, data_of_obj=lambda s: s.encode(), " + "obj_of_data=lambda b: b.decode()))\n" + " filepath: s[k] = s3dol.Filepath(v)" + ) + + +class _IterableFileObj(io.RawIOBase): + """A minimal read-only file object over an iterable of bytes chunks.""" + + def __init__(self, chunks: Iterable[bytes]): + self._iterator = iter(chunks) + self._buffer = b"" + + def readable(self): + return True + + def readinto(self, target): + while len(self._buffer) < len(target): + try: + self._buffer += next(self._iterator) + except StopIteration: + break + n = min(len(target), len(self._buffer)) + target[:n] = self._buffer[:n] + self._buffer = self._buffer[n:] + return n + + +# --------------------------------------------------------------------------- # +# as_fileobj — the open extension point (ADR-0005 §1) +# --------------------------------------------------------------------------- # + + +@singledispatch +def as_fileobj(v) -> BinaryIO: + """A binary file object for any member of the write domain. + + ``functools.singledispatch``: the union is open for extension (users + register their own ref types) and closed for modification. Everything + non-``bytes`` routes through ``upload_fileobj`` downstream, so seekability + is s3transfer's problem. + """ + if isinstance(v, str): + reject_str_value(v) + if isinstance(v, os.PathLike): + raise TypeError( + f"Got a bare path object ({type(v).__name__}). Wrap it: " + f"s3dol.Filepath({v!r}) — one word, zero ambiguity (ADR-0005 §2)." + ) + raise TypeError( + f"Cannot write a {type(v).__name__} to an s3dol store. Accepted: " + f"bytes, bytearray, memoryview, an open binary file, " + f"s3dol.Filepath(path), s3dol.Chunks(iterable_of_bytes), or " + f"s3dol.Streamable(open_stream). Extend with " + f"s3dol.values.as_fileobj.register(YourType)." + ) + + +@as_fileobj.register(bytes) +def _(v: bytes): + return io.BytesIO(v) + + +@as_fileobj.register(bytearray) +def _(v: bytearray): + return io.BytesIO(bytes(v)) + + +@as_fileobj.register(memoryview) +def _(v: memoryview): + return io.BytesIO(bytes(v)) + + +# Register on io.IOBase, NEVER typing.BinaryIO: @register(BinaryIO) is accepted +# at definition time and then never fires — io.BytesIO is not in +# typing.BinaryIO's MRO (measured, ADR-0005 §1). io.IOBase covers BytesIO, +# BufferedReader, botocore.response.StreamingBody, SpooledTemporaryFile and +# urllib3 responses. +@as_fileobj.register(io.IOBase) +def _(v: io.IOBase): + return v + + +@as_fileobj.register(Filepath) +def _(v: Filepath): + return open(v.path, "rb") + + +@as_fileobj.register(Chunks) +def _(v: Chunks): + return _IterableFileObj(v._consume_once()) + + +@as_fileobj.register(Streamable) +def _(v: Streamable): + stream = v.open_stream() + if isinstance(stream, io.IOBase): + return stream + return _IterableFileObj(stream) + + +def is_bytes_like(v) -> bool: + """Whether ``v`` is an in-memory bytes value (the ``PutObject`` fast path).""" + return isinstance(v, (bytes, bytearray, memoryview)) diff --git a/s3dol/writes.py b/s3dol/writes.py new file mode 100644 index 0000000..509f0a3 --- /dev/null +++ b/s3dol/writes.py @@ -0,0 +1,99 @@ +"""Write strategies (ADR-0005 §5): how ``s[k] = v`` reaches the wire. + +A strategy is a callable ``(client, bucket, key, value) -> None``, injected at +store construction — ``__setitem__`` stays ``__setitem__``; *how* it uploads is +a constructor parameter. This is the answer to "infra-specific optimisation +without polluting the interface": multipart is *transport*, not storage-model. + +Strategies are frozen dataclasses, not closures, because every s3dol store +must pickle (ADR-0008 §6 / ADR-0012 D1) and a store carries its strategy. + +Every non-``bytes`` source routes through ``upload_fileobj``/``TransferManager``, +never ``PutObject`` — so seekability is s3transfer's problem, and the size +threshold is never consulted for a non-seekable source (where it is undecidable +anyway). Note ``io.UnsupportedOperation`` and botocore's +``UnseekableStreamError`` raise *before the request is built*, so they reach the +caller directly (they are not ``ClientError``s and the error seam does not — +and must not — convert them to ``KeyError``). + +>>> strategy = transfer_writes(multipart_threshold=16 * 2**20) +>>> import pickle; pickle.loads(pickle.dumps(strategy)) == strategy +True +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Protocol + +from s3dol.values import as_fileobj, is_bytes_like, reject_str_value + + +class WriteStrategy(Protocol): + def __call__(self, client: Any, bucket: str, key: str, value: Any) -> None: ... + + +#: boto3's own default multipart threshold (8 MiB). +DEFAULT_MULTIPART_THRESHOLD = 8 * 2**20 + + +@dataclass(frozen=True) +class TransferWrites: + """The default: small in-memory values take a single ``PutObject``; file + objects and refs go through ``upload_fileobj`` (which goes multipart above + the threshold, transparently). The overhead on small objects is one branch. + + Providers that cap multipart parts (Scaleway: 1000 — see the preset's + ``Capabilities.max_multipart_parts``) may need a larger + ``multipart_chunksize`` for very large objects: parts ≈ size / chunksize. + """ + + multipart_threshold: int = DEFAULT_MULTIPART_THRESHOLD + multipart_chunksize: int = DEFAULT_MULTIPART_THRESHOLD + max_concurrency: int = 10 + use_threads: bool = True + + def __call__(self, client, bucket: str, key: str, value) -> None: + if isinstance(value, str): + reject_str_value(value) + if is_bytes_like(value) and len(value) < self.multipart_threshold: + client.put_object(Bucket=bucket, Key=key, Body=bytes(value)) + return + from boto3.s3.transfer import TransferConfig + + config = TransferConfig( + multipart_threshold=self.multipart_threshold, + multipart_chunksize=self.multipart_chunksize, + max_concurrency=self.max_concurrency, + use_threads=self.use_threads, + ) + with as_fileobj(value) as fileobj: + client.upload_fileobj(fileobj, bucket, key, Config=config) + + +@dataclass(frozen=True) +class SimpleWrites: + """Everything through a single ``PutObject`` — refs are materialised in + memory first. For small-object workloads and providers with no multipart + at all (``Capabilities.multipart=False``).""" + + def __call__(self, client, bucket: str, key: str, value) -> None: + if isinstance(value, str): + reject_str_value(value) + if not is_bytes_like(value): + with as_fileobj(value) as fileobj: + value = fileobj.read() + client.put_object(Bucket=bucket, Key=key, Body=bytes(value)) + + +def transfer_writes(**kwargs) -> TransferWrites: + """Factory spelling of :class:`TransferWrites` (the constructor-injection + idiom of ADR-0005 §5: ``BucketStore(..., writes=transfer_writes(...))``).""" + return TransferWrites(**kwargs) + + +def simple_writes() -> SimpleWrites: + return SimpleWrites() + + +DEFAULT_WRITES = TransferWrites() From 024e682c4ec2b3733c747453b39a96cf8e7c2efc Mon Sep 17 00:00:00 2001 From: Thor Whalen <1906276+thorwhalen@users.noreply.github.com> Date: Tue, 11 Aug 2026 18:19:02 +0100 Subject: [PATCH 3/3] fix(delete_many): report partial-failure keys in the caller's key space MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An exception payload carrying keys is one of the shapes ADR-0011 lists as inexpressible for a codec boundary (the raise site has no boundary frame) — surfaced concretely by the dol P1 trial (dol#86). But a free function holds the store, so the inverse walk IS available here: .succeeded/.failures now carry the keys the caller passed, not wire keys they never used. Claude-Session: https://claude.ai/code/session_01KqivR5Be21Wh7yRdEG4fiY --- s3dol/capabilities.py | 22 +++++++++++++++++++++- s3dol/tests/test_capabilities.py | 18 ++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/s3dol/capabilities.py b/s3dol/capabilities.py index 610077b..6909e4b 100644 --- a/s3dol/capabilities.py +++ b/s3dol/capabilities.py @@ -351,10 +351,30 @@ def delete_many(store, keys: Iterable[str]) -> None: failures[key] = S3PartialFailure( str(code_of(error)), succeeded=[], failures={} ) + if failures or succeeded: + # Map the reported keys BACK to the caller's key space. A partial- + # failure payload is "keys embedded in an exception" — one of the + # shapes ADR-0011 lists as inexpressible for a codec boundary, because + # the raise site has no boundary frame. Here we have the store as an + # argument, so the inverse walk is available and the caller gets keys + # they can act on rather than wire keys they never used. + def outward(wire_key): + try: + return ( + _map_key_outward(store, leaf._key_of_id(wire_key)) + if len(chain) > 1 + else leaf._key_of_id(wire_key) + ) + except Exception: + return wire_key # un-mappable: report it raw rather than drop it + + succeeded = [outward(k) for k in succeeded] + failures = {outward(k): error for k, error in failures.items()} if failures: raise S3PartialFailure( f"delete_many: {len(failures)} of {len(wire_keys)} deletions " - f"failed ({len(succeeded)} succeeded).", + f"failed ({len(succeeded)} succeeded). .succeeded/.failures are " + f"keyed in YOUR key space, not the wire's.", succeeded=succeeded, failures=failures, ) diff --git a/s3dol/tests/test_capabilities.py b/s3dol/tests/test_capabilities.py index 060cefe..5f59413 100644 --- a/s3dol/tests/test_capabilities.py +++ b/s3dol/tests/test_capabilities.py @@ -189,6 +189,24 @@ def _op_bulk_delete(self, ids): assert info.value.succeeded == ["a"] and set(info.value.failures) == {"b"} +def test_partial_failure_reports_keys_in_the_callers_key_space(): + # "Keys embedded in an exception payload" is one of the shapes a codec + # boundary cannot map (the raise site has no boundary frame) — but a free + # function holds the store, so the inverse walk IS available here. + from s3dol.testing import FakeBucketStore, _FakeBackend + + class Failing(FakeBucketStore): + def _op_bulk_delete(self, ids): + return [], [(ids[0], "AccessDenied", "nope")] + + store = Failing("b", backing=_FakeBackend({"p/x/a": b"1"}), prefix="p") + wrapped = KeyCodecs.prefixed("x/")(store) + with pytest.raises(S3PartialFailure) as info: + s3dol.delete_many(wrapped, ["a"]) + # 'a' — the key the caller passed — not the wire key 'p/x/a' + assert set(info.value.failures) == {"a"} + + def test_delete_many_emulates_when_batch_delete_is_unsupported(): # GCS and any provider that rejects the mandatory DeleteObjects checksum: # loop DeleteObject; identical observable result, only cost differs.