diff --git a/README.md b/README.md index 1c7d885..72881ba 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,63 @@ To install: ```pip install s3dol``` [Documentation](https://i2mint.github.io/s3dol/) +## Quick start + +```python +import s3dol + +s = s3dol.s3_store('my-bucket') # MutableMapping[str, bytes] +s['hello.txt'] = b'world' +s['hello.txt'] # b'world' +list(s); 'hello.txt' in s; del s['hello.txt'] + +s = s3dol.s3_store('my-bucket', prefix='logs/') # scoped to a prefix +s = s3dol.s3_store('open-data', anon=True) # public bucket, no credentials +s = s3dol.s3_store('b', preset='minio', endpoint_url='http://localhost:9000') +``` + +Starting from nothing (the default never creates a bucket from a typo): + +```python +s = s3dol.s3_store('brand-new-bucket', on_missing_bucket='create') +``` + +**Big objects go through the same interface** — no `upload_multipart` method: + +```python +s['video.mp4'] = s3dol.Filepath('/tmp/video.mp4') # streamed, multipart above 8 MiB +s['stream.bin'] = s3dol.Chunks(chunk_iterator) # never fully in memory +``` + +**Keyed capabilities are stores you index**, not methods (a `dol` key wrapper +hands a method the *unmapped* key — so s3dol has none): + +```python +s3dol.handles(s)['video.mp4'].read(offset=0, length=1024) # ranged read +s3dol.handles(s)['video.mp4'].url(expires_in=3600) # presigned URL +s3dol.urls(s)['video.mp4'] # ...or directly +s3dol.info(s)['video.mp4'].size # one HeadObject +``` + +Everything else is a free function taking the store first: + +```python +s3dol.sub(s, 'folder/'); s3dol.prefixes(s) +s3dol.delete_many(s, ['a', 'b']); s3dol.delete_bucket(endpoint, 'name', force=True) +``` + +**Test without a cloud** — no network, no docker, no moto: + +```python +from s3dol.testing import mock_s3, run_conformance + +def test_my_service(): + assert MediaService(mock_s3(data={'a.mp4': b'...'})).play('a.mp4') +``` + +`run_conformance(make_store)` is exported too: run the same laws against your +own store (or a sibling `*dol` package's). + ## Heads-up: v1 is coming — run `s3dol.diagnose()` now A major, behaviour-clarifying v1 is in progress @@ -23,7 +80,7 @@ Before upgrading to v1, run the diagnosis with the same arguments you pass ```python import s3dol -s3dol.diagnose(bucket_name="my-bucket", endpoint_url="...", path="...") +s3dol.diagnose(bucket_name='my-bucket', endpoint_url='...', path='...') ``` It prints what resolves (endpoint, region, signing, credential *source* — never diff --git a/conftest.py b/conftest.py index 1b661d5..b231c0b 100644 --- a/conftest.py +++ b/conftest.py @@ -33,11 +33,18 @@ _S3_FREE_TEST_STEMS = frozenset( { "test_url_for", - # Layer A (v1) suites: tier-1 pure + moto-mocked tier-2 — no endpoint. + # v1 suites: tier-1 pure + moto-mocked tier-2 — no live endpoint. "test_errors", "test_presets", "test_connection", "test_diagnose", + "test_values", + "test_keyed_surface", + "test_conformance", + "test_base", + "test_capabilities", + "test_store_shim", + "test_recipes", } ) diff --git a/misc/docs/decisions/0011-keyed-capability-surface.md b/misc/docs/decisions/0011-keyed-capability-surface.md index c108770..7b21606 100644 --- a/misc/docs/decisions/0011-keyed-capability-surface.md +++ b/misc/docs/decisions/0011-keyed-capability-surface.md @@ -227,6 +227,39 @@ documented limitation, and the real fix is upstream: `content_url` must resolve chain **and** call the innermost `url_for` with the resolved key — resolving without that second half would double-transform. +#### D3b-amended (2026-08-11, during implementation) — the guarded method cannot serve both call paths + +The guard above assumed one call path. There are **two**, and they hand the *same signature* +**different key domains**: + +| caller | what `url_for` receives | correct action | +|---|---|---| +| `dol.content_url(wrapped, k)` | the key already mapped through the **outer** layers — because the walk stops at the layer owning `url_for` "*because that layer applies its own*" (`dol/content.py::_url_for_provider_and_key`) | apply `self._id_of_key` | +| `wrapped.url_for(k)` (delegation) | the raw **outer** key | resolve through the whole chain | + +Applying `_id_of_key` is right for the first and wrong for the second; re-resolving through the +chain is right for the second and **double-transforms** the first (measured: a store scoped to +`p/` under a `x/` key codec produced `p/x/x/g`). Nothing at the callee distinguishes them — +this is dol#83's defect reaching the one method the ADR kept. + +**Decision:** `url_for` applies `self._id_of_key(k)` (dol's stated contract), and **refuses** +— `NotSupported`, naming `s3dol.urls(store)[k]` — as soon as the leaf has ever been wrapped. +Consequences, stated plainly: + +- `dol.content_url` is correct for an **unwrapped** s3dol store (the common case, and the one + `dol/content.py`'s docstring shows) and **raises** for a wrapped one. Loud, never wrong. +- The refusal is driven by a *sticky flag* set when dol probes the leaf (`hasattr(store, + 'KeysView')` in `Store.__init__`), implemented as a descriptor because a `KvReader` leaf + would otherwise satisfy the probe from `MappingViewMixin` and never see it. Unlike the + `wrapped_self` weakref, the flag survives the wrapper's death — so the refusal is reliable, + not intermittent. The documented false positive stands: a once-wrapped leaf later used bare + also refuses. + +**This is the P1 evidence for dol#86/#83**: a keyed capability *method* is unfixable in the +general case — not merely fragile, and not fixable by a better resolution primitive, because +the ambiguity is in the *call*, not the resolution. Sibling stores (D2) route through +`__getitem__`, which has one key domain and one caller, and so remain correct by construction. + ### D4 — Destructive operations are free functions `delete_many(keys)` and `EndpointStore.delete(name, force=True)` both leave Layer B. diff --git a/misc/docs/state-of-play.md b/misc/docs/state-of-play.md index 99a70b6..b5782fa 100644 --- a/misc/docs/state-of-play.md +++ b/misc/docs/state-of-play.md @@ -67,7 +67,20 @@ consumer's requirements. | Resolved design questions | **§7** / **#14** → [ADR-0011](decisions/0011-keyed-capability-surface.md) · **§8** / **#15** → [ADR-0012](decisions/0012-credential-and-endpoint-resolution.md) | | Open design questions | none — implementation is unblocked; see §11 | -**Nothing has been implemented.** The repo still ships v0.1.9 unchanged. +**Implementation is under way** (2026-08-11). Layer A landed as an additive +0.1.x change (PR #19: `errors.py`, `presets.py`, `connection.py`, `_diagnose.py` +— ADR-0007 §5 step 0, so dependents can record what resolves *before* anything +moves), and the v1 surface followed on top (`values`/`writes`/`reads`/`base`/ +`capabilities`/`recipes`/`testing` + the `store.py` compat shim). Read §11 for +what remains. + +One design amendment came out of implementation and is recorded where it +belongs, in [ADR-0011 §D3b-amended](decisions/0011-keyed-capability-surface.md): +the guarded `url_for` shim **cannot** serve both of its call paths, because +`dol.content_url` and a direct delegated call hand the same signature different +key domains. It now refuses once the leaf has been wrapped. That is the P1 +evidence for dol#86/#83 — a keyed capability *method* is unfixable in the +general case, and not for want of a better resolution primitive. ## 3. The twelve ADRs, in one paragraph each diff --git a/pyproject.toml b/pyproject.toml index 16a762b..50794b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,12 @@ classifiers = [ "Topic :: Software Development :: Libraries :: Python Modules", ] dependencies = [ - "dol", + # A hard floor, not a preference (ADR-0006 §3's upstream policy: raise the + # floor the day each fix lands, and delete the local workaround in the same + # commit). v1 needs dol.inner_most_key exported (0.3.62), content_url + # resolving keys through the wrapper chain (0.3.62), and filesys + # is_valid_key fixed — all shipped by 0.3.63. + "dol>=0.3.63", "boto3", # ADR-0012: the endpoint env vars (AWS_ENDPOINT_URL_S3 / AWS_ENDPOINT_URL) do not exist # before botocore 1.31, and ADR-0003 §3's checksum knobs @@ -49,12 +54,17 @@ test = [ "pytest>=7.0", "pytest-cov>=4.0", "moto[s3]>=5.0", + # conformance assertion 2b fetches a presigned URL; moto patches + # `requests` but NOT urllib (which would leave the machine). Declared + # explicitly rather than relied on transitively via moto[s3]. + "requests", ] dev = [ "pytest>=7.0", "pytest-cov>=4.0", "ruff>=0.1.0", "moto[s3]>=5.0", + "requests", ] docs = [ "sphinx>=6.0", @@ -144,10 +154,14 @@ enabled = false [tool.wads.ci.testing] python_versions = [ "3.10", + "3.12", ] pytest_args = [ "-v", "--tb=short", + # Every documented example is executable against the in-memory fake, so + # doctests run in the default gate (ADR-0008). + "--doctest-modules", ] coverage_enabled = true coverage_threshold = 0 diff --git a/s3dol/__init__.py b/s3dol/__init__.py index edae93b..d957e00 100644 --- a/s3dol/__init__.py +++ b/s3dol/__init__.py @@ -1,26 +1,117 @@ -""" -s3 (through boto3) with a simple (dict-like or list-like) interface -""" +"""s3dol — S3 and S3-compatible object storage behind a dict-like interface. + +The one-liner:: + + import s3dol + s = s3dol.s3_store('my-bucket') # MutableMapping[str, bytes] + s['k'] = b'v'; s['k']; list(s); del s['k'] + +Keyed capabilities are **sibling stores you index** (never methods — a dol key +wrapper would hand a method the unmapped key; ADR-0011):: + + s3dol.handles(s)['k'] # ObjectHandle (ranged reads, streams, metadata) + s3dol.urls(s)['k'] # presigned URL + s3dol.info(s)['k'] # ObjectInfo + +Non-keyed operations are free functions taking the store first:: + + s3dol.sub(s, 'folder/') s3dol.prefixes(s) + s3dol.delete_many(s, keys) s3dol.delete_bucket(endpoint, name, force=True) -from s3dol.base import S3Dol -from s3dol.store import S3Store +Test without a cloud: ``from s3dol.testing import mock_s3, run_conformance``. +Migrating from v0? ``s3dol.diagnose(**your_S3Store_kwargs)`` prints what +resolves, from where, and whether v1 moves it. The legacy ``S3Store`` keeps +working via ``s3dol.store`` (deprecated, removed in v2). -# New (v1) layer-A names, loaded lazily (PEP 562) so `import s3dol` does not -# pay for what a script does not use. `s3dol.diagnose(...)` is the ADR-0007 §5 -# step-0 migration safety mechanism — run it with your current S3Store kwargs -# to see what resolves, from where, and whether the upcoming v1 moves it. +Everything here loads lazily (PEP 562): ``import s3dol`` does not import boto3. +""" + +# name -> module path (attribute of the same name), or (module path, attribute) _LAZY_ATTRIBUTES = { - "diagnose": ("s3dol._diagnose", "diagnose"), + # Layer C — recipes + "s3_store": "s3dol.recipes", + "S3Texts": "s3dol.recipes", + "S3Jsons": "s3dol.recipes", + "S3Pickles": "s3dol.recipes", + # Layer B — stores and the handle + "BucketCollection": "s3dol.base", + "BucketReader": "s3dol.base", + "BucketStore": "s3dol.base", + "EndpointCollection": "s3dol.base", + "EndpointReader": "s3dol.base", + "EndpointStore": "s3dol.base", + "ObjectHandle": "s3dol.base", + "ObjectInfo": "s3dol.base", + "S3Profiles": "s3dol.base", + # sibling capability stores + accessors (ADR-0011 D2) + "BucketHandles": "s3dol.capabilities", + "BucketUrls": "s3dol.capabilities", + "BucketInfo": "s3dol.capabilities", + "handles": "s3dol.capabilities", + "urls": "s3dol.capabilities", + "info": "s3dol.capabilities", + # free functions (ADR-0011 D3/D4) + "sub": "s3dol.capabilities", + "prefixes": "s3dol.capabilities", + "delete_many": "s3dol.capabilities", + "delete_bucket": "s3dol.capabilities", + # value refs (ADR-0005) + "Filepath": "s3dol.values", + "Chunks": "s3dol.values", + "Streamable": "s3dol.values", + # Layer A — connection + presets "S3Connection": "s3dol.connection", "resolve": "s3dol.connection", "Resolution": "s3dol.connection", + "StaticCredentials": "s3dol.connection", + "ProfileCredentials": "s3dol.connection", + "CallableCredentials": "s3dol.connection", "Preset": "s3dol.presets", "Capabilities": "s3dol.presets", "register_preset": ("s3dol.presets", "register"), + # errors (the taxonomy — `except s3dol.S3Error` catches the package) + "S3Error": "s3dol.errors", + "ObjectNotFound": "s3dol.errors", + "BucketNotFound": "s3dol.errors", + "KeyNotValid": "s3dol.errors", + "ObjectArchived": "s3dol.errors", + "AccessDenied": "s3dol.errors", + "CredentialsError": "s3dol.errors", + "NotSupported": "s3dol.errors", + "BucketNotEmpty": "s3dol.errors", + "S3PartialFailure": "s3dol.errors", + "ConfigurationError": "s3dol.errors", + "S3DolWarning": "s3dol.errors", + # diagnostics + "diagnose": ("s3dol._diagnose", "diagnose"), + # legacy (deprecated; removed in v2) + "S3Store": "s3dol.store", } +__all__ = sorted(_LAZY_ATTRIBUTES) + ["S3Dol", "__version__"] + def __getattr__(name): + if name == "__version__": + import importlib.metadata + + try: + return importlib.metadata.version("s3dol") + except importlib.metadata.PackageNotFoundError: + return "unknown" + if name == "S3Dol": + # v0's profiles->buckets->keys explorer, renamed (ADR-0007 §1). + import warnings + + warnings.warn( + "s3dol.S3Dol is deprecated (removed in v2): it is now " + "s3dol.S3Profiles — its keys are AWS profile names.", + DeprecationWarning, + stacklevel=2, + ) + from s3dol.base import S3Profiles + + return S3Profiles try: target = _LAZY_ATTRIBUTES[name] except KeyError: @@ -33,12 +124,12 @@ def __getattr__(name): module_path, attribute = target, name value = getattr(importlib.import_module(module_path), attribute) # Cache the resolved attribute. This ALSO deliberately overrides the - # submodule binding that `import s3dol.diagnose` just created on this - # package — without it, `s3dol.diagnose` would resolve to the module on - # every access after the first, shadowing the function. + # submodule binding a same-named submodule import creates on this package + # (e.g. `info` vs a hypothetical info module) — without it, the second + # access would resolve to the module and shadow the function. globals()[name] = value return value def __dir__(): - return sorted(set(globals()) | set(_LAZY_ATTRIBUTES)) + return sorted(set(globals()) | set(__all__)) diff --git a/s3dol/_diagnose.py b/s3dol/_diagnose.py index 32f335e..fbd84a8 100644 --- a/s3dol/_diagnose.py +++ b/s3dol/_diagnose.py @@ -26,7 +26,7 @@ import io import re import sys -from typing import Mapping, Optional +from typing import Mapping, Optional, Union from s3dol.connection import S3Connection, Resolution, load_aws_config, resolve from s3dol.errors import redact @@ -49,13 +49,48 @@ } ) -_SECRET_ARG_NAMES = frozenset({"aws_secret_access_key", "aws_session_token"}) +#: Argument names whose VALUE is safe to echo. An **allowlist**, not a +#: denylist: a denylist of the two v0 secret names let every v1 shape through, +#: so ``credentials=('AK', 'SECRET')`` printed the secret verbatim into the one +#: artifact users are told to run and share (ADR-0012 D6: diagnose holds no +#: credential value, only a provenance label). +_ECHOABLE_ARG_NAMES = frozenset( + { + "bucket_name", + "path", + "prefix", + "region_name", + "profile_name", + "profile", + "preset", + "make_bucket", + "skip_bucket_check", + "is_supabase_endpoint", + "on_missing_bucket", + "anon", + "addressing_style", + "checksum", + "payload_signing_enabled", + "deny_means_absent", + "signature_version", + } +) + +#: Echoed, but redacted first (a URL is the residual credential carrier). +_REDACTED_ARG_NAMES = frozenset({"endpoint_url"}) #: Reasons a v0 branch is excluded from the differential (ADR-0012 D7): these #: two branches build clients through the module-level ``boto3.client(...)``, #: which resolves through the process-global cached DEFAULT_SESSION — they are #: not functions of their arguments. _EXCLUDED_BRANCHES = { + "env-profile (v0 RAISES: no env credentials)": ( + "v0 raises S3DolException('Missing AWS credentials in environment " + "variables') for this call. v1 instead falls through to botocore's " + "credential chain, so it may succeed — with whatever identity the " + "chain resolves (~/.aws, SSO, IMDS). If you relied on v0 failing " + "loudly here, add an explicit check." + ), "supabase": ( "v0's Supabase branch builds its client through boto3's process-global " "session, so its behaviour is not a function of these arguments. Note: " @@ -105,6 +140,12 @@ def _v0_mangle(args: Mapping, environ: Mapping) -> tuple[str, Optional[dict]]: ) return "env-credentials (endpoint dropped)", effective if profile == "environment variables": + if not env_creds: + # v0's get_client('environment variables') RAISES S3DolException + # here. v1 falls through to botocore's chain, which may resolve a + # DIFFERENT identity (~/.aws, SSO, IMDS) — a behaviour change worth + # naming, and not expressible as a resolution diff. + return "env-profile (v0 RAISES: no env credentials)", None effective.update( aws_access_key_id=environ.get("AWS_ACCESS_KEY_ID"), aws_secret_access_key=environ.get("AWS_SECRET_ACCESS_KEY"), @@ -141,6 +182,40 @@ def _spec_from_v0_args(args: Mapping) -> S3Connection: ) +def _credential_identity(spec, environ: Mapping) -> tuple: + """A comparable, non-secret credential *identity* (ADR-0012 D7: compared + by a boolean predicate, never by snapshot, and never by reading + ``.access_key`` — which performs network I/O on a refreshable credential). + + Provenance *labels* are the wrong comparator in both directions: the v0 + side labels env credentials "explicit static (AKIA…)" while the v1 side + labels the same keys "chain (env credentials present)" (a false alarm for + the most common configuration), and two different explicit keys both + truncate to "AKIA…" (a missed alarm for the very case step 0 exists to + catch — v0 letting env silently override explicit credentials). + """ + from s3dol.connection import ProfileCredentials, StaticCredentials + + if spec.anon: + return ("anonymous",) + credentials = spec.credentials + if isinstance(credentials, StaticCredentials): + return ("keys", credentials.access_key) # full id: compared, not printed + if isinstance(credentials, ProfileCredentials): + return ("profile", credentials.profile) + if credentials is not None: + return ("callable", credentials.provenance()) + if spec.profile: + return ("profile", spec.profile) + # The chain: its top rung is env credentials, which is what a v0-mangled + # side would have materialised as explicit keys — so identical inputs + # compare EQUAL here, which is the point. + env_key = environ.get("AWS_ACCESS_KEY_ID") + if env_key: + return ("keys", env_key) + return ("chain", environ.get("AWS_PROFILE") or "default") + + def _redact_url(url: Optional[str]) -> Optional[str]: """Redact query secrets and any userinfo from a URL for display.""" if url is None: @@ -186,21 +261,52 @@ def _divergence_rows(args: Mapping, environ: Mapping, aws_config) -> list[str]: f" {axis}: v0 -> {_redact_url(v0_value)!r}, " f"v1 -> {_redact_url(v1_value)!r} ** CHANGES ON UPGRADE **" ) - if v0_res.credential_provenance != v1_res.credential_provenance: + v0_identity = _credential_identity(_spec_from_v0_args(effective), environ) + v1_identity = _credential_identity(_spec_from_v0_args(args), environ) + if v0_identity != v1_identity: + moved = True + rows.append( + f" credential identity: v0 -> {_identity_label(v0_identity)}, " + f"v1 -> {_identity_label(v1_identity)} ** CHANGES ON UPGRADE **" + ) + # v1 introduces changes v0 never made, and they cannot show up above: + # BOTH sides run through v1's resolver, so pass-2 detection cancels out. + # Report them explicitly, or the announcement instrument cannot see the + # very path->virtual flip ADR-0012 D4 promises to announce. + if v1_res.preset_source == "detected": moved = True rows.append( - f" credentials: v0 -> {v0_res.credential_provenance}, " - f"v1 -> {v1_res.credential_provenance} ** CHANGES ON UPGRADE **" + f" provider detected from the endpoint: " + f"{v1_res.preset.name!r} — v1 now sets " + f"addressing_style={v1_res.addressing_style.value!r}" + + ( + f", checksum={v1_res.checksum.value!r}" + if v1_res.checksum.passed + else "" + ) + + f", region={v1_res.region_name.value!r}. v0 passed none of " + f"these (botocore defaults applied). ** NEW IN v1 **" ) if not moved: rows.append(" no divergence: v1 resolves this call exactly as v0 does.") return rows +def _identity_label(identity: tuple) -> str: + """A printable form of a credential identity — key ids truncated.""" + kind, *rest = identity + if kind == "keys": + return f"access key {str(rest[0])[:4]}…" + if kind == "anonymous": + return "anonymous" + return f"{kind}:{rest[0]}" + + def diagnose( connection: Optional[S3Connection] = None, *, environ: Optional[Mapping[str, str]] = None, + aws_config: Union[Mapping, None] = None, file=None, **kwargs, ) -> str: @@ -246,12 +352,18 @@ def emit(line=""): import os environ = dict(os.environ) - aws_config: object - try: - aws_config = load_aws_config() - except Exception as error: - aws_config = {} - emit(f"note: aws config unreadable ({type(error).__name__}: {error})") + if aws_config is None: + # The one impure read (the shared AWS config file), matching what + # a real construction resolves against. Injectable — without that, + # a "tier 1" test silently consults the developer's own + # ~/.aws/config and goes red on a machine whose profile sets an + # endpoint_url (ADR-0012 D2 made resolve() injectable exactly so + # this channel is controllable). + try: + aws_config = load_aws_config() + except Exception as error: + aws_config = {} + emit(f"note: aws config unreadable ({type(error).__name__}: {error})") v0_args = {k: v for k, v in kwargs.items() if k in _V0_ARG_NAMES} v1_kwargs = {k: v for k, v in kwargs.items() if k not in _V0_ARG_NAMES} @@ -261,7 +373,14 @@ def emit(line=""): emit() emit("arguments:") for key in sorted(kwargs): - shown = "" if key in _SECRET_ARG_NAMES else repr(kwargs[key]) + if key in _REDACTED_ARG_NAMES: + shown = repr(_redact_url(kwargs[key])) + elif key in _ECHOABLE_ARG_NAMES: + shown = repr(kwargs[key]) + elif key == "aws_access_key_id" and kwargs[key]: + shown = f"{str(kwargs[key])[:4]!r}… (prefix only)" + else: + shown = "" emit(f" {key} = {shown}") emit() diff --git a/s3dol/base.py b/s3dol/base.py index e8eda38..a818caa 100644 --- a/s3dol/base.py +++ b/s3dol/base.py @@ -1,150 +1,484 @@ -"""S3Dol with a dict-like interface for AWS S3 buckets. -Supports relative paths to access files under a prefix ending with slash. -Hints added to help setup credentials. - -bucket = S3Dol()[profile][bucket_name] -bucket['level1/level2/test-key'] == bucket['level1/']['level2/']['test-key'] +"""Layer B — close-to-metal stores over one bucket or one endpoint. + +Design record: `misc/docs/architecture.md` (the contract tables) on +ADR-0001 (layering; the prefix lives HERE, in the leaf), ADR-0004 (one error +seam), ADR-0005 (write domain / strategies), ADR-0006 (prefix arithmetic and +key validity), ADR-0008 (`__len__` absent by design), ADR-0010 (bucket policy), +ADR-0011 (**no key-taking public methods** — one guarded exception, `url_for`), +ADR-0012 (connection). + +The class triads mirror ``dol.filesys`` and ``azuredol``:: + + BucketCollection -> BucketReader -> BucketStore (keys: object keys) + EndpointCollection -> EndpointReader -> EndpointStore (keys: bucket names) + ObjectHandle (not a Mapping) + +Reader-only classes are *real classes* (a type checker catches +``reader[k] = v``; a read-scoped credential never even attempts a write) — +``dol.mk_read_only`` is non-functional on a dol store (verified, ADR-0001). + +**The rule for what may join a class here: nothing that takes a key** +(ADR-0011 D1). `dol` wrappers hand any non-dunder method the outer, unmapped +key, and the standard escape (``wrapped_self``) is itself silently wrong when +nothing references the wrapper. Keyed capabilities are sibling stores +(``s3dol.handles(store)[k]`` — `capabilities.py`); per-object richness lives on +:class:`ObjectHandle` (key bound at construction); everything else is a free +function taking the store first. The single exception is +:meth:`BucketReader.url_for`, kept for ``dol.SupportsUrlFor``, guarded to be +correct-or-loud (D3b). A reflective conformance test enforces the rule (D5). """ +from __future__ import annotations + from dataclasses import dataclass -import functools -import os -from typing import Union -from collections.abc import Iterable, Mapping +from datetime import datetime +from typing import Any, Iterator, Mapping, Optional, Union + +from dol.base import Collection as DolCollection, KvPersister, KvReader + +from s3dol.connection import S3Connection +from s3dol.errors import ( + BucketNotEmpty, + BucketNotFound, + ConfigurationError, + CredentialsError, + KeyNotValid, + NotSupported, + ObjectArchived, + S3Error, + classify_client_error, + code_of, + status_of, + translate_s3_errors, + translating_s3_errors, +) +from s3dol.reads import DEFAULT_READS, ReadStrategy +from s3dol.writes import DEFAULT_WRITES, WriteStrategy + +#: S3's real limit is 1024 **UTF-8 bytes** (enforced client-side so moto and +#: AWS agree — moto accepts longer keys; ADR-0006 §4). +MAX_KEY_BYTES = 1024 + +#: SigV4's presigned-URL lifetime cap (botocore does not enforce it). +MAX_PRESIGN_EXPIRES = 604_800 + + +def _validate_key(k, *, delimiter: str = "/") -> str: + """Normalise key-validity failures to :class:`KeyNotValid` **before the + wire** (ADR-0006 §4) — otherwise botocore types escape ``__getitem__`` + (``''`` -> ``ParamValidationError``; a lone surrogate -> + ``UnicodeEncodeError``; a 1025-byte key -> fine on moto, error on AWS).""" + if not isinstance(k, str): + raise KeyNotValid( + f"S3 keys are str; got {type(k).__name__}. " + f"(If you need structured keys, put a dol key codec on top.)" + ) + if k == "": + raise KeyNotValid("The empty string is not a valid S3 key.") + try: + encoded = k.encode("utf-8") + except UnicodeEncodeError as error: + raise KeyNotValid( + f"Key contains code points that cannot encode to UTF-8 " + f"(lone surrogate?): {k!r}" + ) from error + if len(encoded) > MAX_KEY_BYTES: + raise KeyNotValid( + f"Key is {len(encoded)} UTF-8 bytes; S3's limit is {MAX_KEY_BYTES}. " + f"(Checked client-side so moto and AWS agree.)" + ) + return k -from botocore.exceptions import ClientError -import boto3 -import dol -from s3dol.utility import KeyNotValidError, Resp, S3DolException, S3KeyError +@dataclass(frozen=True) +class ObjectInfo: + """Object metadata — from ``HeadObject`` or from a LIST row (which already + carries size/mtime/etag/storage class that v0 threw away; ADR-0009).""" + key: str + size: Optional[int] = None + last_modified: Optional[datetime] = None + etag: Optional[str] = None + content_type: Optional[str] = None + storage_class: Optional[str] = None + restore_status: Optional[str] = None + version_id: Optional[str] = None -noCredentialsFound = S3DolException( - "No AWS credentials found. Configure your AWS credentials as environment variables or in ~/.aws/credentials. " - "See https://github.com/i2mint/s3dol/#set-up-credentials for more information." -) +def _normalize_prefix(prefix: Optional[str], delimiter: str = "/") -> str: + """The mandatory normalisation, first (ADR-0006 §1): a prefix that does + not terminate in the delimiter is normalised, never accepted as-is — + ``prefix='logs'`` would otherwise expose ``logs2/…`` as readable *and + writable* keys one character away. -def get_aws_credentials(): - aws_access_key = os.environ.get("AWS_ACCESS_KEY_ID") - aws_secret_key = os.environ.get("AWS_SECRET_ACCESS_KEY") - aws_region = os.environ.get("AWS_DEFAULT_REGION") - aws_session_token = os.environ.get("AWS_SESSION_TOKEN") + >>> _normalize_prefix('logs'), _normalize_prefix('/logs/'), _normalize_prefix('') + ('logs/', 'logs/', '') + """ + if not prefix: + return "" + return f"{prefix.strip(delimiter)}{delimiter}" - return { - "aws_access_key_id": aws_access_key, - "aws_secret_access_key": aws_secret_key, - "region_name": aws_region, - "aws_session_token": aws_session_token, - } +class _WrapProbe: + """Descriptor recording dol's wrap-time ``hasattr(store, 'KeysView')`` + probe on the instance, then reporting the attribute absent. Class-level + access (``getattr(cls, 'KeysView')``) records nothing.""" -def list_profile_names(): - """ - Return a list of available AWS profiles - """ - session = boto3.session.Session() - available_profiles = session.available_profiles - # Check environment variables for default profile - if os.environ.get("AWS_ACCESS_KEY_ID") and os.environ.get("AWS_SECRET_ACCESS_KEY"): - available_profiles.append("environment variables") + def __get__(self, instance, owner=None): + if instance is not None: + instance.__dict__["_ever_wrapped"] = True + raise AttributeError("KeysView") - return available_profiles +class _BucketBase: + """Shared base of the bucket-level classes: owns the normalised prefix, + the connection, the key arithmetic, and the error context — so those exist + exactly once (ADR-0011 D2). Plain class; the dol ABCs are mixed in by the + concrete triad classes.""" -def get_client(profile_name=None, endpoint_url=None, **session_kwargs): - """ - Return a boto3 client for the specified profile - """ - if profile_name is None: - return _find_default_credentials(endpoint_url=endpoint_url, **session_kwargs) - - if profile_name == "environment variables": - aws_credentials = get_aws_credentials() - if ( - not aws_credentials["aws_access_key_id"] - or not aws_credentials["aws_secret_access_key"] - ): - raise S3DolException( - "Missing AWS credentials in environment variables: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY" + def __init__( + self, + bucket: str, + *, + connection: Union[S3Connection, None] = None, + prefix: str = "", + delimiter: str = "/", + reads: Optional[ReadStrategy] = None, + writes: Optional[WriteStrategy] = None, + on_missing_bucket: str = "assume", + strict_delete: bool = False, + ): + if not isinstance(bucket, str) or not bucket: + raise ConfigurationError(f"bucket must be a non-empty str; got {bucket!r}") + if on_missing_bucket not in ("assume", "create", "raise"): + raise ConfigurationError( + f"on_missing_bucket must be 'assume', 'create' or 'raise'; got " + f"{on_missing_bucket!r} (ADR-0010 §1)." ) - _skw = {**session_kwargs, **aws_credentials} - session = boto3.Session(**_skw) - else: - # Get credentials from profile - session = boto3.Session(profile_name=profile_name, **session_kwargs) - client = session.client("s3", endpoint_url=endpoint_url) - return client - - -def _find_default_credentials(endpoint_url=None, **session_kwargs): - try: - return get_client("environment variables", **session_kwargs) - except S3DolException: - pass - session = boto3.Session(**session_kwargs) - if session.get_credentials() is None: - raise noCredentialsFound - client = session.client("s3", endpoint_url=endpoint_url) - return client - - -@dataclass -class BaseS3BucketReader(dol.base.KvReader): - """Dict-like interface for AWS S3 buckets""" - - client: boto3.client - bucket_name: str - prefix: str = None - delimiter: str = "/" - - def __post_init__(self): - self.prefix = ( - f"{self.prefix.strip(self.delimiter)}{self.delimiter}" - if self.prefix - else "" + self.bucket = bucket + self.connection = connection if connection is not None else S3Connection() + self.delimiter = delimiter + self.prefix = _normalize_prefix(prefix, delimiter) + self.reads = reads if reads is not None else DEFAULT_READS + self.writes = writes if writes is not None else DEFAULT_WRITES + self.on_missing_bucket = on_missing_bucket + self.strict_delete = strict_delete + self._bucket_reachable = False + if on_missing_bucket == "raise": + # The explicit opt-in that performs ONE HeadBucket at construction + # (documented I/O — the only construction-time request; ADR-0010). + self._probe_bucket_or_raise() + + # -- key arithmetic (the leaf owns the prefix — ADR-0001/0006) --------- # + + def _id_of_key(self, k: str) -> str: + return f"{self.prefix}{_validate_key(k, delimiter=self.delimiter)}" + + def _key_of_id(self, _id: str) -> str: + # RAISE rather than slice a non-matching id: the server-side Prefix + # makes out-of-scope ids unreachable on the common path; this is the + # belt to that suspenders, because a provider that ignores Prefix must + # not silently produce corrupt keys (ADR-0006 §1). + if self.prefix and not _id.startswith(self.prefix): + raise KeyNotValid( + f"Backend returned id {_id!r} outside this store's prefix " + f"{self.prefix!r} — refusing to relativise it (the provider " + f"may be ignoring ListObjectsV2's Prefix)." + ) + return _id[len(self.prefix) :] + + # -- plumbing ---------------------------------------------------------- # + + @property + def client(self): + return self.connection.client + + def _error_context(self) -> dict: + row = self._resolution().preset + return dict( + bucket=self.bucket, + endpoint=self._endpoint_label(), + overrides=row.error_overrides if row is not None else (), + deny_means_absent=self.connection.deny_means_absent, + head_ambiguity_resolver=self._head_ambiguity_resolver, ) - def object_list_pages(self) -> Iterable[dict]: - if self._bucket_exists(): - yield from self.client.get_paginator("list_objects").paginate( - Bucket=self.bucket_name, Prefix=self.prefix + def _resolution(self): + cached = self.__dict__.get("_resolution_cache") + if cached is None: + cached = self.connection.resolution() + self.__dict__["_resolution_cache"] = cached + return cached + + def _endpoint_label(self) -> Optional[str]: + built = self.connection.__dict__.get("_client") + if built is not None: + return getattr(built.meta, "endpoint_url", None) + return self._resolution().endpoint_url.value + + def _mark_bucket_reachable(self): + self._bucket_reachable = True + + def _head_ambiguity_resolver(self) -> bool: + """Resolve the body-less ``HeadObject`` 404 (ADR-0004 §2): ``True`` + iff the bucket is proven reachable (so the miss is the object's). + Costs one ``HeadBucket``, once — cached. An auth failure here raises + ``AccessDenied`` rather than masquerading as a missing bucket.""" + if self._bucket_reachable: + return True + from botocore.exceptions import ClientError + + try: + self._op_head_bucket() + except ClientError as error: + classified = classify_client_error( + "HeadBucket", + code_of(error), + status_of(error), + overrides=self._error_context()["overrides"], + ) + if classified.kind == "bucket_absent": + return False + raise # auth/transport: never disguise as absence (ADR-0004 §1) + self._mark_bucket_reachable() + return True + + def _probe_bucket_or_raise(self): + if not self._head_ambiguity_resolver(): + raise BucketNotFound( + f"Bucket {self.bucket!r} not found (on_missing_bucket='raise'). " + f"Note only ('404', 404) means missing; a 403 would have raised " + f"AccessDenied naming s3:ListBucket." ) - def __iter__(self) -> Iterable[str]: - for resp in self.object_list_pages(): - Resp.ascertain_200_status_code(resp) - yield from filter( - lambda k: not k.endswith(self.delimiter), - map(Resp.key, Resp.contents(resp)), + def _refuse_anonymous_write(self, operation: str): + # botocore offers nothing here — moto even accepts an unsigned PUT — + # so the refusal is local (ADR-0012 D5). + if self.connection.anon: + raise CredentialsError( + f"{operation} refused: this store is anonymous (anon=True). " + f"Anonymous credentials cannot sign writes." ) - def __getitem__(self, k: str): - try: - return self.client.get_object(Bucket=self.bucket_name, Key=k)["Body"].read() - except self.client.exceptions.NoSuchKey as ex: - raise KeyError(f"Key {k} does not exist") from ex + def _sibling_kwargs(self) -> dict: + """Constructor kwargs for a sibling capability store over the same + key space (ADR-0011 D2) — the key arithmetic exists once, here.""" + return dict( + bucket=self.bucket, + connection=self.connection, + prefix=self.prefix, + delimiter=self.delimiter, + reads=self.reads, + writes=self.writes, + on_missing_bucket="assume", + strict_delete=self.strict_delete, + ) + + def _make_sibling(self, kind: str, **options): + """Build the sibling capability store of ``kind`` over the same key + space. Overridable so `s3dol.testing`'s fake hands out fake siblings.""" + from s3dol import capabilities as _capabilities + + sibling_class = { + "handles": _capabilities.BucketHandles, + "urls": _capabilities.BucketUrls, + "info": _capabilities.BucketInfo, + }[kind] + return sibling_class(**{**self._sibling_kwargs(), **options}) + + # -- the backend protocol (ADR-0002's module-internal seam) ------------ # + # Layer B talks to the backend ONLY through these. Implemented once by + # botocore here; s3dol.testing's in-memory fake overrides exactly these + # (raising synthesized botocore ClientErrors, so the error taxonomy and + # every guard above this line is exercised identically). Deliberately NOT + # a public extension point in v1: a protocol with one real implementer + # would be an abstraction shaped like its implementation. + + def _op_list_pages( + self, *, delimiter: Optional[str] = None, prefix: Optional[str] = None + ): + row = self._resolution().preset + use_v2 = row.capabilities.list_objects_v2 if row is not None else True + paginator = self.client.get_paginator( + "list_objects_v2" if use_v2 else "list_objects" + ) + kwargs: dict = dict( + Bucket=self.bucket, Prefix=self.prefix if prefix is None else prefix + ) + if delimiter: + kwargs["Delimiter"] = delimiter + return paginator.paginate(**kwargs) + + def _op_head_object(self, _id: str) -> dict: + return self.client.head_object(Bucket=self.bucket, Key=_id) + def _op_head_bucket(self) -> None: + self.client.head_bucket(Bucket=self.bucket) + + def _op_read(self, _id: str): + return self.reads(self.client, self.bucket, _id) + + def _op_write(self, _id: str, value) -> None: + self.writes(self.client, self.bucket, _id, value) + + def _op_delete(self, _id: str) -> None: + self.client.delete_object(Bucket=self.bucket, Key=_id) + + def _op_bulk_delete(self, _ids: list) -> tuple: + """One DeleteObjects call (≤1000 ids). Returns ``(deleted_ids, + errors)`` where errors is ``[(id, code, message), ...]`` — parsed out + of the HTTP **200** response (ADR-0010 §2).""" + response = self.client.delete_objects( + Bucket=self.bucket, + Delete={"Objects": [{"Key": k} for k in _ids], "Quiet": False}, + ) + deleted = [d["Key"] for d in response.get("Deleted", ())] + errors = [ + (e.get("Key"), e.get("Code"), e.get("Message")) + for e in response.get("Errors", ()) + ] + return deleted, errors + + def _op_create_bucket(self) -> None: + _create_bucket( + self.client, self.bucket, region=self._resolution().region_name.value + ) + + def _op_presign( + self, _id: str, *, expires_in: int, client_method: str, **params + ) -> Optional[str]: + return _presign( + self.connection, + bucket=self.bucket, + key=_id, + expires_in=expires_in, + client_method=client_method, + **params, + ) + + def _with(self, **overrides) -> "_BucketBase": + """An explicit-fields sub-store builder (never + ``type(self)(**self.__dict__)`` — ADR-0001). Overridden prefixes are + re-normalised by ``__init__``.""" + kwargs = self._sibling_kwargs() + kwargs["on_missing_bucket"] = self.on_missing_bucket + kwargs.update(overrides) + return type(self)(kwargs.pop("bucket"), **kwargs) + + #: The wrap probe (ADR-0011 D1a §3, adapted). dol.base.Store.__init__ + #: probes its wrapped store with ``hasattr(store, 'KeysView')``. On a + #: plain leaf that reaches ``__getattr__``; on a KvReader-derived leaf it + #: would be satisfied by MappingViewMixin's class attribute and the probe + #: would be undetectable — so we shadow it with a descriptor that records + #: the touch and raises AttributeError. keys()/values()/items() below + #: bypass it. (Documented false positive: a once-wrapped leaf later used + #: bare will refuse url_for — accepted, D1a.) + KeysView = _WrapProbe() + + def keys(self): + from dol.base import BaseKeysView + + return BaseKeysView(self) + + def values(self): + from dol.base import BaseValuesView + + return BaseValuesView(self) + + def items(self): + from dol.base import BaseItemsView + + return BaseItemsView(self) + + def __repr__(self): + # bucket, prefix, endpoint host, mode. No secrets. (architecture.md) + bits = [repr(self.bucket)] + if self.prefix: + bits.append(f"prefix={self.prefix!r}") + endpoint = self._endpoint_label() + if endpoint: + from urllib.parse import urlsplit + + bits.append(f"endpoint={urlsplit(endpoint).hostname!r}") + if self.connection.anon: + bits.append("anon=True") + return f"{type(self).__name__}({', '.join(bits)})" + + # -- pickling: every s3dol store pickles (ADR-0008 §6) ----------------- # + + def __getstate__(self): + state = dict(self.__dict__) + state.pop("_resolution_cache", None) + return state + + def __setstate__(self, state): + self.__dict__.update(state) + + +class BucketCollection(_BucketBase, DolCollection): + """Keys of one bucket (under the prefix). ``Collection`` only — + iteration and membership, no reads.""" + + def __iter__(self) -> Iterator[str]: + # Lazy, paginated, prefix pushed down to the server. RAISES if the + # bucket is missing or unlistable — never yields empty (goal 4). + with translating_s3_errors(self, operation="ListObjectsV2"): + for page in self._op_list_pages(): + self._mark_bucket_reachable() + for entry in page.get("Contents", ()): + _id = entry["Key"] + if _id == self.prefix: + # The exact-prefix marker object would relativise to + # the forbidden key '' — the scoped view filters it + # out; it stays addressable by absolute key on an + # unscoped store (ADR-0006 §4). + continue + yield self._key_of_id(_id) + + @translate_s3_errors(operation="HeadObject", key_arg="k") def __contains__(self, k) -> bool: + from botocore.exceptions import ClientError + try: - self.client.head_object(Bucket=self.bucket_name, Key=k) - return True # if all went well - except KeyNotValidError as e: - raise - except ClientError as e: - if e.response["Error"]["Code"] == "404": - # The object does not exist. + self._op_head_object(self._id_of_key(k)) + except ClientError as error: + classified = classify_client_error( + "HeadObject", + code_of(error), + status_of(error), + overrides=self._error_context()["overrides"], + ) + if classified.kind == "ambiguous_head": + if self._head_ambiguity_resolver(): + return False # bucket proven reachable: the object is absent + raise # bucket absent: let the seam raise BucketNotFound + if classified.kind == "object_absent": return False - else: - # Something else has gone wrong. - raise + if classified.kind == "access_denied" and self.connection.deny_means_absent: + return False + raise # auth/config/transport: never a silent False (ADR-0004) + self._mark_bucket_reachable() + return True + + def __len__(self): + # Explicit raise: dol's Collection.__len__ counts by iterating, so + # len(store) would be a full paginated listing and list(store) would + # cost TWO listings via the length hint (ADR-0008 §cost model). + raise TypeError( + f"{type(self).__name__} does not implement __len__ (a length is a " + f"full paginated listing, and list() would take it as a hint and " + f"pay twice). Use sum(1 for _ in store) if you accept the scan." + ) - def _bucket_exists(self) -> bool: - try: - self.client.head_bucket(Bucket=self.bucket_name) - return True - except ClientError: - return False + +class BucketReader(BucketCollection, KvReader): + """+ ``__getitem__`` -> bytes (or the read strategy's value type).""" + + @translate_s3_errors(operation="GetObject", key_arg="k") + def __getitem__(self, k): + value = self._op_read(self._id_of_key(k)) + self._mark_bucket_reachable() + return value def url_for( self, @@ -153,180 +487,496 @@ def url_for( expires_in: int = 3600, client_method: str = "get_object", **params, - ) -> str: - """Presigned URL for key ``k`` (default: a GET valid for ``expires_in`` - seconds). - - Lets a caller 302-redirect to the object store so it serves the bytes - (and HTTP ``Range``) directly, instead of proxy-streaming every byte - through the app. Works for any S3-compatible endpoint (AWS, R2, MinIO, - Supabase) — a pure client call, no object fetch. Extra ``params`` merge - into the presign ``Params`` (e.g. ``ResponseContentType='video/mp4'``). - See issue #7. + ) -> Optional[str]: + """Presigned URL for ``k`` — **the one keyed method** (ADR-0011 D3b), + kept solely because ``dol.SupportsUrlFor`` requires a *method* and + ``dol.content_url`` reaches it by ``getattr``. The canonical form is + ``s3dol.urls(store)[k]``. + + Guarded to be correct-or-loud, never silently wrong: + + - **unwrapped store**: ``k`` is a key in this store's own key space, + so the URL addresses ``self._id_of_key(k)``. Correct, and this is + also the contract ``dol.content_url`` calls under — it resolves the + key through the *outer* layers and stops at the layer owning + ``url_for``, "because that layer applies its own" + (``dol/content.py``); + - **this leaf has ever been wrapped**: **raises**. Two call paths then + reach this one signature with *different key domains* and nothing + distinguishes them: ``dol.content_url`` hands over a key already + mapped through the wrapper layers, while a direct + ``wrapped.url_for(k)`` (delegation) hands over the raw outer key. + Applying ``_id_of_key`` is right for the first and wrong for the + second; re-resolving through the chain is right for the second and + double-transforms the first. Since a wrong presigned URL addresses a + *different real object*, the only honest answer is refusal, naming + the sibling store — which is correct by construction because it goes + through ``__getitem__`` (ADR-0011 D2/D3b, and see the note below); + - **anonymous connection**: returns ``None`` (``SupportsUrlFor`` is + ``Optional[str]``; raising would turn a public-bucket streaming + fallback into a 500 — ADR-0012 D5). The check reads the built + client's signature version, never ``spec.anon``. + + .. note:: This refusal is a *finding*, not a limitation we chose. + ADR-0011 D3b expected a guarded method to serve both paths; it + cannot, on a leaf that owns its prefix. Reported upstream on + dol#86/#83 as evidence that a keyed capability *method* is + unfixable in the general case, and that the sibling-store form is + the only one correct by construction. """ - return self.client.generate_presigned_url( - client_method, - Params={"Bucket": self.bucket_name, "Key": k, **params}, - ExpiresIn=expires_in, + if self.__dict__.get("_ever_wrapped"): + raise NotSupported( + "url_for refused: this store has been wrapped by a dol " + "key/value wrapper, and the two call paths that reach this " + "method (dol.content_url, which pre-resolves the key, and a " + "direct delegated call, which does not) are indistinguishable " + "here — one of them would get a URL for the WRONG object. Use " + "the sibling store, which is correct by construction: " + "s3dol.urls(store)[key] (derive it from the unwrapped store " + "and apply the same wrapper to both)." + ) + return self._op_presign( + self._id_of_key(k), + expires_in=expires_in, + client_method=client_method, + **params, ) -class BaseS3BucketDol(BaseS3BucketReader, dol.base.KvPersister): - skip_bucket_exists_check: bool = False +class BucketStore(BucketReader, KvPersister): + """+ ``__setitem__`` / ``__delitem__``. The user-facing read-write store.""" + @translate_s3_errors(operation="PutObject", key_arg="k") def __setitem__(self, k, v): - if not self.skip_bucket_exists_check and not self._bucket_exists(): - # create_bucket will not be silent if permission to create bucket is denied - self.client.create_bucket(Bucket=self.bucket_name) - self.client.put_object(Bucket=self.bucket_name, Key=k, Body=v) + self._refuse_anonymous_write("__setitem__") + _id = self._id_of_key(k) + from botocore.exceptions import ClientError - def __delitem__(self, k): - self.client.delete_object(Bucket=self.bucket_name, Key=k) + try: + self._op_write(_id, v) + except ClientError as error: + # 'create' recovers, never probes: attempt, and on NoSuchBucket + # create-then-retry once (ADR-0010 §1). + if self.on_missing_bucket == "create" and code_of(error) == "NoSuchBucket": + self._op_create_bucket() + self._mark_bucket_reachable() + self._op_write(_id, v) + else: + raise + self._mark_bucket_reachable() + @translate_s3_errors(operation="DeleteObject", key_arg="k") + def __delitem__(self, k): + self._refuse_anonymous_write("__delitem__") + _id = self._id_of_key(k) + if self.strict_delete: + # The opt-in probe (one extra request, TOCTOU-racy — documented). + if k not in self: + raise ObjectNotFoundForDelete(k) + self._op_delete(_id) + # Idempotent by default: S3's DeleteObject returns 204 for an absent + # key; raising KeyError would require a HeadObject probe on EVERY + # delete (architecture.md contract table). + + # -- ObjectArchived must not be swallowed by the defaulted accessors --- # + # MutableMapping.setdefault is `try: self[k] except KeyError: self[k] = + # default` — a Glacier KeyError would let setdefault OVERWRITE the + # archived object with the default, silently (measured; ADR-0004 §4). + + def setdefault(self, k, default=None): + try: + return self[k] + except ObjectArchived: + raise + except KeyError: + self[k] = default + return default -# TODO: Use wrap_kvs and filt_iter from dol to handle key and id conversions -@dataclass -class S3BucketReader(BaseS3BucketReader): - """ - Enables path delimitations to access objects under a prefix ending with slash such as 'level1/level2/'. + def pop(self, k, *default): + try: + value = self[k] + except ObjectArchived: + raise + except KeyError: + if default: + return default[0] + raise + del self[k] + return value - s3_bucket['level1/']['level2/']['test-key'] == s3_bucket['level1/level2/test-key'] - """ + def update(self, other=(), /, **kwargs) -> None: + # Explicit override for the type: MutableMapping's value type is + # invariant, so `dst.update(src)` with refs (Filepath/Chunks) fails a + # type checker without this widened signature (ADR-0005 §3). + super().update(other, **kwargs) - skip_bucket_exists_check: bool = False - def _id_of_key(self, k): - return f"{self.prefix}{k}" +class ObjectNotFoundForDelete(S3Error, KeyError): + """Raised by ``strict_delete=True`` deletes of absent keys.""" - def _key_of_id(self, id): - return id[len(self.prefix) :] + def __str__(self): + return f"Key {self.args[0]!r} was absent (strict_delete=True)." - def __iter__(self): - return ( - self._key_of_id(_id) - for _id in super().__iter__() - if _id.startswith(self.prefix) - ) - def __getitem__(self, k: str): - _id = self._id_of_key(k) - if _id.endswith(self.delimiter): - _kw = {**self.__dict__, "prefix": _id} - return type(self)(**_kw) - return super().__getitem__(_id) +def _create_bucket(client, bucket: str, *, region: Optional[str]): + """CreateBucket with the region rules that bite (ADR-0010 §1): + ``CreateBucketConfiguration`` everywhere except us-east-1 (where passing + it is an error), and ``BucketAlreadyOwnedByYou`` tolerated (re-create is + only idempotent in us-east-1).""" + from botocore.exceptions import ClientError - def __contains__(self, k) -> bool: - _id = self._id_of_key(k) - return super().__contains__(_id) + kwargs: dict = {"Bucket": bucket} + if region and region not in ("us-east-1", "auto"): + kwargs["CreateBucketConfiguration"] = {"LocationConstraint": region} + try: + client.create_bucket(**kwargs) + except ClientError as error: + if code_of(error) not in ("BucketAlreadyOwnedByYou", "BucketAlreadyExists"): + raise - def url_for(self, k: str, **kwargs) -> str: - """Prefix-aware presigned URL: maps ``k`` through ``_id_of_key`` (as - ``__getitem__`` does) so the URL points at the same object.""" - return super().url_for(self._id_of_key(k), **kwargs) +def clamp_expires_in(expires_in: int) -> int: + """Clamp to SigV4's 604800 s cap, warning. botocore does not enforce it — + it just signs a URL that fails after 7 days.""" + import warnings -class S3BucketDol(S3BucketReader, BaseS3BucketDol): - def __setitem__(self, k, v): - _id = self._id_of_key(k) + if expires_in > MAX_PRESIGN_EXPIRES: + warnings.warn( + f"expires_in={expires_in} exceeds SigV4's hard cap of " + f"{MAX_PRESIGN_EXPIRES}s (botocore does not enforce it; the URL " + f"would simply fail after 7 days). Clamping.", + stacklevel=3, + ) + return MAX_PRESIGN_EXPIRES + return expires_in + + +def _presign( + connection: S3Connection, + *, + bucket: str, + key: str, + expires_in: int, + client_method: str = "get_object", + **params, +) -> Optional[str]: + """Shared presigning core: SigV4 (the connection guarantees it), the + anonymous ``None``, the 604800 s cap, and the credential-expiry clamp.""" + import warnings + + from botocore import UNSIGNED + + client = connection.presign_client + if client.meta.config.signature_version is UNSIGNED: + return None # no capability, not an error (ADR-0012 D5) + expires_in = clamp_expires_in(expires_in) + credentials = client._request_signer._credentials + expiry = getattr(credentials, "_expiry_time", None) + if expiry is not None: + from datetime import timezone + + remaining = (expiry - datetime.now(timezone.utc)).total_seconds() + if 0 < remaining < expires_in: + warnings.warn( + f"The signing credentials expire in ~{int(remaining)}s, " + f"before expires_in={expires_in}: a presigned URL cannot " + f"outlive the credential that signed it (STS/SSO/instance " + f"sessions). The URL will die with the session.", + stacklevel=3, + ) + return client.generate_presigned_url( + client_method, + Params={"Bucket": bucket, "Key": key, **params}, + ExpiresIn=expires_in, + ) - # Directly call BaseS3BucketDol.__setitem__ to ensure we respect skip_bucket_exists_check - if not self.skip_bucket_exists_check and not self._bucket_exists(): - self.client.create_bucket(Bucket=self.bucket_name) - self.client.put_object(Bucket=self.bucket_name, Key=_id, Body=v) - def __delitem__(self, k): - _id = self._id_of_key(k) - return super().__delitem__(_id) +# --------------------------------------------------------------------------- # +# ObjectHandle — the per-object escape hatch (NOT a Mapping) +# --------------------------------------------------------------------------- # -class S3ClientReader(dol.base.KvReader): - ignore_404_when_endpoint_has_substring = ".supabase." +class ObjectHandle: + """One object, key bound **at construction** — which is what makes it + immune to the delegation trap (a key codec over the store cannot corrupt a + key that was resolved before the handle existed; ADR-0011 D1, following + ``azuredol.BlobHandle``). This is where ranged reads, streaming, + download-to-disk, metadata and presigned URLs live.""" def __init__( - self, *, s3_bucket_dol=S3BucketDol, profile_name=None, **session_kwargs + self, bucket: str, key: str, *, connection: Optional[S3Connection] = None ): - self.client = get_client(profile_name=profile_name, **session_kwargs) - self.s3_bucket_dol = s3_bucket_dol + self.bucket = bucket + self.key = key # the ABSOLUTE (wire) key + self.connection = connection if connection is not None else S3Connection() + + @property + def client(self): + return self.connection.client + + def _error_context(self) -> dict: + return dict(bucket=self.bucket, endpoint=None) + + def __repr__(self): + return f"{type(self).__name__}({self.bucket!r}, {self.key!r})" + + def __eq__(self, other): + return isinstance(other, ObjectHandle) and ( + self.bucket, + self.key, + self.connection, + ) == (other.bucket, other.key, other.connection) + + def __hash__(self): + return hash((self.bucket, self.key, self.connection)) + + @translate_s3_errors(operation="GetObject") + def read( + self, *, offset: Optional[int] = None, length: Optional[int] = None + ) -> bytes: + """The object's bytes; a ranged read when ``offset``/``length`` given.""" + kwargs: dict = {"Bucket": self.bucket, "Key": self.key} + if offset is not None or length is not None: + start = offset or 0 + end = "" if length is None else start + length - 1 + kwargs["Range"] = f"bytes={start}-{end}" + return self.client.get_object(**kwargs)["Body"].read() + + @translate_s3_errors(operation="GetObject") + def stream(self, *, chunk_size: int = 8 * 2**20) -> Iterator[bytes]: + """Lazy chunk iterator over the body.""" + from functools import partial + + body = self.client.get_object(Bucket=self.bucket, Key=self.key)["Body"] + return iter(partial(body.read, chunk_size), b"") + + @translate_s3_errors(operation="GetObject") + def download_to(self, path) -> str: + """Download to disk (never fully in memory). Accepts a path or + :class:`~s3dol.values.Filepath`; returns the path written.""" + from s3dol.values import Filepath + + target = path.path if isinstance(path, Filepath) else str(path) + self.client.download_file(self.bucket, self.key, target) + return target + + @translate_s3_errors(operation="PutObject") + def write(self, value) -> None: + """Write the object (same value domain as ``store[k] = v``).""" + if self.connection.anon: + raise CredentialsError("Anonymous credentials cannot sign writes.") + DEFAULT_WRITES(self.client, self.bucket, self.key, value) + + @translate_s3_errors(operation="DeleteObject") + def delete(self) -> None: + if self.connection.anon: + raise CredentialsError("Anonymous credentials cannot sign writes.") + self.client.delete_object(Bucket=self.bucket, Key=self.key) + + def exists(self) -> bool: + from botocore.exceptions import ClientError + + try: + self.client.head_object(Bucket=self.bucket, Key=self.key) + except ClientError as error: + if classify_client_error( + "HeadObject", code_of(error), status_of(error) + ).kind in ("object_absent", "ambiguous_head"): + return False + raise + return True + + @translate_s3_errors(operation="HeadObject") + def info(self) -> ObjectInfo: + """One ``HeadObject`` -> :class:`ObjectInfo`.""" + head = self.client.head_object(Bucket=self.bucket, Key=self.key) + return ObjectInfo( + key=self.key, + size=head.get("ContentLength"), + last_modified=head.get("LastModified"), + etag=head.get("ETag"), + content_type=head.get("ContentType"), + storage_class=head.get("StorageClass") or "STANDARD", + restore_status=head.get("Restore"), + version_id=head.get("VersionId"), + ) + + def url( + self, + *, + expires_in: int = 3600, + client_method: str = "get_object", + **params, + ) -> Optional[str]: + """Presigned URL. Always SigV4; ``None`` when anonymous; capped at + SigV4's 604800 s; warns when the signing credential expires sooner + (the URL cannot outlive it).""" + return _presign( + self.connection, + bucket=self.bucket, + key=self.key, + expires_in=expires_in, + client_method=client_method, + **params, + ) + + @translate_s3_errors(operation="RestoreObject") + def restore(self, days: int = 1, *, tier: str = "Standard") -> None: + """Request restoration of an archived (Glacier) object.""" + self.client.restore_object( + Bucket=self.bucket, + Key=self.key, + RestoreRequest={"Days": days, "GlacierJobParameters": {"Tier": tier}}, + ) + + +# --------------------------------------------------------------------------- # +# Endpoint level (keys = bucket names) +# --------------------------------------------------------------------------- # - def __iter__(self): - return (b["Name"] for b in self.client.list_buckets().get("Buckets", [])) - def __contains__(self, k): +class _EndpointBase: + """Shared base of the endpoint-level classes.""" + + def __init__(self, connection: Optional[S3Connection] = None, **bucket_kwargs): + if connection is not None and not isinstance(connection, S3Connection): + raise ConfigurationError( + f"connection must be an S3Connection (got " + f"{type(connection).__name__}). Live boto3 objects are not " + f"accepted anywhere in v1 (ADR-0012 D1)." + ) + self.connection = connection if connection is not None else S3Connection() + #: kwargs forwarded to the bucket stores this endpoint hands out. + self.bucket_kwargs = bucket_kwargs + + @property + def client(self): + return self.connection.client + + def _error_context(self) -> dict: + row = self.connection.resolution().preset + return dict( + bucket=None, + endpoint=None, + overrides=row.error_overrides if row is not None else (), + deny_means_absent=self.connection.deny_means_absent, + ) + + def __repr__(self): + endpoint = self.connection.resolution().endpoint_url.value + inner = "" if not endpoint else f"endpoint={endpoint!r}" + return f"{type(self).__name__}({inner})" + + +class EndpointCollection(_EndpointBase, DolCollection): + """Bucket names at one endpoint.""" + + def __iter__(self) -> Iterator[str]: + with translating_s3_errors(self, operation="ListBuckets"): + for bucket in self.client.list_buckets().get("Buckets", ()): + yield bucket["Name"] + + @translate_s3_errors(operation="HeadBucket", key_arg="k") + def __contains__(self, k) -> bool: + if not isinstance(k, str) or not k: + return False + from botocore.exceptions import ClientError + try: self.client.head_bucket(Bucket=k) - return True - except Exception as e: - # Check for a 400 error code in the response attributes - if ( - hasattr(e, "response") - and e.response.get("Error", {}).get("Code") == "400" - ): - # Use the endpoint URL to decide if it's Supabase - endpoint = getattr(self.client.meta, "endpoint_url", "") - if "supabase" in endpoint: - return True - elif "404" in str(e): + except ClientError as error: + classified = classify_client_error( + "HeadBucket", + code_of(error), + status_of(error), + overrides=self._error_context()["overrides"], + ) + if classified.kind == "bucket_absent": return False - raise S3DolException(f"Error checking bucket existence: {e}") from e + raise # auth/config: never "the bucket doesn't exist" (ADR-0004) + return True + + def __len__(self) -> int: + # Bucket counts are small — an administrative resource (ADR-0008). + return sum(1 for _ in self) + + +class EndpointReader(EndpointCollection, KvReader): + """+ ``__getitem__`` -> :class:`BucketReader`.""" + + _bucket_class = BucketReader def __getitem__(self, k: str): - # if k not in self: - # raise S3KeyError(f'Bucket {k} does not exist') + if not isinstance(k, str) or not k: + raise KeyNotValid(f"Bucket names are non-empty str; got {k!r}") + return self._bucket_class(k, connection=self.connection, **self.bucket_kwargs) - return self.s3_bucket_dol(client=self.client, bucket_name=k) +class EndpointStore(EndpointReader, KvPersister): + """+ bucket creation/deletion. ``del endpoint[name]`` refuses a non-empty + bucket; the explicit cascading form is the free function + ``s3dol.delete_bucket(endpoint, name, force=True)`` (ADR-0010 §3 as + amended by ADR-0011 D4 — a public keyed destructive *method* is the shape + the family census found destroying wrong data).""" -class S3ClientDol(S3ClientReader, dol.base.KvPersister): - def __init__( - self, *, s3_bucket_dol=S3BucketDol, profile_name=None, **session_kwargs - ): - super().__init__( - s3_bucket_dol=s3_bucket_dol, profile_name=profile_name, **session_kwargs - ) + _bucket_class = BucketStore + @translate_s3_errors(operation="CreateBucket", key_arg="k") def __setitem__(self, k: str, v: Mapping): - """Populate a bucket with a mapping of keys to objects. Create bucket if it does not exist. - - :param k: bucket name - :type k: str - :param v: mapping of keys to objects to populate bucket with - :type v: Mapping - """ + """Create bucket ``k`` and populate it from mapping ``v`` (pass ``{}`` + to just create).""" + if self.connection.anon: + raise CredentialsError("Anonymous credentials cannot create buckets.") if not isinstance(v, Mapping): raise TypeError( - f"Value must be a mapping (dict-like) object. Got {type(v)}" + f"EndpointStore values are Mappings of objects to populate the " + f"bucket with (use {{}} to just create it); got {type(v).__name__}." ) - self.client.create_bucket(Bucket=k) + _create_bucket( + self.client, k, region=self.connection.resolution().region_name.value + ) bucket = self[k] - for bucket_item_key, bucket_item_value in v.items(): - bucket[bucket_item_key] = bucket_item_value + for key, value in v.items(): + bucket[key] = value - def __delitem__(self, k): - res = self.client.list_objects_v2(Bucket=k) - if "Contents" in res: - objects = [{"Key": obj["Key"]} for obj in res["Contents"]] - self.client.delete_objects(Bucket=k, Delete={"Objects": objects}) - self.client.delete_bucket(Bucket=k) + @translate_s3_errors(operation="DeleteBucket", key_arg="k") + def __delitem__(self, k: str): + if self.connection.anon: + raise CredentialsError("Anonymous credentials cannot delete buckets.") + from botocore.exceptions import ClientError + try: + self.client.delete_bucket(Bucket=k) + except ClientError as error: + if code_of(error) == "BucketNotEmpty": + raise BucketNotEmpty( + f"Bucket {k!r} is not empty. Deleting it would destroy " + f"its objects; that never happens implicitly. Use " + f"s3dol.delete_bucket(endpoint, {k!r}, force=True) for " + f"the explicit, paginated cascade." + ) from error + raise -class S3Dol(dol.base.KvReader): - """S3 profiles -> buckets -> keys -> objects""" - def __init__(self, s3_client_dol=S3ClientDol, s3_bucket_dol=S3BucketDol): - if len(list_profile_names()) == 0: - raise noCredentialsFound - self.s3_client_dol = s3_client_dol - self.s3_bucket_dol = s3_bucket_dol +class S3Profiles(KvReader): + """AWS **profile** names -> endpoint stores (the v0 ``S3Dol``, renamed — + its keys are verifiably profile names, never endpoints; ADR-0007 §1). - def __iter__(self): - return iter(list_profile_names()) + >>> # S3Profiles()['prod'] -> EndpointStore over the 'prod' profile + """ - def __getitem__(self, k: str | dict): - if isinstance(k, str): - return self.s3_client_dol(profile_name=k, s3_bucket_dol=self.s3_bucket_dol) - return self.s3_client_dol(s3_bucket_dol=self.s3_bucket_dol, **k) + def __init__(self, *, bucket_kwargs: Optional[dict] = None, readonly: bool = False): + self.bucket_kwargs = bucket_kwargs or {} + self.readonly = readonly + def __iter__(self) -> Iterator[str]: + import botocore.session -S3DolReadOnly = functools.partial( - S3Dol, s3_client_dol=S3ClientReader, s3_bucket_dol=S3BucketReader -) + yield from botocore.session.Session().available_profiles + + def __contains__(self, k) -> bool: + return isinstance(k, str) and k in set(self) + + def __getitem__(self, k: str): + endpoint_class = EndpointReader if self.readonly else EndpointStore + return endpoint_class(connection=S3Connection(profile=k), **self.bucket_kwargs) diff --git a/s3dol/capabilities.py b/s3dol/capabilities.py new file mode 100644 index 0000000..6909e4b --- /dev/null +++ b/s3dol/capabilities.py @@ -0,0 +1,416 @@ +"""Keyed capabilities as sibling stores; non-keyed operations as free +functions (ADR-0011 D2/D3/D4). + +A keyed capability is a Layer B ``KvReader`` over the **same key space** whose +``__getitem__`` returns the capability:: + + BucketStore k -> bytes + BucketHandles s3dol.handles(s) k -> ObjectHandle + BucketUrls s3dol.urls(s) k -> presigned URL (str) | None + BucketInfo s3dol.info(s) k -> ObjectInfo + +``__getitem__`` is the one thing `dol` maps correctly at every wrapper depth, +so sibling stores are correct *by construction*: no ``inner_most_key``, no +``wrapped_self``, no guard — immune to both the lost-reference hole (D1a) and +the non-``Store``-layer hole (D3a), the only form in ADR-0011 that is. + +**The cost, stated plainly** (D2): a user who wraps the data store must wrap +the sibling in parallel:: + + s = s3_store('bucket', prefix='p/') + h = s3dol.handles(s) + c = KeyCodecs.prefixed('x/') # a user-applied key codec + s2, h2 = c(s), c(h) # wrap BOTH + h2['f'] # -> handle on 'p/x/f'; correct at any depth + +The accessors therefore derive a sibling from an **unwrapped** store and +**raise** on a wrapped one, naming that remedy — s3dol will not guess a user's +codec chain (that is dol#10). + +Free functions (D3) resolve keys with the store held as an argument (alive, so +no weakref registry is consulted). Per D3a they are *more* reliable than +methods, not reliable: a hand-rolled non-``Store`` passthrough layer in the +chain can still defeat the resolution — no docstring here claims "correct at +any wrapper depth"; only the sibling stores may claim that. +""" + +from __future__ import annotations + +from typing import Iterable, Iterator, Optional + +from dol.base import KvReader, Store as DolStore + +from s3dol.base import ( + BucketCollection, + ObjectHandle, + ObjectInfo, + _BucketBase, + _EndpointBase, + _presign, +) +from s3dol.errors import ( + BucketNotEmpty, + ConfigurationError, + CredentialsError, + S3PartialFailure, + code_of, + translate_s3_errors, + translating_s3_errors, +) + +# --------------------------------------------------------------------------- # +# The sibling capability stores (D2) +# --------------------------------------------------------------------------- # + + +class BucketHandles(BucketCollection, KvReader): + """``k -> ObjectHandle`` (key bound at construction; zero round-trips).""" + + def __getitem__(self, k) -> ObjectHandle: + return ObjectHandle(self.bucket, self._id_of_key(k), connection=self.connection) + + +class BucketUrls(BucketCollection, KvReader): + """``k -> presigned URL`` (zero object requests; ``None`` when anonymous). + The canonical spelling of what ``url_for`` shims (ADR-0011 D3b).""" + + def __init__( + self, *args, expires_in: int = 3600, client_method: str = "get_object", **kwargs + ): + super().__init__(*args, **kwargs) + self.expires_in = expires_in + self.client_method = client_method + + def __getitem__(self, k) -> Optional[str]: + return _presign( + self.connection, + bucket=self.bucket, + key=self._id_of_key(k), + expires_in=self.expires_in, + client_method=self.client_method, + ) + + +class BucketInfo(BucketCollection, KvReader): + """``k -> ObjectInfo`` (one ``HeadObject``).""" + + @translate_s3_errors(operation="HeadObject", key_arg="k") + def __getitem__(self, k) -> ObjectInfo: + _id = self._id_of_key(k) + head = self.client.head_object(Bucket=self.bucket, Key=_id) + self._mark_bucket_reachable() + return ObjectInfo( + key=_id, + size=head.get("ContentLength"), + last_modified=head.get("LastModified"), + etag=head.get("ETag"), + content_type=head.get("ContentType"), + storage_class=head.get("StorageClass") or "STANDARD", + restore_status=head.get("Restore"), + version_id=head.get("VersionId"), + ) + + +# --------------------------------------------------------------------------- # +# The accessors: derive a sibling from an UNWRAPPED store; refuse a wrapped one +# --------------------------------------------------------------------------- # + +_PARALLEL_WRAP_REMEDY = ( + "derive the sibling from the *unwrapped* store and apply the same " + "wrapper to both: s2, sib2 = codec(s), codec(s3dol.{fn}(s)). " + "s3dol will not guess your codec chain (that is dol#10)." +) + + +def _leaf_of(store, *, fn: str) -> _BucketBase: + if isinstance(store, _BucketBase): + return store + if isinstance(store, DolStore): + raise ConfigurationError( + f"s3dol.{fn}() needs an unwrapped s3dol store, but got a dol " + f"wrapper ({type(store).__name__}). Returning an unwrapped " + f"sibling here would silently ignore your key/value transforms — " + f"instead, {_PARALLEL_WRAP_REMEDY.format(fn=fn)}" + ) + raise ConfigurationError( + f"s3dol.{fn}() takes an s3dol bucket store; got {type(store).__name__}." + ) + + +def handles(store) -> BucketHandles: + """The sibling store of :class:`ObjectHandle`\\ s: ``handles(s)[k]``.""" + return _leaf_of(store, fn="handles")._make_sibling("handles") + + +def urls( + store, *, expires_in: int = 3600, client_method: str = "get_object" +) -> BucketUrls: + """The sibling store of presigned URLs: ``urls(s)[k]``.""" + return _leaf_of(store, fn="urls")._make_sibling( + "urls", expires_in=expires_in, client_method=client_method + ) + + +def info(store) -> BucketInfo: + """The sibling store of :class:`ObjectInfo`: ``info(s)[k]``.""" + return _leaf_of(store, fn="info")._make_sibling("info") + + +# --------------------------------------------------------------------------- # +# Free functions (D3/D4) +# --------------------------------------------------------------------------- # + + +def _outward_key_chain(store) -> list: + """The wrapper layers outermost-first, ending at the s3dol leaf. + Refuses (rather than guesses) on a non-``Store`` middle layer — D3a's + documented hole.""" + chain = [store] + current = store + while not isinstance(current, _BucketBase): + if not isinstance(current, DolStore): + raise ConfigurationError( + f"Cannot resolve keys through {type(current).__name__}: it is " + f"neither an s3dol leaf nor a dol Store wrapper. Free " + f"functions refuse rather than guess on hand-rolled layers " + f"(ADR-0011 D3a)." + ) + inner = current.store + chain.append(inner) + current = inner + return chain + + +def _map_key_outward(store, wire_relative_key: str) -> str: + """Map a key from the LEAF's relative key space out to the caller's — + the inverse walk ``prefixes`` needs (each layer's ``_key_of_id``, + innermost first). Property-tested round trip: + ``inner_most_key(store, _map_key_outward(store, k_rel))`` recovers the + wire key for pure key-codec chains.""" + chain = _outward_key_chain(store) + key = wire_relative_key + for layer in reversed(chain[:-1]): # innermost wrapper -> outermost + key_of_id = getattr(layer, "_key_of_id", None) + if callable(key_of_id): + key = key_of_id(key) + return key + + +def sub(store, prefix: str): + """A store scoped to ``prefix``, **in the caller's key space** (D3). + + Two branches, deliberately different (each documented in the ADR): + + - **unwrapped s3dol store**: ``leaf._with(prefix=...)`` — cheap, keeps + server-side prefix pushdown, returns the same class; + - **dol-wrapped store**: composes ``Pipe(filt_iter.prefixes(p), + KeyCodecs.prefixed(p))`` over the *outer* store (filter FIRST — the only + safe composition, ADR-0006 §1), which preserves the user's codecs but + loses pushdown (a full scan filtered client-side; D8 closed this: the + cheap path is the unwrapped branch) and returns a different type. + """ + if isinstance(store, _BucketBase): + return store._with(prefix=f"{store.prefix}{_norm(prefix, store.delimiter)}") + if isinstance(store, DolStore): + from dol import KeyCodecs, Pipe, filt_iter + + normalized = _norm(prefix, "/") + return Pipe(filt_iter.prefixes(normalized), KeyCodecs.prefixed(normalized))( + store + ) + raise ConfigurationError( + f"s3dol.sub() takes an s3dol store or a dol wrapper over one; got " + f"{type(store).__name__}." + ) + + +def _norm(prefix: str, delimiter: str) -> str: + from s3dol.base import _normalize_prefix + + return _normalize_prefix(prefix, delimiter) + + +def _wire_scope_base(chain: list) -> str: + """The absolute (wire) prefix the *caller's* key space is rooted at: + the leaf's own prefix extended by each wrapper layer's contribution + (``layer._id_of_key('')``, outermost first). Only delimiter-shaped + contributions are accepted — a codec that maps ``''`` to anything else + (a suffix codec, say) has no well-defined directory structure and is + refused loudly rather than listed wrongly.""" + leaf: _BucketBase = chain[-1] + mapped = "" + for layer in chain[:-1]: # outermost wrapper -> innermost + id_of_key = getattr(layer, "_id_of_key", None) + if not callable(id_of_key): + continue + try: + mapped = id_of_key(mapped) + except Exception as error: + raise ConfigurationError( + f"s3dol.prefixes(): the wrapper layer {type(layer).__name__} " + f"cannot map the key-space root ({type(error).__name__}); its " + f"codec has no well-defined directory structure. List on the " + f"unwrapped store instead." + ) from error + if not isinstance(mapped, str) or ( + mapped and not mapped.endswith(leaf.delimiter) + ): + raise ConfigurationError( + f"s3dol.prefixes(): the wrapper layer {type(layer).__name__} " + f"maps the key-space root to {mapped!r}, which is not a " + f"delimiter-terminated scope — its codec has no well-defined " + f"directory structure. List on the unwrapped store instead." + ) + return f"{leaf.prefix}{mapped}" + + +def prefixes(store) -> list: + """The 'directories' one level under the store's scope, **relative to the + caller's key space** — one ``ListObjectsV2(Delimiter='/')`` reading + ``CommonPrefixes`` (which v0 parsed and then called from nowhere; + ADR-0009). Through a dol wrapper chain, the listing is rooted at the + caller's scope (each prefix layer's contribution) and results are mapped + back **outward** (``_key_of_id`` per layer, innermost first — the inverse + walk `dol` does not provide; ADR-0011 D3). A wrapper whose codec cannot + express the scope raises — loud, not wrong.""" + chain = _outward_key_chain(store) + leaf: _BucketBase = chain[-1] + base = _wire_scope_base(chain) + with translating_s3_errors(leaf, operation="ListObjectsV2"): + found = [] + for page in leaf._op_list_pages(delimiter=leaf.delimiter, prefix=base): + for row in page.get("CommonPrefixes", ()): + relative = leaf._key_of_id(row["Prefix"]) + found.append( + _map_key_outward(store, relative) if len(chain) > 1 else relative + ) + return found + + +#: AWS's DeleteObjects cap. moto accepts 1001, so tier 2 can never catch a +#: missing chunker (ADR-0010 §2) — hence the constant is also unit-tested. +DELETE_MANY_CHUNK = 1000 + + +def delete_many(store, keys: Iterable[str]) -> None: + """Bulk delete (ADR-0010 §2; a free function per ADR-0011 D4 — keyed + + destructive + delegated is the census's data-destroying shape). + + Chunks at 1000 (AWS's cap), parses the ``Errors`` list out of the HTTP + 200 response, and on partial failure raises one :class:`S3PartialFailure` + carrying ``.succeeded`` and ``.failures``. Absent keys are reported by S3 + as ``Deleted`` — this function does **not** distinguish them (same + idempotency as ``del store[k]``). + + Keys are the *caller's* keys: with the store held as an argument the + chain is resolved via ``dol.inner_most_key`` (alive store — no weakref + hole). Validate-the-target caveat (D4): the first argument must be an + s3dol store or a dol wrapper over one. + """ + from dol import inner_most_key + + chain = _outward_key_chain(store) # validates the target + leaf: _BucketBase = chain[-1] + if leaf.connection.anon: + raise CredentialsError("Anonymous credentials cannot sign deletes.") + if len(chain) > 1: + wire_keys = [inner_most_key(store, k) for k in list(keys)] + bad = [k for k in wire_keys if not isinstance(k, str)] + if bad: + raise ConfigurationError( + f"Key resolution through the wrapper chain produced non-str " + f"values ({bad[:3]!r}…): refusing to delete." + ) + else: + wire_keys = [leaf._id_of_key(k) for k in keys] + + row = leaf._resolution().preset + batch_ok = row.capabilities.batch_delete if row is not None else True + succeeded: list = [] + failures: dict = {} + with translating_s3_errors(leaf, operation="DeleteObjects"): + if batch_ok: + for start in range(0, len(wire_keys), DELETE_MANY_CHUNK): + chunk = wire_keys[start : start + DELETE_MANY_CHUNK] + deleted, errors = leaf._op_bulk_delete(chunk) + succeeded.extend(deleted) + for err_key, err_code, err_message in errors: # in an HTTP 200! + failures[err_key] = S3PartialFailure( + f"{err_code}: {err_message}", succeeded=[], failures={} + ) + else: + # Exact, cheap emulation (GCS & checksum-rejecting providers): + # identical observable result, only cost differs (ADR-0003 §2). + from botocore.exceptions import ClientError + + for key in wire_keys: + try: + leaf._op_delete(key) + succeeded.append(key) + except ClientError as error: + failures[key] = S3PartialFailure( + str(code_of(error)), succeeded=[], failures={} + ) + if failures or succeeded: + # Map the reported keys BACK to the caller's key space. A partial- + # failure payload is "keys embedded in an exception" — one of the + # shapes ADR-0011 lists as inexpressible for a codec boundary, because + # the raise site has no boundary frame. Here we have the store as an + # argument, so the inverse walk is available and the caller gets keys + # they can act on rather than wire keys they never used. + def outward(wire_key): + try: + return ( + _map_key_outward(store, leaf._key_of_id(wire_key)) + if len(chain) > 1 + else leaf._key_of_id(wire_key) + ) + except Exception: + return wire_key # un-mappable: report it raw rather than drop it + + succeeded = [outward(k) for k in succeeded] + failures = {outward(k): error for k, error in failures.items()} + if failures: + raise S3PartialFailure( + f"delete_many: {len(failures)} of {len(wire_keys)} deletions " + f"failed ({len(succeeded)} succeeded). .succeeded/.failures are " + f"keyed in YOUR key space, not the wire's.", + succeeded=succeeded, + failures=failures, + ) + + +def delete_bucket(endpoint, name: str, *, force: bool = False) -> None: + """Delete bucket ``name``. Refuses a non-empty bucket unless + ``force=True``, in which case the cascade **paginates** — v0 listed one + page, deleted ≤1000 objects, then failed: a partial, non-idempotent + destruction (ADR-0010 §3). A free function, not a method (ADR-0011 D4).""" + if not isinstance(endpoint, _EndpointBase): + raise ConfigurationError( + f"s3dol.delete_bucket() takes an EndpointStore/Reader first; got " + f"{type(endpoint).__name__}." + ) + if endpoint.connection.anon: + raise CredentialsError("Anonymous credentials cannot delete buckets.") + client = endpoint.client + with translating_s3_errors(endpoint, operation="DeleteBucket", key=name): + if force: + paginator = client.get_paginator("list_objects_v2") + for page in paginator.paginate(Bucket=name): + contents = page.get("Contents", ()) + if contents: + client.delete_objects( + Bucket=name, + Delete={"Objects": [{"Key": row["Key"]} for row in contents]}, + ) + from botocore.exceptions import ClientError + + try: + client.delete_bucket(Bucket=name) + except ClientError as error: + if code_of(error) == "BucketNotEmpty": + raise BucketNotEmpty( + f"Bucket {name!r} is not empty; pass force=True for the " + f"explicit, paginated cascade." + ) from error + raise diff --git a/s3dol/connection.py b/s3dol/connection.py index 799ff35..2e3227b 100644 --- a/s3dol/connection.py +++ b/s3dol/connection.py @@ -57,6 +57,7 @@ AmbiguousResolution, ConfigurationError, MissingEndpoint, + MissingPresetParam, PresetConflict, PresetHostMismatch, ) @@ -167,9 +168,18 @@ class CallableCredentials(CredentialProvider): fetch: Callable[[], Mapping] + def _name(self) -> str: + # NEVER repr(self.fetch): functools.partial's repr prints its bound + # arguments, and the rejection message below recommends a partial — + # a partial closing over a vault token would leak it (D1 §4). + target = getattr(self.fetch, "func", self.fetch) # unwrap a partial + return getattr(target, "__qualname__", None) or type(target).__name__ + + def __repr__(self): + return f"{type(self).__name__}(fetch={self._name()})" + def provenance(self) -> str: - name = getattr(self.fetch, "__qualname__", None) or repr(type(self.fetch)) - return f"callable:{name}" + return f"callable:{self._name()}" @staticmethod def _normalize(result: Mapping) -> dict: @@ -196,15 +206,6 @@ def pick(*names): def _load_botocore_credentials(self): from botocore.credentials import Credentials, RefreshableCredentials - first = self._normalize(self.fetch()) - if first["expiry_time"] is None: - return Credentials( - first["access_key"], - first["secret_key"], - first["token"], - method="s3dol-callable", - ) - def refresh(): fresh = self._normalize(self.fetch()) return { @@ -214,8 +215,18 @@ def refresh(): "expiry_time": fresh["expiry_time"], } + # ONE fetch on the first load — a token vendor may be rate-limited or + # single-use, so the initial credential is the one we keep. + first = refresh() + if first["expiry_time"] is None: + return Credentials( + first["access_key"], + first["secret_key"], + first["token"], + method="s3dol-callable", + ) return RefreshableCredentials.create_from_metadata( - metadata=refresh(), refresh_using=refresh, method="s3dol-callable" + metadata=first, refresh_using=refresh, method="s3dol-callable" ) @@ -269,7 +280,14 @@ def normalize_credentials(value: CredentialsInput) -> Optional[CredentialProvide f"credentials= tuple must be (key, secret) or " f"(key, secret, token); got {len(value)} items." ) - return StaticCredentials(*map(str, value)) + access, secret, *rest = value + token = rest[0] if rest else None + # str()-ing a None token would yield the truthy string 'None' and sign + # every request with X-Amz-Security-Token: None — the natural + # `credentials=(k, s, maybe_token)` shape must behave like the 2-tuple. + return StaticCredentials( + str(access), str(secret), None if token is None else str(token) + ) if isinstance(value, Mapping): unknown = set(value) - set(_AWS_CRED_KEYS) if unknown: @@ -292,11 +310,14 @@ def normalize_credentials(value: CredentialsInput) -> Optional[CredentialProvide try: pickle.dumps(value) except Exception as error: + target = getattr(value, "func", value) + name = getattr(target, "__qualname__", None) or type(target).__name__ raise ConfigurationError( - f"credentials= callable {value!r} is not picklable " + f"credentials= callable {name} is not picklable " f"({type(error).__name__}). Every s3dol store pickles " f"(ADR-0012 D1); use a module-level function or a " - f"functools.partial of one." + f"functools.partial of one. (Its repr is deliberately not " + f"shown — a bound argument may be a secret.)" ) from error return CallableCredentials(value) raise ConfigurationError( @@ -353,29 +374,42 @@ class Resolution: "AWS_RESPONSE_CHECKSUM_VALIDATION", ) -#: Region-looking tokens: ``us-east-1``, ``fr-par``, ``us-west-004``, hetzner's -#: ``fsn1``, and R2's literal ``auto``. -_REGION_TOKEN = re.compile(r"[a-z]{2}-[a-z]+-\d+|[a-z]{2}-[a-z]{3,}|[a-z]{3}\d|auto") +#: Region-shaped tokens, deliberately NARROW: ``us-east-1``, ``fr-par``, +#: ``us-west-004``. A looser pattern (an earlier draft also accepted any +#: ``xxxN`` first label, for hetzner's ``fsn1``) captures ordinary hostname +#: labels — ``srv1.corp``, ``eu-storage.corp`` — and because host capture +#: outranks ``AWS_DEFAULT_REGION`` in the ladder, that garbage would be handed +#: to botocore as the SigV4 credential scope and silently override the user's +#: exported region. Providers whose region is genuinely part of the host state +#: it in their preset row instead (hetzner binds ``{location}``). +_REGION_TOKEN = re.compile( + r"[a-z]{2}-[a-z]+-\d+|[a-z]{2}-[a-z]{3,}-\d{2,}|[a-z]{2}-[a-z]{3,}" +) def _region_from_host(endpoint_url: Optional[str]) -> Optional[str]: - """Best-effort region capture from a resolved endpoint host.""" + """Best-effort region capture — ONLY from the canonical ``s3..`` + (or ``s3-.``) host shape, never from an arbitrary label.""" if not endpoint_url: return None from urllib.parse import urlsplit host = urlsplit(endpoint_url).hostname or "" match = re.search(r"(?:^|\.)s3[.-]([a-z0-9-]+)\.", host) - candidates = [match.group(1)] if match else [] - labels = host.split(".") - if labels: - candidates.append(labels[0]) - for candidate in candidates: - if _REGION_TOKEN.fullmatch(candidate): - return candidate + if match and _REGION_TOKEN.fullmatch(match.group(1)): + return match.group(1) return None +def _looks_secret(name: str) -> bool: + """Whether an option name plausibly carries a secret (repr scrubbing).""" + lowered = str(name).lower() + return any( + marker in lowered + for marker in ("secret", "token", "password", "passwd", "credential") + ) + + _CREDENTIAL_KEY_MARKERS = ( "access_key", "secret", @@ -386,6 +420,15 @@ def _region_from_host(endpoint_url: Optional[str]) -> Optional[str]: ) +def _redact_env_value(key: str, value: Optional[str]) -> Optional[str]: + """A display-safe form of a consulted environment value.""" + from s3dol.errors import redact_url + + if value and "ENDPOINT" in key: + return redact_url(value) + return value + + def _scrub_credentials(config: Mapping) -> dict: """Recursively drop credential-shaped keys from an aws-config mapping.""" @@ -427,10 +470,16 @@ def _lower_rung_endpoint( """The endpoint the rungs *below* s3dol's two would supply — read purely, for diagnose() and the C1/C2 guards **only**, never to override (ADR-0012 D3; and never pass ``Config(ignore_configured_endpoint_urls=True)``).""" + from s3dol.errors import redact_url + for env_key in _ENV_ENDPOINT_KEYS: value = environ.get(env_key) if value: - return value, f"env:{env_key}" + # Redact at capture. This rung is never PASSED to botocore (which + # reads the env itself), only reported — so the Resolution, the C2 + # messages and diagnose all become leak-proof at once, while the + # host survives for detection and the guards. + return redact_url(value), f"env:{env_key}" if aws_config: try: from botocore.configprovider import ConfiguredEndpointProvider @@ -443,7 +492,7 @@ def _lower_rung_endpoint( ) value = provider.provide() if value: - return str(value), "aws-config" + return redact_url(str(value)), "aws-config" except Exception: # config parsing must never fail resolution return None return None @@ -508,28 +557,32 @@ def config() -> Mapping: endpoint = Sourced(lower[0], lower[1], passed=False) elif named is not None: missing = named.missing_endpoint_params(region_name=spec.region_name) - if named.requires_endpoint or missing: + if missing: + raise MissingPresetParam( + f"Preset {named.name!r} has unbound endpoint template " + f"parameter(s) {sorted(missing)}, so it supplies no " + f"endpoint — and nothing below supplied one either. Bind " + f"them (preset=get_preset({named.name!r}).bind(...)), or " + f"pass endpoint_url=. Falling back to AWS silently is " + f"structurally impossible (ADR-0012 C1)." + ) + if named.requires_endpoint: raise MissingEndpoint( - f"Preset {named.name!r} needs an endpoint and nothing " - f"supplied one" - + ( - f" (unbound template parameter(s): {sorted(missing)} — " - f"bind with get_preset({named.name!r}).bind(...))" - if missing - else "" - ) - + ". Pass endpoint_url=, or set AWS_ENDPOINT_URL_S3. " - f"Falling back to AWS silently is structurally impossible " - f"(ADR-0012 C1)." + f"Preset {named.name!r} cannot be reached without an " + f"explicit endpoint and nothing supplied one. Pass " + f"endpoint_url=, or set AWS_ENDPOINT_URL_S3 (ADR-0012 C1)." ) endpoint = Sourced(None, "SDK resolver", passed=False) else: endpoint = Sourced(None, "SDK resolver", passed=False) # -- C2 guard: a named, patterned row whose endpoint arrived from below + # ADR-0012 C2 exempts no *named* row: `soft` governs detection (pass 2), + # not this guard. Exempting 'aws' would silence it for exactly the + # population that declared "I want AWS" and was then retargeted elsewhere + # by an environment variable. if ( named is not None - and not named.soft and named.host_patterns and not endpoint.passed and endpoint.value @@ -680,7 +733,16 @@ def config() -> Mapping: preset_source=preset_source, credential_provenance=provenance, capabilities=row.capabilities, - environ_consulted=tuple((key, environ.get(key)) for key in _ENV_CONSULTED_KEYS), + # Redacted at capture: these are display values (diagnose prints them, + # and a Resolution is the natural thing to log), and an endpoint env + # var may carry userinfo we neither refuse nor pass on. + environ_consulted=tuple( + ( + key, + _redact_env_value(key, environ.get(key)), + ) + for key in _ENV_CONSULTED_KEYS + ), notes=tuple(notes), ) @@ -743,10 +805,44 @@ def __post_init__(self): f"checksum must be 'when_supported', 'when_required', or None; " f"got {self.checksum!r}." ) + if not isinstance(self.signature_version, str) or not self.signature_version: + # NOT a "None == do not override" field, unlike its neighbours: + # ADR-0003 §4 requires signature_version to be set explicitly on + # every client. Leaving it to botocore silently downgrades + # presigning to SigV2 in the regions endpoints.json still lists as + # v2-capable (us-east-1 among them — every conventional MinIO + # setup), and client.meta.config reports 's3v4' while doing it. + raise ConfigurationError( + f"signature_version must be a non-empty str (default 's3v4'); " + f"got {self.signature_version!r}. It is never left to botocore: " + f"that silently re-enables SigV2 presigning (ADR-0003 §4, " + f"issue #10). Pass anon=True for unsigned access." + ) if isinstance(self.client_kwargs, Mapping): object.__setattr__( self, "client_kwargs", tuple(sorted(self.client_kwargs.items())) ) + smuggled = sorted( + key + for key, _ in self.client_kwargs + if key + in ( + "aws_access_key_id", + "aws_secret_access_key", + "aws_session_token", + "config", + ) + ) + if smuggled: + raise ConfigurationError( + f"client_kwargs must not carry {smuggled}: boto3 would accept " + f"them, bypassing credential normalisation, the anon " + f"contradiction check, and repr redaction (the secret would " + f"then print in every log line that reprs this spec). Use " + f"credentials=/profile=, and the connection's own fields for " + f"Config (addressing_style, checksum, signature_version, " + f"payload_signing_enabled, verify)." + ) if self.anon: if self.credentials is not None or self.profile: raise ConfigurationError( @@ -789,7 +885,9 @@ def resolution( return resolve(self, environ if environ is not None else os.environ, aws_config) - def _build_client(self): + def _build_client( + self, *, _override_s3_settings: Optional[dict] = None, _override_endpoint=None + ): import warnings import boto3 @@ -809,6 +907,8 @@ def _build_client(self): s3_settings["payload_signing_enabled"] = ( resolution.payload_signing_enabled.value ) + if _override_s3_settings: + s3_settings.update(_override_s3_settings) config_kwargs: dict = dict(resolution.preset.config_kwargs or ()) if resolution.checksum.passed: config_kwargs["request_checksum_calculation"] = resolution.checksum.value @@ -837,18 +937,52 @@ def _build_client(self): client_kwargs = dict(self.client_kwargs) if self.verify is not None: client_kwargs["verify"] = self.verify + endpoint_to_pass = ( + resolution.endpoint_url.value if resolution.endpoint_url.passed else None + ) + if _override_endpoint is not None: + endpoint_to_pass = _override_endpoint return boto3.Session(botocore_session=botocore_session).client( "s3", - endpoint_url=( - resolution.endpoint_url.value - if resolution.endpoint_url.passed - else None - ), + endpoint_url=endpoint_to_pass, region_name=resolution.region_name.value, config=config, **client_kwargs, ) + @property + def presign_client(self): + """The client presigned URLs are generated with. + + For most providers this is :attr:`client`. Two providers genuinely + need different config for presigning than for the API (ADR-0003 §1): + Hetzner presigns virtual-hosted while the API is path-style; R2 + presigns only on the S3 API domain. When the resolved preset carries + ``presign_endpoint_url``/``presign_addressing_style``, a second client + is built (lazily, once, under the same lock) from the merged config. + """ + resolution = self.resolution() + row = resolution.preset + if row is None or ( + row.presign_endpoint_url is None and row.presign_addressing_style is None + ): + return self.client + built = self.__dict__.get("_presign_client") + if built is None: + with self._lock: + built = self.__dict__.get("_presign_client") + if built is None: + overrides = {} + if row.presign_addressing_style is not None: + overrides["addressing_style"] = row.presign_addressing_style + endpoint = row.presign_endpoint_url + built = self._build_client( + _override_s3_settings=overrides, + _override_endpoint=endpoint, + ) + object.__setattr__(self, "_presign_client", built) + return built + # -- pickling: mandatory for every connection (ADR-0012 D1 §3) --------- # def __getstate__(self): @@ -864,9 +998,18 @@ def __setstate__(self, state): # -- repr: non-default fields only; secrets never (D1 §4) -------------- # def __repr__(self): + from s3dol.errors import redact_url + shown = [] for f in fields(self): value = getattr(self, f.name) - if value != f.default: - shown.append(f"{f.name}={value!r}") + if value == f.default: + continue # only non-defaults, so the repr stays readable + if f.name == "client_kwargs": + value = tuple( + (key, "***" if _looks_secret(key) else item) for key, item in value + ) + elif f.name == "endpoint_url": + value = redact_url(value) + shown.append(f"{f.name}={value!r}") return f"{type(self).__name__}({', '.join(shown)})" diff --git a/s3dol/errors.py b/s3dol/errors.py index fcf1ccc..fe4c364 100644 --- a/s3dol/errors.py +++ b/s3dol/errors.py @@ -173,8 +173,10 @@ class MissingEndpoint(ConfigurationError): supplied (ADR-0012 C1).""" -class MissingPresetParam(ConfigurationError): - """A preset endpoint template has an unbound required parameter.""" +class MissingPresetParam(MissingEndpoint): + """A preset's endpoint template has an unbound required parameter, so the + row contributes no endpoint rung (ADR-0012 C1). A ``MissingEndpoint``, so + either ``except`` works.""" class UnknownPresetParam(ConfigurationError): @@ -231,6 +233,30 @@ class AnonymousFallback(S3DolWarning): ) +def redact_url(url: Optional[str]) -> Optional[str]: + """A URL safe to display: userinfo, query and fragment removed. + + A URL is the residual credential carrier (ADR-0012 D4). Endpoint hygiene + refuses those shapes for values *s3dol* is given, but an endpoint read + from ``AWS_ENDPOINT_URL_S3`` or an aws-config profile is not ours to + refuse — it must merely never be echoed intact. + + >>> redact_url('https://user:pw@minio.internal:9000/x?sig=abc#f') + 'https://REDACTED@minio.internal:9000/x?' + """ + if not url: + return url + from urllib.parse import urlsplit, urlunsplit + + parts = urlsplit(url) + netloc = parts.netloc + if "@" in netloc: + netloc = f"REDACTED@{netloc.rsplit('@', 1)[1]}" + return urlunsplit( + (parts.scheme, netloc, parts.path, "" if parts.query else "", "") + ) + + def redact(text: str) -> str: """Redact signed-URL query secrets from ``text``. @@ -318,6 +344,12 @@ class Classified: ((None, "AuthorizationHeaderMalformed", None), "wrong_region"), # -- archived objects ((None, "InvalidObjectState", None), "archived"), + # -- clock skew / request expiry: 403s that are NOT permission problems. + # They must sit ABOVE the wildcard 403 rows, or deny_means_absent + # would turn a skewed clock into "every key is absent". + ((None, "RequestTimeTooSkewed", None), "credentials"), + ((None, "RequestExpired", None), "credentials"), + ((None, "AuthorizationQueryParametersError", None), "credentials"), # -- credentials ((None, "ExpiredToken", None), "credentials"), ((None, "InvalidAccessKeyId", None), "credentials"), @@ -448,15 +480,21 @@ def translate_client_error( kind = classified.kind if kind == "ambiguous_head": - if head_ambiguity_resolver is not None and head_ambiguity_resolver(): - kind = "object_absent" - elif head_ambiguity_resolver is not None: - kind = "bucket_absent" - else: - kind = "object_absent" # documented default when no resolver bound + if head_ambiguity_resolver is None: + # ADR-0004 §2: the body-less HeadObject 404 must NOT be resolved + # by guessing — "the object is absent" and "the bucket is absent" + # are indistinguishable here on real AWS. With no disambiguator + # bound we propagate the backend error untouched (loud) rather + # than manufacture an absence a caller's .get() would swallow. + return None + kind = "object_absent" if head_ambiguity_resolver() else "bucket_absent" if kind == "access_denied": - if deny_means_absent and key is not None: + # Only a genuine permission code may be read as absence. A 403 that + # means something else (clock skew, expired request) is classified + # above; this guard is the second belt. + permission_code = code in (None, "AccessDenied", "403", "AllAccessDisabled") + if deny_means_absent and key is not None and permission_code: kind = "object_absent" else: return AccessDenied( @@ -497,6 +535,39 @@ def translate_client_error( return exc_type(message) +from contextlib import contextmanager + + +@contextmanager +def translating_s3_errors(store, *, operation: str, key=None, **translate_kwargs): + """Generator/inline form of the seam, for code the decorator cannot wrap + (``__iter__`` bodies: a decorated generator function returns its generator + before the body runs, so the decorator's ``except`` never fires). + + Same contract as :func:`translate_s3_errors`; the store supplies + ``_error_context()``. + """ + from botocore.exceptions import ClientError, NoCredentialsError + + try: + yield + except ClientError as error: + context = dict(getattr(store, "_error_context", dict)()) + context.update(translate_kwargs) + replacement = translate_client_error( + error, operation=operation, key=key, **context + ) + if replacement is None: + raise + raise replacement from error + except NoCredentialsError as error: + raise CredentialsError( + "No AWS credentials found. Configure credentials (env vars, " + "~/.aws/credentials, SSO, or S3Connection(credentials=...)), " + "or pass anon=True for public data." + ) from error + + def translate_s3_errors( *, operation: str, @@ -518,6 +589,14 @@ def decorator(func): import inspect sig = inspect.signature(func) + if key_arg is not None and key_arg not in sig.parameters: + raise TypeError( + f"translate_s3_errors(key_arg={key_arg!r}) on " + f"{func.__qualname__}: that function has no such parameter " + f"(it has {list(sig.parameters)[1:]}). A silently unresolved " + f"key_arg would drop the key from every error message and " + f"disable the deny_means_absent conversion." + ) @wraps(func) def wrapper(self, *args, **kwargs): @@ -529,6 +608,7 @@ def wrapper(self, *args, **kwargs): key = None if key_arg is not None: bound = sig.bind(self, *args, **kwargs) + bound.apply_defaults() # a defaulted key is still the key key = bound.arguments.get(key_arg) context = dict(getattr(self, "_error_context", dict)()) context.update(translate_kwargs) diff --git a/s3dol/presets.py b/s3dol/presets.py index 2a14a50..cee65f1 100644 --- a/s3dol/presets.py +++ b/s3dol/presets.py @@ -36,9 +36,8 @@ from s3dol.errors import ( ConfigurationError, InvalidEndpoint, - MissingPresetParam, - UnknownPresetParam, Rule, + UnknownPresetParam, ) # --------------------------------------------------------------------------- # @@ -115,10 +114,20 @@ def validate_endpoint_hygiene(url: str, *, what: str = "endpoint_url") -> None: f"with '@' in its netloc. Pass credentials via " f"S3Connection(credentials=...), never in a URL." ) + # The refusal messages deliberately do NOT echo the URL: its query/ + # fragment is exactly the secret carrier this check exists to stop, and an + # exception message lands in tracebacks, logs and diagnose reports. if parts.query: - raise InvalidEndpoint(f"{what} must not carry a query string: {url!r}") + raise InvalidEndpoint( + f"{what} must not carry a query string (host " + f"{parts.hostname!r}): the query is where signed-URL and SAS-style " + f"credentials live, and botocore would retain it verbatim in " + f"client.meta.endpoint_url. Pass the bare endpoint." + ) if parts.fragment: - raise InvalidEndpoint(f"{what} must not carry a fragment: {url!r}") + raise InvalidEndpoint( + f"{what} must not carry a fragment (host {parts.hostname!r})." + ) @dataclass(frozen=True) @@ -200,6 +209,23 @@ def __post_init__(self): f"Preset {self.name!r}: config_kwargs values must be " f"scalars; {key!r} is a {type(value).__name__}." ) + from s3dol.errors import KINDS + + for rule in self.error_overrides: + try: + (_operation, _code, _status), kind = rule + except (TypeError, ValueError): + raise ConfigurationError( + f"Preset {self.name!r}: each error_overrides row is " + f"((operation, code, status), kind); got {rule!r}." + ) from None + if kind not in KINDS: + raise ConfigurationError( + f"Preset {self.name!r}: error_overrides kind {kind!r} is " + f"not one of {list(KINDS)}. An unvalidated kind would " + f"detonate inside the error-translation seam at request " + f"time, masking the real backend error." + ) unknown = set(dict(self.params)) - self.template_params() if unknown: raise UnknownPresetParam( diff --git a/s3dol/reads.py b/s3dol/reads.py new file mode 100644 index 0000000..1ed0560 --- /dev/null +++ b/s3dol/reads.py @@ -0,0 +1,62 @@ +"""Read strategies (ADR-0005 §6): what ``s[k]`` returns. + +``s[k]`` returns ``bytes`` — always, under the default strategy, because the +N1 canonical form demands it. ``stream_reads`` is the documented *runtime* +variation (the store's values become chunk iterators); it does not change the +class's static value type — honest typing there would need a generic store, +which v1 does not do. + +Ranged reads and download-to-disk are per-object concerns and live on +``ObjectHandle`` (``handle.read(offset=, length=)``, ``handle.download_to``), +not here — a store-level "ranged" strategy would need a per-key range, which a +Mapping cannot express. + +Strategies are frozen dataclasses, not closures, because every s3dol store +must pickle (ADR-0008 §6) and a store carries its strategy. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from functools import partial +from typing import Any, Iterator, Protocol + + +class ReadStrategy(Protocol): + def __call__(self, client: Any, bucket: str, key: str) -> Any: ... + + +#: Default streaming chunk size (8 MiB, matching the write side). +DEFAULT_CHUNK_SIZE = 8 * 2**20 + + +@dataclass(frozen=True) +class BytesReads: + """The default: one ``GetObject``, fully read -> ``bytes``.""" + + def __call__(self, client, bucket: str, key: str) -> bytes: + return client.get_object(Bucket=bucket, Key=key)["Body"].read() + + +@dataclass(frozen=True) +class StreamReads: + """Values become lazy chunk iterators (the body streams; nothing is held + in memory). The ``GetObject`` itself — and its ``KeyError`` — still + happens eagerly at ``s[k]``; only the byte transfer is lazy.""" + + chunk_size: int = DEFAULT_CHUNK_SIZE + + def __call__(self, client, bucket: str, key: str) -> Iterator[bytes]: + body = client.get_object(Bucket=bucket, Key=key)["Body"] + return iter(partial(body.read, self.chunk_size), b"") + + +def bytes_reads() -> BytesReads: + return BytesReads() + + +def stream_reads(**kwargs) -> StreamReads: + return StreamReads(**kwargs) + + +DEFAULT_READS = BytesReads() diff --git a/s3dol/recipes.py b/s3dol/recipes.py new file mode 100644 index 0000000..b255c40 --- /dev/null +++ b/s3dol/recipes.py @@ -0,0 +1,138 @@ +"""Layer C — the one-liner factory and codec facades. Composition ONLY. + +If a recipe cannot be expressed as Layer B + `dol` wrappers, that is evidence +the capability belongs in Layer B as a *parameter* — this module never grows a +class statement (ADR-0001; the rule that keeps per-vendor classes from +reappearing). + +The one-liner:: + + s = s3dol.s3_store('my-bucket') # zero ceremony + s = s3dol.s3_store('my-bucket', prefix='logs/') # scoped + s = s3dol.s3_store('pub-bucket', anon=True) # public data + +With ``on_missing_bucket='assume'`` (the default; no probe, ever — ADR-0010) +the two-line quickstart raises on a bucket that does not exist yet; start from +nothing with ``on_missing_bucket='create'``. +""" + +from __future__ import annotations + +import json +from typing import Optional, Union + +from dol import wrap_kvs + +from s3dol.base import BucketReader, BucketStore +from s3dol.connection import S3Connection +from s3dol.errors import ConfigurationError +from s3dol.presets import Preset +from s3dol.reads import ReadStrategy +from s3dol.writes import WriteStrategy + + +def s3_store( + bucket: str, + *, + prefix: str = "", + preset: Union[str, Preset, None] = None, + connection: Optional[S3Connection] = None, + endpoint_url: Optional[str] = None, + region_name: Optional[str] = None, + profile: Optional[str] = None, + credentials=None, + anon: bool = False, + on_missing_bucket: str = "assume", + readonly: bool = False, + value_codec=None, + reads: Optional[ReadStrategy] = None, + writes: Optional[WriteStrategy] = None, +): + """One bucket as a ``MutableMapping[str, bytes]`` (a ``Mapping`` when + ``readonly=True``). + + Annotated (deliberately) as the abstract Mapping, **not** ``-> BucketStore``: + with ``value_codec`` the concrete type is a generated `dol` wrapper class + that is *not* a ``BucketStore`` subclass. (v0 annotated its factory + ``-> Store`` and returned something that was not one; inverting that lie + is not an improvement — architecture.md.) + + Connection axes (``preset``/``endpoint_url``/``region_name``/``profile``/ + ``credentials``/``anon``) are conveniences that build an + :class:`~s3dol.connection.S3Connection`; pass ``connection=`` instead for + full control (they are mutually exclusive with it). + """ + if connection is not None: + if ( + any( + x is not None + for x in (preset, endpoint_url, region_name, profile, credentials) + ) + or anon + ): + raise ConfigurationError( + "Pass either connection= or the individual connection axes " + "(preset/endpoint_url/region_name/profile/credentials/anon), " + "not both — one SSOT for where the bytes go." + ) + else: + connection = S3Connection( + preset=preset, + endpoint_url=endpoint_url, + region_name=region_name, + profile=profile, + credentials=credentials, + anon=anon, + ) + store_class = BucketReader if readonly else BucketStore + store = store_class( + bucket, + connection=connection, + prefix=prefix, + reads=reads, + writes=writes, + on_missing_bucket=on_missing_bucket, + ) + if value_codec is not None: + store = value_codec(store) + return store + + +# --------------------------------------------------------------------------- # +# Codec facades. Note S3Jsons uses value_encoder/decoder, NOT +# ValueCodecs.json(): that codec encodes to *str*, which Layer B rejects +# (ADR-0005 §2), and value_codec= does not compose via Pipe. A conformance +# assertion checks every shipped recipe's encoder output is a BytesSource. +# And never `bytes.decode` bare as a transform — dol's signature-inference +# would misread it (dol issue #9): wrap in a lambda. +# --------------------------------------------------------------------------- # + +S3Texts = wrap_kvs( + BucketStore, + obj_of_data=lambda b: b.decode(), + data_of_obj=lambda s: s.encode(), +) +S3Texts.__doc__ = """``BucketStore`` whose values are ``str`` (utf-8 on the wire).""" + +S3Jsons = wrap_kvs( + BucketStore, + obj_of_data=json.loads, + data_of_obj=lambda obj: json.dumps(obj).encode(), +) +S3Jsons.__doc__ = """``BucketStore`` whose values are JSON-able Python objects. + +Note the canonical-form law (ADR-0005 N3): ``s[k] = v`` then ``s[k]`` gives +``json.loads(json.dumps(v))`` — tuples come back as lists, int dict keys as str. +""" + + +def _pickle_codec_store(): + import pickle + + return wrap_kvs(BucketStore, obj_of_data=pickle.loads, data_of_obj=pickle.dumps) + + +S3Pickles = _pickle_codec_store() +S3Pickles.__doc__ = ( + """``BucketStore`` whose values are arbitrary picklable Python objects.""" +) diff --git a/s3dol/store.py b/s3dol/store.py index a63fe60..3a71c9f 100644 --- a/s3dol/store.py +++ b/s3dol/store.py @@ -1,207 +1,288 @@ -"""S3 Store Class""" +"""COMPAT SHIM — the v0 ``S3Store``, forwarded onto the v1 machinery. + +Removed in v2. This module **must survive as an importable module**: external +dependents do ``from s3dol.store import S3Store`` (fully qualified), not +``from s3dol import S3Store`` (ADR-0007 §3). + +The shim keeps v0's exact signature (``bucket_name`` positional-or-keyword, +``path=`` not renamed, the ``make_bucket`` tri-state, and +``is_supabase_endpoint=`` which an early draft forgot) and is **a fix delivery +mechanism**: callers get the corrected endpoint/credential resolution without +changing a line. Three v0 behaviours a dependent's *passing tests* rely on are +deliberately preserved here (and only here — ADR-0007 §4): + +- ``store[k] = 'a str'`` is accepted (encoded utf-8, with a + ``DeprecationWarning``); +- ``store['folder/']`` returns a sub-store; +- ``del store[absent]`` silently succeeds (v1 deletes are idempotent anyway). + +And one v0 behaviour is preserved on the Supabase branch only: the read-side +de-chunker for objects that v0's misconfigured writes stored with their +``aws-chunked`` HTTP framing verbatim (ADR-0003 §3). It is implemented +*correctly* here (multi-chunk; full-parse-or-passthrough) where v0's truncated +multi-chunk payloads. Per ADR-0003's sequencing rule the codec is not deleted +until a repair path ships — :func:`detect_chunked_framing` is that path's +diagnostic. + +When v1 resolves a call differently than v0 *effectively* did (v0 dropped an +explicit ``endpoint_url`` whenever env credentials were set, and let env +credentials override explicit ones), the shim warns with +:class:`~s3dol.errors.S3DolResolutionChanged` — the ADR-0012 D7 differential. +This module and that warning die together in v2. +""" + +from __future__ import annotations + +import os +import re +import warnings +from typing import MutableMapping, Optional + +from dol.base import KvPersister + +from s3dol._diagnose import _spec_from_v0_args, _v0_mangle +from s3dol.base import BucketStore +from s3dol.connection import resolve +from s3dol.errors import S3DolResolutionChanged + +# --------------------------------------------------------------------------- # +# The corrected aws-chunked de-framer (Supabase legacy objects) +# --------------------------------------------------------------------------- # + +_CHUNK_HEAD = re.compile(rb"^[0-9a-fA-F]{1,16}(;|\r\n)") + + +def dechunk_aws_chunked(data: bytes) -> bytes: + """Undo verbatim-persisted ``aws-chunked`` framing, if present. + + Full-parse-or-passthrough: either the ENTIRE payload parses as chunked + framing (every chunk header, every CRLF, through the terminal ``0`` chunk) + and the concatenated chunks are returned, or the payload is returned + untouched. v0's parser truncated any payload larger than one chunk and + misfired on payloads that merely *started* with hex digits; requiring a + complete valid parse removes both failure modes (a random payload that + happens to be a complete chunked document is astronomically unlikely). + + >>> dechunk_aws_chunked(b'5\\r\\nhello\\r\\n0\\r\\n\\r\\n') + b'hello' + >>> framed = b'3\\r\\nabc\\r\\n2\\r\\nde\\r\\n0\\r\\nx-amz-checksum-crc32:AAAA\\r\\n\\r\\n' + >>> dechunk_aws_chunked(framed) # multi-chunk (v0 truncated this) + b'abcde' + >>> dechunk_aws_chunked(b'cafe is hex-ish but not framed') + b'cafe is hex-ish but not framed' + """ + if not _CHUNK_HEAD.match(data): + return data + parts = [] + position = 0 + while True: + crlf = data.find(b"\r\n", position) + if crlf < 0: + return data + size_token = data[position:crlf].split(b";")[0] + try: + size = int(size_token, 16) + except ValueError: + return data + body_start = crlf + 2 + if size == 0: + # Trailer (e.g. x-amz-checksum-crc32:…) then blank line — accept + # anything through the final CRLFCRLF (or end of payload). + rest = data[body_start:] + if rest and not rest.endswith(b"\r\n"): + return data + return b"".join(parts) + chunk = data[body_start : body_start + size] + if ( + len(chunk) < size + or data[body_start + size : body_start + size + 2] != b"\r\n" + ): + return data + parts.append(chunk) + position = body_start + size + 2 + + +def detect_chunked_framing(store, *, sample: int = 20) -> list: + """Which of the first ``sample`` keys hold verbatim ``aws-chunked`` + framing (objects corrupted by v0 Supabase writes). The repair path: + ``store[k] = dechunk_aws_chunked(raw[k])`` per flagged key, reading raw + bytes through an un-codec'd store.""" + flagged = [] + for i, key in enumerate(store): + if i >= sample: + break + raw = store[key] + if isinstance(raw, bytes) and dechunk_aws_chunked(raw) != raw: + flagged.append(key) + return flagged + + +# --------------------------------------------------------------------------- # +# The shim store: v0 conveniences over a v1 BucketStore +# --------------------------------------------------------------------------- # + + +class _V0CompatStore(KvPersister): + """The v0 ergonomics (str writes, ``'folder/'`` sub-stores, counting + ``len()``) over a v1 :class:`BucketStore`. Shim-only; not public API.""" + + def __init__(self, inner: BucketStore, *, dechunk: bool = False): + self._inner = inner + self._dechunk = dechunk -from typing import Optional -from dol import Store + def __getitem__(self, k): + if isinstance(k, str) and k.endswith(self._inner.delimiter): + return type(self)( + self._inner._with(prefix=f"{self._inner.prefix}{k}"), + dechunk=self._dechunk, + ) + value = self._inner[k] + if self._dechunk and isinstance(value, bytes): + value = dechunk_aws_chunked(value) + return value -import boto3 -from s3dol.base import S3BucketDol, S3ClientDol -from s3dol.utility import S3DolException + def __setitem__(self, k, v): + if isinstance(v, str): + warnings.warn( + "Writing a str to an s3dol store is deprecated (removed in " + "v2): it will be stored as its utf-8 bytes. Be explicit: " + "s[k] = v.encode() — or s3dol.Filepath(v) if you meant a " + "file's contents.", + DeprecationWarning, + stacklevel=2, + ) + v = v.encode() + self._inner[k] = v + + def __delitem__(self, k): + # v1 deletes are idempotent (no probe), which is exactly v0's + # "del store[absent] silently succeeds" — nothing to shim. + del self._inner[k] + + def __iter__(self): + return iter(self._inner) + + def __contains__(self, k): + if isinstance(k, str) and k.endswith(self._inner.delimiter): + # v0: `'d/' in s` answered via HeadObject on the marker; keep it. + return self._inner.__contains__(k) if k else False + return k in self._inner + + def __len__(self): + # v0's len() worked (by counting — two listings via the length hint). + # The v1 store raises; the shim keeps v0's behaviour, cost and all. + return sum(1 for _ in self._inner) + + def url_for(self, k: str, **kwargs) -> Optional[str]: + # lacing calls store.url_for(k) on the v0 store; forward to the v1 + # guarded shim (prefix-aware, SigV4 — this is what closes issue #10). + return self._inner.url_for(k, **kwargs) + + def __repr__(self): + return f"S3Store({self._inner!r})" + + +def _warn_if_resolution_changed(v0_args: dict) -> None: + """The ADR-0012 D7 runtime differential: one v0 argument-mangler run + through v1's own resolver twice; warn iff the outcomes differ. Cost when + env credentials are absent: two environ lookups.""" + branch, effective = _v0_mangle(v0_args, os.environ) + if effective is None or effective == v0_args: + return # excluded branches, or v0 didn't mangle anything + try: + environ = dict(os.environ) + v1_now = resolve(_spec_from_v0_args(v0_args), environ) + v0_was = resolve(_spec_from_v0_args(effective), environ) + except Exception: + return # the differential must never break the construction + moved = [ + axis + for axis in ("endpoint_url", "region_name") + if getattr(v1_now, axis).value != getattr(v0_was, axis).value + ] + if v1_now.credential_provenance != v0_was.credential_provenance: + moved.append("credential identity") + if moved: + warnings.warn( + f"s3dol v1 resolves this S3Store(...) call differently than v0 " + f"did (v0 branch: {branch}; changed: {', '.join(moved)}). v0 was " + f"silently discarding an explicit argument; v1 honours it. Run " + f"s3dol.diagnose(**your_S3Store_kwargs) for the full table.", + S3DolResolutionChanged, + stacklevel=3, + ) -# TODO: Add support for key-based grouping (store['folder/subfolder/'] is the store with path = original_path + 'folder/subfolder/' -# TODO: Make profile_name handling work for SupabaseS3BucketDol and S3BucketDolWithouBucketCheck -# TODO: Review this S3Store (with SupabaseS3BucketDol etc.) setup and see if we can make it simpler/cleaner def S3Store( bucket_name: str, *, - make_bucket: bool | None = None, - path: str | None = None, - aws_access_key_id: str = None, - aws_secret_access_key: str = None, - aws_session_token: str = None, - endpoint_url: str = None, - region_name: str = None, - profile_name: str = None, - skip_bucket_check: bool | None = None, - is_supabase_endpoint: bool | None = None, -) -> Store: - """S3 Bucket Store - - :param bucket_name: name of bucket to store data in - :param make_bucket: if True, create bucket if it does not exist. - If None, skip bucket existence checks completely. - If False, check for existence but don't create. - :param path: prefix to use for bucket keys - :param aws_access_key_id: AWS access key ID - :param aws_secret_access_key: AWS secret access key - :param aws_session_token: AWS session token - :param endpoint_url: URL of S3 endpoint - :param region_name: AWS region name - :param profile_name: AWS profile name - :return: S3BucketDol + make_bucket: Optional[bool] = None, + path: Optional[str] = None, + aws_access_key_id: Optional[str] = None, + aws_secret_access_key: Optional[str] = None, + aws_session_token: Optional[str] = None, + endpoint_url: Optional[str] = None, + region_name: Optional[str] = None, + profile_name: Optional[str] = None, + skip_bucket_check: Optional[bool] = None, + is_supabase_endpoint: Optional[bool] = None, +) -> MutableMapping: + """DEPRECATED — the v0 entry point, kept working until v2. + + Use :func:`s3dol.s3_store` instead: ``s3_store(bucket, prefix=...)``. + + Signature is v0's exactly (``path=`` not renamed; ``make_bucket`` + tri-state maps ``True -> on_missing_bucket='create'``, ``False -> + 'raise'``, ``None -> 'assume'``). Behaviour differences vs v0 are bug + fixes only (explicit endpoint/credentials honoured; listing errors raise + instead of returning ``[]``; writes no longer create buckets unless asked) + and announce themselves via ``S3DolResolutionChanged`` when they change + this call's resolution. """ - - if is_supabase_endpoint is None: - is_supabase_endpoint = _is_supabase_endpoint(endpoint_url) - if skip_bucket_check is None and is_supabase_endpoint: - skip_bucket_check = True - # For Supabase endpoints, create a direct client without session token - - params = dict( + warnings.warn( + "s3dol.store.S3Store is deprecated and will be removed in s3dol 2.0: " + "use s3dol.s3_store(bucket, prefix=..., ...) (note: prefix=, not " + "path=).", + DeprecationWarning, + stacklevel=2, + ) + v0_args = dict( + bucket_name=bucket_name, + make_bucket=make_bucket, + path=path, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, aws_session_token=aws_session_token, endpoint_url=endpoint_url, region_name=region_name, - ) - - if is_supabase_endpoint: - # For Supabase endpoints, use a custom store that doesn't check for bucket existence - return SupabaseS3BucketDol.from_params( - **params, - bucket_name=bucket_name, - path=path, - ) - elif skip_bucket_check: - return S3BucketDolWithouBucketCheck.from_params( - **params, - bucket_name=bucket_name, - prefix=path, - ) - - # For standard AWS, use the regular flow - - s3cr = S3ClientDol( - **params, profile_name=profile_name, + skip_bucket_check=skip_bucket_check, + is_supabase_endpoint=is_supabase_endpoint, ) + _warn_if_resolution_changed(v0_args) - bucket = s3cr[bucket_name] - - if make_bucket is None: - bucket.skip_bucket_exists_check = True - elif make_bucket is True and bucket_name not in s3cr: - s3cr[bucket_name] = {} - - if path: - # Apply the key prefix in the standard flow too. It was documented - # (":param path: prefix to use for bucket keys") but silently dropped - # here — only the Supabase / skip-check branches honored it. Normalize - # exactly as S3BucketReader.__post_init__ does. #7 follow-up. - bucket.prefix = f"{path.strip(bucket.delimiter)}{bucket.delimiter}" - - return bucket - - -def _is_supabase_endpoint(endpoint_url: str) -> bool: - """Check if the endpoint URL is a Supabase endpoint""" - return endpoint_url and ".supabase." in endpoint_url - - -def validate_bucket(bucket_name: str, s3_client: S3ClientDol, make_bucket: bool | None): - """Validate bucket name and create if needed - - If make_bucket is None, skip existence check entirely. - If make_bucket is True, create bucket if it doesn't exist. - If make_bucket is False, check existence but don't create. - """ - if make_bucket is None: - # Skip validation - just return a bucket object without checking existence - return s3_client[bucket_name] - elif make_bucket is True and bucket_name not in s3_client: - s3_client[bucket_name] = {} - return s3_client[bucket_name] - - -class S3BucketDolWithouBucketCheck(S3BucketDol): - """A S3BucketDol that completely avoids the bucket exists check. - This is needed for Supabase endpoints where the bucket is not created - until the first object is uploaded, for example. - """ + if is_supabase_endpoint is None: + is_supabase_endpoint = bool(endpoint_url and ".supabase." in endpoint_url) - def __setitem__(self, k, v): - _id = self._id_of_key(k) # TODO: Smelly. use trans tools - self.client.put_object(Bucket=self.bucket_name, Key=_id, Body=v) - - def _bucket_exists(self): - return True - - @classmethod - def from_params( - cls, - bucket_name: str, - *, - path: str | None = None, - aws_access_key_id: str = None, - aws_secret_access_key: str = None, - aws_session_token: str = None, - region_name: str = None, - endpoint_url: str = None, - ): - """Create a S3BucketDol without session token""" - client = boto3.client( - "s3", - aws_access_key_id=aws_access_key_id, - aws_secret_access_key=aws_secret_access_key, - endpoint_url=endpoint_url, - aws_session_token=aws_session_token, - region_name=region_name, + connection = _spec_from_v0_args(v0_args) + if is_supabase_endpoint: + # Route through the preset row: path addressing + checksum + # when_required — the WRITE-side cure for what the read-side + # de-chunker treats (ADR-0003 §3). + from s3dol.connection import S3Connection + + connection = S3Connection( + preset="supabase", + endpoint_url=connection.endpoint_url, + region_name=connection.region_name, + profile=connection.profile, + credentials=connection.credentials, ) - - return cls(client=client, bucket_name=bucket_name, prefix=path) - - -# TODO: Messy. Should use wrap_kvs. -class SupabaseS3BucketDol(S3BucketDolWithouBucketCheck): - """ - S3BucketDol for Supabase endpoints. - Over the S3BucketDolWithouBucketCheck, it adds support to unravel the returned - content from the stuff (number of bytes, check sum, etc.) superbase adds to the - content. - """ - - def __getitem__(self, k): - _id = self._id_of_key(k) # TODO: Smelly. use trans tools - - # Handle nested key access for keys ending with delimiter - if _id.endswith(self.delimiter): - _kw = {**self.__dict__, "prefix": _id} - return type(self)(**_kw) - - try: - response = self.client.get_object(Bucket=self.bucket_name, Key=_id) - raw_data = response["Body"].read() - - # Process HTTP chunked encoding directly on bytes - # Look for the pattern: [chunk size in hex]\r\n[data]\r\n0\r\n[optional headers]\r\n\r\n - if ( - raw_data.startswith(b"0") - or raw_data[0:2].isdigit() - or ( - raw_data[0:1].isdigit() - and raw_data[1:2].isalpha() - and raw_data[1:2].lower() in b"abcdef" - ) - ): - # Find the first CRLF - first_crlf_pos = raw_data.find(b"\r\n") - if first_crlf_pos != -1: - # Extract what should be the hex chunk size - hex_size = raw_data[:first_crlf_pos] - try: - # Skip the chunk size and the CRLF - content_start = first_crlf_pos + 2 - # Find the end of the chunk (marked by another CRLF) - content_end = raw_data.find(b"\r\n", content_start) - if content_end != -1: - # Extract just the content between the CRLFs - return raw_data[content_start:content_end] - except ValueError: - pass # Not valid hex, continue to return raw data - - # If we couldn't parse it as chunked encoding or any step failed, - # return the raw data as a fallback - return raw_data - - except self.client.exceptions.NoSuchKey as ex: - raise KeyError(f"Key {k} does not exist") from ex + on_missing_bucket = {True: "create", False: "raise", None: "assume"}[make_bucket] + inner = BucketStore( + bucket_name, + connection=connection, + prefix=path or "", + on_missing_bucket=on_missing_bucket, + ) + return _V0CompatStore(inner, dechunk=is_supabase_endpoint) diff --git a/s3dol/testing.py b/s3dol/testing.py new file mode 100644 index 0000000..c651d62 --- /dev/null +++ b/s3dol/testing.py @@ -0,0 +1,426 @@ +"""``s3dol.testing`` — a shipped in-memory fake and the exported conformance +suite (ADR-0008). Downstream users can test services that take a store as a +dependency with **no network, no docker, no moto**:: + + from s3dol.testing import mock_s3 + + def test_my_service(): + assert MediaService(mock_s3()).url('a') + +The fake is not a re-implementation: :class:`FakeBucketStore` *is* the real +:class:`~s3dol.base.BucketStore` with only the backend protocol (the ``_op_*`` +seam of ADR-0002) swapped for a dict — so the prefix arithmetic, key validity, +error taxonomy (the fake raises *synthesized botocore ClientErrors* through +the same translation seam), guards and Mapping semantics are structurally +identical, and drift fails the same conformance suite that gates the real +store. + +The suite is parameterized by capability flags: ``mock_s3()`` runs the +structural presign assertion (2a) but not the fetch-it assertion (2b — there +is no HTTP server), so the honest claim is *"the same suite, parameterized by +an ``endpoint_is_fetchable`` capability"* (ADR-0008). +""" + +from __future__ import annotations + +import pickle +from datetime import datetime, timezone +from typing import Callable, Optional +from urllib.parse import quote + +from s3dol.base import BucketStore, ObjectInfo, clamp_expires_in +from s3dol.capabilities import BucketHandles, BucketInfo, BucketUrls +from s3dol.errors import KeyNotValid, ObjectNotFound, S3Error +from s3dol.values import as_fileobj, is_bytes_like, reject_str_value + + +def _client_error(code: str, status: int, operation: str): + """A synthesized botocore ClientError, exactly like the wire produces.""" + from botocore.exceptions import ClientError + + return ClientError( + { + "Error": {"Code": code, "Message": "synthesized by s3dol.testing"}, + "ResponseMetadata": {"HTTPStatusCode": status}, + }, + operation, + ) + + +class _FakeBackend: + """The shared in-memory 'bucket': absolute (wire) keys -> bytes.""" + + def __init__(self, data: Optional[dict] = None, *, bucket_exists: bool = True): + self.data: dict = dict(data or {}) + self.bucket_exists = bucket_exists + + +class _FakeOps: + """Mixin overriding exactly the ``_op_*`` backend protocol with dict + operations that raise synthesized ``ClientError``s — everything above the + seam (guards, taxonomy, key arithmetic) runs unchanged.""" + + def __init__(self, *args, backing: Optional[_FakeBackend] = None, **kwargs): + self._backing = backing if backing is not None else _FakeBackend() + super().__init__(*args, **kwargs) + + # the fake never builds a client + @property + def client(self): + raise S3Error( + "s3dol.testing fakes have no boto3 client — a code path reached " + "the backend outside the _op_* protocol seam (that is a bug worth " + "reporting)." + ) + + def _check_bucket(self, operation): + if not self._backing.bucket_exists: + raise _client_error("NoSuchBucket", 404, operation) + + def _op_list_pages( + self, *, delimiter: Optional[str] = None, prefix: Optional[str] = None + ): + self._check_bucket("ListObjectsV2") + base = self.prefix if prefix is None else prefix + keys = sorted(k for k in self._backing.data if k.startswith(base)) + if delimiter is None: + contents = [ + { + "Key": key, + "Size": len(self._backing.data[key]), + "LastModified": datetime.now(timezone.utc), + "ETag": '"fake"', + } + for key in keys + ] + yield {"Contents": contents} + return + seen, prefixes = set(), [] + for key in keys: + rest = key[len(base) :] + if delimiter in rest.rstrip(delimiter): + head = rest.split(delimiter, 1)[0] + delimiter + if head not in seen: + seen.add(head) + prefixes.append({"Prefix": f"{base}{head}"}) + yield {"CommonPrefixes": prefixes} + + def _op_head_object(self, _id): + self._check_bucket("HeadObject") + if _id not in self._backing.data: + # moto returns a body here; real AWS does not. The fake mimics + # REAL AWS (code synthesized from the status) so the ambiguity + # path of ADR-0004 §2 is exercised, which moto cannot do. + raise _client_error("404", 404, "HeadObject") + value = self._backing.data[_id] + return { + "ContentLength": len(value), + "LastModified": datetime.now(timezone.utc), + "ETag": '"fake"', + "ContentType": "binary/octet-stream", + } + + def _op_head_bucket(self): + if not self._backing.bucket_exists: + raise _client_error("404", 404, "HeadBucket") + + def _op_read(self, _id): + self._check_bucket("GetObject") + if _id not in self._backing.data: + raise _client_error("NoSuchKey", 404, "GetObject") + return self._backing.data[_id] + + def _op_write(self, _id, value): + self._check_bucket("PutObject") + if isinstance(value, str): + reject_str_value(value) + if is_bytes_like(value): + self._backing.data[_id] = bytes(value) + else: + with as_fileobj(value) as fileobj: + self._backing.data[_id] = fileobj.read() + + def _op_delete(self, _id): + self._check_bucket("DeleteObject") + self._backing.data.pop(_id, None) # idempotent, like the wire + + def _op_bulk_delete(self, _ids): + self._check_bucket("DeleteObjects") + deleted = [] + for _id in _ids: + self._backing.data.pop(_id, None) + deleted.append(_id) # absent keys report as Deleted, like S3 + return deleted, [] + + def _op_create_bucket(self): + self._backing.bucket_exists = True + + def _op_presign(self, _id, *, expires_in, client_method, **params): + if self.connection.anon: + return None + expires_in = clamp_expires_in(expires_in) # same guard as the real one + return ( + f"https://{self.bucket}.mock-s3.invalid/{quote(_id)}" + f"?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Expires={expires_in}" + f"&X-Amz-Signature=mock" + ) + + def _sibling_kwargs(self): + return super()._sibling_kwargs() + + def _make_sibling(self, kind, **options): + sibling_class = { + "handles": FakeBucketHandles, + "urls": FakeBucketUrls, + "info": FakeBucketInfo, + }[kind] + return sibling_class( + **{**super()._sibling_kwargs(), **options}, backing=self._backing + ) + + def _with(self, **overrides): + kwargs = super()._sibling_kwargs() + kwargs["on_missing_bucket"] = self.on_missing_bucket + kwargs.update(overrides) + return type(self)(kwargs.pop("bucket"), backing=self._backing, **kwargs) + + +class FakeObjectHandle: + """The fake's per-object escape hatch (mirrors ``ObjectHandle``).""" + + def __init__(self, bucket: str, key: str, *, backing: _FakeBackend, anon=False): + self.bucket = bucket + self.key = key + self._backing = backing + self._anon = anon + + def __repr__(self): + return f"{type(self).__name__}({self.bucket!r}, {self.key!r})" + + def _bytes(self) -> bytes: + if self.key not in self._backing.data: + raise ObjectNotFound(f"GetObject failed: key={self.key!r}") + return self._backing.data[self.key] + + def read(self, *, offset=None, length=None) -> bytes: + data = self._bytes() + if offset is None and length is None: + return data + start = offset or 0 + return data[start : None if length is None else start + length] + + def stream(self, *, chunk_size=8 * 2**20): + data = self._bytes() + for start in range(0, len(data), chunk_size): + yield data[start : start + chunk_size] + + def download_to(self, path) -> str: + from s3dol.values import Filepath + + target = path.path if isinstance(path, Filepath) else str(path) + with open(target, "wb") as f: + f.write(self._bytes()) + return target + + def write(self, value) -> None: + if isinstance(value, str): + reject_str_value(value) + if is_bytes_like(value): + self._backing.data[self.key] = bytes(value) + else: + with as_fileobj(value) as fileobj: + self._backing.data[self.key] = fileobj.read() + + def delete(self) -> None: + self._backing.data.pop(self.key, None) + + def exists(self) -> bool: + return self.key in self._backing.data + + def info(self) -> ObjectInfo: + return ObjectInfo( + key=self.key, + size=len(self._bytes()), + last_modified=datetime.now(timezone.utc), + etag='"fake"', + content_type="binary/octet-stream", + storage_class="STANDARD", + ) + + def url(self, *, expires_in=3600, client_method="get_object", **params): + if self._anon: + return None + return ( + f"https://{self.bucket}.mock-s3.invalid/{quote(self.key)}" + f"?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Expires={expires_in}" + f"&X-Amz-Signature=mock" + ) + + +class FakeBucketStore(_FakeOps, BucketStore): + """The real ``BucketStore`` over a dict backend.""" + + +class FakeBucketHandles(_FakeOps, BucketHandles): + def __getitem__(self, k): + return FakeObjectHandle( + self.bucket, + self._id_of_key(k), + backing=self._backing, + anon=self.connection.anon, + ) + + +class FakeBucketUrls(_FakeOps, BucketUrls): + def __getitem__(self, k): + return self._op_presign( + self._id_of_key(k), + expires_in=self.expires_in, + client_method=self.client_method, + ) + + +class FakeBucketInfo(_FakeOps, BucketInfo): + def __getitem__(self, k): + _id = self._id_of_key(k) + if _id not in self._backing.data: + raise ObjectNotFound(f"HeadObject failed: key={k!r}") + return ObjectInfo( + key=_id, + size=len(self._backing.data[_id]), + last_modified=datetime.now(timezone.utc), + etag='"fake"', + content_type="binary/octet-stream", + storage_class="STANDARD", + ) + + +def mock_s3( + bucket: str = "mock-bucket", + *, + prefix: str = "", + data: Optional[dict] = None, + bucket_exists: bool = True, + anon: bool = False, +) -> FakeBucketStore: + """An in-process fake ``BucketStore``: no network, no docker, no moto. + + ``data`` seeds the bucket with *absolute* (wire) keys -> bytes. + + >>> s = mock_s3(data={'logs/a.txt': b'hi', 'other/z': b'no'}, prefix='logs') + >>> list(s), s['a.txt'] + (['a.txt'], b'hi') + >>> s['b'] = b'new'; sorted(s) + ['a.txt', 'b'] + """ + from s3dol.connection import S3Connection + + return FakeBucketStore( + bucket, + backing=_FakeBackend(data, bucket_exists=bucket_exists), + connection=S3Connection(anon=anon) if anon else S3Connection(), + prefix=prefix, + ) + + +# --------------------------------------------------------------------------- # +# The exported conformance suite (ADR-0008) +# --------------------------------------------------------------------------- # + + +def run_conformance( + make_store: Callable[[], object], + *, + endpoint_is_fetchable: bool = False, + normalize: Callable = lambda v: v, + check_pickle: bool = True, + check_urls: bool = True, + secret: Optional[str] = None, +) -> None: + """Assert the s3dol store laws against a fresh store from ``make_store``. + + ``make_store`` must return an EMPTY, writable store (each call may return + a new one over the same empty backend). ``normalize`` is the per-layer + canonical form of ADR-0005 N3 (identity for Layer B; ``json.loads ∘ + json.dumps`` for a json recipe, …). ``secret``, when given, is asserted + absent from every repr this suite touches. + + Sibling packages and user code are welcome to run this against their own + stores — that is why it ships in the package rather than in ``tests/``. + """ + from urllib.parse import urlsplit + + store = make_store() + + # -- Mapping laws + the value law (N3) + store["a"] = b"alpha" + store["b/c"] = b"beta" + assert store["a"] == normalize(b"alpha") + assert sorted(store) == ["a", "b/c"] + assert "a" in store and "b/c" in store + assert "missing" not in store + # iter/contains agreement — the law EncodingType breaks (ADR-0006 §4) + assert all(k in store for k in store) + try: + store["missing"] + raise AssertionError("reading an absent key must raise KeyError") + except KeyError: + pass + assert store.get("missing") is None # the defaulted accessor degrades + del store["b/c"] + assert "b/c" not in store + del store["b/c"] # idempotent (documented; costs no probe) + + # -- len() raises with guidance (ADR-0008 cost model) + try: + len(store) + raise AssertionError("len(store) must raise TypeError") + except TypeError: + pass + + # -- keys that must round-trip (and KeyNotValid before the wire) + for key in ["plain", "a b", "café", "a+b", "a\rb", "p/x y", "a%20b"]: + store[key] = b"v" + assert all(k in store for k in store) + for bad in ["", 123, "x" * 2000]: + try: + store[bad] = b"v" + raise AssertionError(f"key {bad!r} must be rejected") + except (KeyNotValid, KeyError, ValueError, TypeError): + pass + + # -- str values are rejected, loudly (ADR-0005 §2) + try: + store["s"] = "a string" + raise AssertionError("str values must be rejected") + except TypeError: + pass + + # -- url assertions (2a: structural, never substring — ADR-0008) + if check_urls: + url_for = getattr(store, "url_for", None) + if callable(url_for): + url = url_for("a") + if url is not None: + parts = urlsplit(url) + assert "X-Amz-Algorithm=AWS4-HMAC-SHA256" in (parts.query or ""), url + assert "AWSAccessKeyId" not in url, f"SigV2 tell in {url!r}" + assert parts.path.split("/")[-1] in ("a", quote("a")), ( + f"URL path must address the key: {url!r}" + ) + if endpoint_is_fetchable: + import requests + + assert requests.get(url).content == normalize(b"alpha") + + # -- pickling (ADR-0008 §6): a store must cross a process boundary + if check_pickle: + clone = pickle.loads(pickle.dumps(store)) + assert sorted(clone) == sorted(store) + + # -- no secret in repr (ADR-0008 §7) + if secret is not None: + assert secret not in repr(store) + connection = getattr(store, "connection", None) + if connection is not None: + assert secret not in repr(connection) diff --git a/s3dol/tests/test_base.py b/s3dol/tests/test_base.py index f6a61c9..9ca279b 100644 --- a/s3dol/tests/test_base.py +++ b/s3dol/tests/test_base.py @@ -1,190 +1,391 @@ -import os -import pytest +"""Layer B: prefix arithmetic, key validity, contracts, guards. + +Mostly tier 1 against the in-memory fake (which *is* the real store with only +the backend protocol swapped), plus a moto section for what only a real client +can show. +""" + +import pickle +import warnings -from s3dol.base import S3BucketDol, S3ClientReader, S3Dol, S3ClientDol, S3DolReadOnly +import pytest +from dol import KeyCodecs, Pipe, filt_iter + +from s3dol.base import ( + MAX_KEY_BYTES, + BucketStore, + ObjectHandle, + _normalize_prefix, + _validate_key, +) +from s3dol.errors import ( + BucketNotFound, + ConfigurationError, + CredentialsError, + KeyNotValid, + NotSupported, + ObjectNotFound, +) +from s3dol.testing import FakeBucketStore, _FakeBackend, mock_s3 -def overwrite_aws_environment_variables( - aws_access_key_id, aws_secret_access_key, endpoint_url -): - os.environ["AWS_ACCESS_KEY_ID"] = aws_access_key_id - os.environ["AWS_SECRET_ACCESS_KEY"] = aws_secret_access_key - os.environ["AWS_ENDPOINT_URL"] = endpoint_url +# --------------------------------------------------------------------------- # +# prefix arithmetic (ADR-0006 §1) — the boundary the whole scoping story rests on +# --------------------------------------------------------------------------- # @pytest.mark.parametrize( - "aws_access_key_id, aws_secret_access_key, endpoint_url", - [("localstack", "localstack", "http://localhost:4566")], + "given, expected", + [("logs", "logs/"), ("logs/", "logs/"), ("/logs/", "logs/"), ("", ""), (None, "")], ) -def test_s3_dol_crud(aws_access_key_id, aws_secret_access_key, endpoint_url): - overwrite_aws_environment_variables( - aws_access_key_id, aws_secret_access_key, endpoint_url - ) +def test_prefix_normalisation_is_mandatory_and_first(given, expected): + assert _normalize_prefix(given) == expected + + +def test_unnormalised_prefix_cannot_leak_a_sibling_tenant(): + # The one-character boundary violation ADR-0006 §1 exists to prevent: + # prefix='logs' must NOT expose 'logs2/2026.txt' as key '2/2026.txt'. + data = {"logs/2026.txt": b"mine", "logs2/2026.txt": b"NEIGHBOUR", "logsX": b"x"} + store = mock_s3(data=data, prefix="logs") + assert sorted(store) == ["2026.txt"] + assert store["2026.txt"] == b"mine" + assert "2/2026.txt" not in store + with pytest.raises(KeyError): + store["2/2026.txt"] + + +def test_scoping_round_trip_with_siblings_markers_and_non_matching_keys(): + data = { + "a/b": b"1", + "a/c": b"2", + "z": b"3", # non-matching + "ab/x": b"4", # sibling prefix + "a/": b"", # the exact-prefix marker + } + store = mock_s3(data=data, prefix="a/") + assert sorted(store) == ["b", "c"] # marker filtered; siblings excluded + assert all(k in store for k in store) + for key in store: + assert store._key_of_id(store._id_of_key(key)) == key - test_bucket_name = "test-bucket" - test_key = "test-key" - test_value = b"test-value" - - s3 = S3Dol() - assert "environment variables" in s3 - profile_names = list(s3) - assert "environment variables" in profile_names - - # create client from profile - s3_client = s3["environment variables"] - # create bucket - if test_bucket_name in s3_client: - del s3_client[test_bucket_name] - assert test_bucket_name not in s3_client - - s3_client[test_bucket_name] = {} - s3_bucket = s3_client[test_bucket_name] - assert test_bucket_name in s3_client - bucket_names = list(s3_client) - assert test_bucket_name in bucket_names - n_obj = len(list(s3_bucket)) - - # create and delete object - assert test_key not in s3_bucket - s3_bucket[test_key] = test_value - assert test_key in s3_bucket - assert test_key in list(s3_bucket) - assert s3_bucket[test_key] == test_value - assert s3["environment variables"][test_bucket_name][test_key] == test_value - assert len(list(s3_bucket)) == n_obj + 1 - del s3_bucket[test_key] - assert test_key not in s3_bucket - assert list(s3_bucket) == [] - assert len(list(s3_bucket)) == n_obj - - # delete bucket - del s3_client[test_bucket_name] - assert test_bucket_name not in s3_client - assert test_bucket_name not in list(s3_client) - - # delete bucket with object - s3_client[test_bucket_name] = {} - s3_bucket = s3_client[test_bucket_name] - s3_bucket[test_key] = test_value - del s3_client[test_bucket_name] - assert test_bucket_name not in s3_client - - # access bucket object with path delimiations - s3_client[test_bucket_name] = {} - s3_bucket = s3_client[test_bucket_name] - s3_bucket["level1/level2/test-key"] = test_value - - assert_bucket_key_value(s3_bucket, "level1/level2/test-key", test_value) - assert_bucket_key_value(s3_bucket["level1/level2/"], "test-key", test_value) - assert_bucket_key_value(s3_bucket["level1/"]["level2/"], "test-key", test_value) - assert_bucket_key_value(s3_bucket["level1/"], "level2/test-key", test_value) - - # delete object with path delimiations - del s3_bucket["level1/"]["level2/"]["test-key"] - assert "level1/level2/test-key" not in s3_bucket - - -def assert_bucket_key_value(s3_bucket, key, value): - assert key in s3_bucket, f"failed contains test with key: {key}" - assert key in list(s3_bucket), f"failed iter test with key: {key}" - assert s3_bucket[key] == value, f"failed get test with key: {key}" - - -def mk_s3_client_from_env( - aws_access_key_id, aws_secret_access_key, endpoint_url, s3_client_class -): - overwrite_aws_environment_variables( - aws_access_key_id, aws_secret_access_key, endpoint_url - ) - return s3_client_class() +def test_key_of_id_raises_rather_than_slicing_a_non_matching_id(): + # The belt to the server-side Prefix's suspenders: a provider that ignores + # Prefix must not silently produce corrupt keys. + store = mock_s3(prefix="a/") + with pytest.raises(KeyNotValid, match="outside"): + store._key_of_id("zzz/other") -def mk_s3_client_from_params( - aws_access_key_id, aws_secret_access_key, endpoint_url, s3_client_class -): - return s3_client_class( - aws_access_key_id=aws_access_key_id, - aws_secret_access_key=aws_secret_access_key, - endpoint_url=endpoint_url, - ) +def test_exact_prefix_marker_is_hidden_from_its_own_scoped_view(): + # The documented wart: it would relativise to '', which is not a valid key. + store = mock_s3(data={"a/": b"", "a/b": b"1"}, prefix="a/") + assert sorted(store) == ["b"] + # ...and stays addressable by absolute key on an unscoped store. + assert sorted(mock_s3(data={"a/": b"", "a/b": b"1"})) == ["a/", "a/b"] -@pytest.mark.parametrize( - "aws_access_key_id, aws_secret_access_key, endpoint_url", - [("localstack", "localstack", "http://localhost:4566")], -) -@pytest.mark.parametrize( - "mk_s3_client", [mk_s3_client_from_env, mk_s3_client_from_params] -) -def test_s3_client( - aws_access_key_id, aws_secret_access_key, endpoint_url, mk_s3_client -): - test_bucket_name = "test-bucket" - - s3_client = mk_s3_client( - aws_access_key_id=aws_access_key_id, - aws_secret_access_key=aws_secret_access_key, - endpoint_url=endpoint_url, - s3_client_class=S3ClientDol, + +# --------------------------------------------------------------------------- # +# key validity, before the wire (ADR-0006 §4) +# --------------------------------------------------------------------------- # + + +@pytest.mark.parametrize("bad", ["", 123, None, b"bytes"]) +def test_invalid_keys_are_keynotvalid_not_backend_types(bad): + with pytest.raises(KeyNotValid): + _validate_key(bad) + + +def test_key_length_limit_is_utf8_bytes_not_characters(): + _validate_key("é" * (MAX_KEY_BYTES // 2)) # 1024 bytes exactly: fine + with pytest.raises(KeyNotValid, match="UTF-8 bytes"): + _validate_key("é" * (MAX_KEY_BYTES // 2 + 1)) + + +def test_lone_surrogate_key_is_rejected_locally(): + with pytest.raises(KeyNotValid, match="UTF-8"): + _validate_key("bad\ud800key") + + +def test_keynotvalid_is_both_keyerror_and_valueerror(): + store = mock_s3() + with pytest.raises(KeyError): + store[""] + with pytest.raises(ValueError): + store[""] + + +# --------------------------------------------------------------------------- # +# Mapping contracts (architecture.md table) +# --------------------------------------------------------------------------- # + + +def test_len_raises_with_guidance_rather_than_counting(): + with pytest.raises(TypeError, match="sum\\(1 for _ in store\\)"): + len(mock_s3()) + + +def test_delete_is_idempotent_by_default_and_probes_only_when_asked(): + store = mock_s3(data={"a": b"1"}) + del store["a"] + del store["a"] # idempotent: S3 returns 204 for an absent key + strict = FakeBucketStore( + "mock-bucket", backing=_FakeBackend({"a": b"1"}), strict_delete=True ) + del strict["a"] + with pytest.raises(KeyError): + del strict["a"] + + +def test_missing_bucket_raises_on_iteration_never_yields_empty(): + store = FakeBucketStore("mock-bucket", backing=_FakeBackend(bucket_exists=False)) + with pytest.raises(BucketNotFound): + list(store) + + +def test_on_missing_bucket_assume_performs_no_probe(): + # 'assume' must never call HeadBucket at construction (ADR-0010 §1). + backend = _FakeBackend(bucket_exists=False) + FakeBucketStore("mock-bucket", backing=backend) # constructs fine + with pytest.raises(BucketNotFound): + FakeBucketStore("mock-bucket", backing=backend, on_missing_bucket="raise") + + +def test_on_missing_bucket_create_recovers_without_probing(): + backend = _FakeBackend(bucket_exists=False) + store = FakeBucketStore("mock-bucket", backing=backend, on_missing_bucket="create") + store["k"] = b"v" # attempt -> NoSuchBucket -> create -> retry once + assert store["k"] == b"v" and backend.bucket_exists + + +def test_setdefault_never_overwrites_an_archived_object(): + # MutableMapping.setdefault is try/except KeyError -> assign, so an + # ObjectArchived KeyError would silently overwrite the archived object. + from s3dol.errors import ObjectArchived + + class Archived(FakeBucketStore): + def _op_read(self, _id): + raise ObjectArchived("archived") + + store = Archived("mock-bucket", backing=_FakeBackend({"a": b"COLD"})) + with pytest.raises(ObjectArchived): + store.setdefault("a", b"REPLACEMENT") + assert store._backing.data["a"] == b"COLD" # untouched + with pytest.raises(ObjectArchived): + store.pop("a", b"default") + + +def test_stores_pickle(): + store = mock_s3(data={"p/a": b"1"}, prefix="p") + clone = pickle.loads(pickle.dumps(store)) + assert sorted(clone) == ["a"] and clone["a"] == b"1" + + +def test_repr_shows_scope_and_no_secrets(): + store = mock_s3(prefix="tenant-a") + text = repr(store) + assert "mock-bucket" in text and "tenant-a/" in text + + +# --------------------------------------------------------------------------- # +# writes: the value domain +# --------------------------------------------------------------------------- # - if test_bucket_name in s3_client: - del s3_client[test_bucket_name] - assert test_bucket_name not in s3_client - s3_client[test_bucket_name] = {} - s3_bucket = s3_client[test_bucket_name] - assert isinstance(s3_bucket, S3BucketDol) +def test_str_values_are_refused_at_the_store_boundary(): + with pytest.raises(TypeError, match="Filepath"): + mock_s3()["k"] = "a string" - s3_client_reader = mk_s3_client( - aws_access_key_id=aws_access_key_id, - aws_secret_access_key=aws_secret_access_key, - endpoint_url=endpoint_url, - s3_client_class=S3ClientReader, + +def test_filepath_ref_writes_file_content(tmp_path): + from s3dol.values import Filepath + + payload = tmp_path / "big.bin" + payload.write_bytes(b"CONTENT") + store = mock_s3() + store["k"] = Filepath(payload) + assert store["k"] == b"CONTENT" + + +def test_chunks_ref_concatenates(): + from s3dol.values import Chunks + + store = mock_s3() + store["k"] = Chunks(iter([b"ab", b"cd"])) + assert store["k"] == b"abcd" + + +# --------------------------------------------------------------------------- # +# anonymous stores refuse writes locally (ADR-0012 D5) +# --------------------------------------------------------------------------- # + + +def test_anonymous_store_refuses_writes_locally(): + store = mock_s3(anon=True) + with pytest.raises(CredentialsError): + store["k"] = b"v" + with pytest.raises(CredentialsError): + del store["k"] + + +def test_anonymous_url_for_returns_none_not_raises(): + # SupportsUrlFor is Optional[str]; raising would turn a public-bucket + # streaming fallback into a 500 (ADR-0012 D5). + assert mock_s3(data={"k": b"v"}, anon=True).url_for("k") is None + + +# --------------------------------------------------------------------------- # +# url_for: the single guarded keyed method (ADR-0011 D3b) +# --------------------------------------------------------------------------- # + + +def test_url_for_unwrapped_addresses_the_prefixed_key(): + store = mock_s3(data={"p/f": b"v"}, prefix="p") + assert "p/f" in store.url_for("f") + + +def test_url_for_refuses_once_the_leaf_has_been_wrapped(): + # The finding behind ADR-0011 D3b's revision: TWO call paths reach this + # one signature with different key domains — dol.content_url pre-resolves + # the key through the outer layers and stops at the provider "because that + # layer applies its own" (dol/content.py), while a direct delegated call + # hands over the raw outer key. Indistinguishable here, and a wrong + # presigned URL addresses a different REAL object, so: refuse. + for make_call in ( + lambda s: KeyCodecs.prefixed("x/")(s).url_for("g"), # temporary wrapper + lambda s: _wrap_then_call(s), # named wrapper + ): + store = mock_s3(data={"p/x/g": b"v"}, prefix="p") + with pytest.raises(NotSupported, match="s3dol.urls"): + make_call(store) + + +def _wrap_then_call(store): + wrapper = KeyCodecs.prefixed("x/")(store) + return wrapper.url_for("g") + + +def test_url_for_refuses_even_after_the_wrapper_is_collected(): + # A delegated bound method holds no reference to its wrapper, so the + # weakref-based wrapped_self degrades to "never wrapped" here (ADR-0011 + # D1a). The probe-descriptor flag survives that, which is what makes the + # refusal reliable rather than intermittent. + store = mock_s3(data={"p/f": b"v"}, prefix="p") + wrapper = KeyCodecs.prefixed("x/")(store) + del wrapper + with pytest.raises(NotSupported, match="s3dol.urls"): + store.url_for("f") + + +def test_dol_content_url_is_correct_for_an_unwrapped_store(): + # The reason url_for exists at all (dol.SupportsUrlFor is a *method* and + # content_url reaches it by getattr). + from dol import content_url + + store = mock_s3(data={"p/g": b"v"}, prefix="p") + assert "p/g" in content_url(store, "g") + + +def test_the_sibling_store_is_correct_where_url_for_must_refuse(): + # ...and the remedy the refusal names is correct at any depth, because it + # goes through __getitem__ (ADR-0011 D2). + import s3dol + + store = mock_s3(data={"p/x/g": b"v"}, prefix="p") + wrapped_data, wrapped_urls = ( + KeyCodecs.prefixed("x/")(store), + KeyCodecs.prefixed("x/")(s3dol.urls(store)), ) - s3_bucket = s3_client_reader[test_bucket_name] - assert isinstance(s3_bucket, S3BucketDol) + assert wrapped_data["g"] == b"v" + assert "p/x/g" in wrapped_urls["g"] - if test_bucket_name in s3_client: - del s3_client[test_bucket_name] - assert test_bucket_name not in s3_client_reader +def test_url_for_clamps_and_warns_past_sigv4s_cap(): + store = mock_s3(data={"k": b"v"}) + with pytest.warns(UserWarning, match="604800"): + url = store.url_for("k", expires_in=999_999) + assert "X-Amz-Expires=604800" in url -@pytest.mark.parametrize( - "aws_access_key_id, aws_secret_access_key, endpoint_url", - [("localstack", "localstack", "http://localhost:4566")], -) -def test_s3_dol_readonly(aws_access_key_id, aws_secret_access_key, endpoint_url): - overwrite_aws_environment_variables( - aws_access_key_id, aws_secret_access_key, endpoint_url + +# --------------------------------------------------------------------------- # +# ObjectHandle binds its key at construction +# --------------------------------------------------------------------------- # + + +def test_handle_reads_ranges_streams_and_downloads(tmp_path): + from s3dol.testing import FakeObjectHandle + + handle = FakeObjectHandle( + "b", "k", backing=_FakeBackend({"k": b"the quick brown fox"}) ) + assert handle.read() == b"the quick brown fox" + assert handle.read(offset=4, length=5) == b"quick" + assert b"".join(handle.stream(chunk_size=4)) == b"the quick brown fox" + target = handle.download_to(tmp_path / "out.bin") + assert open(target, "rb").read() == b"the quick brown fox" + assert handle.exists() and handle.info().size == 19 - test_bucket_name = "test-bucket" - test_key = "test-key" - test_key2 = "test-key2" - test_value = b"test-value" - s3 = S3Dol() - s3["environment variables"][test_bucket_name] = { - test_key: test_value, - test_key2: test_value, - } +def test_handle_missing_object_raises_keyerror(): + from s3dol.testing import FakeObjectHandle + + handle = FakeObjectHandle("b", "nope", backing=_FakeBackend()) + with pytest.raises(ObjectNotFound): + handle.read() + assert handle.exists() is False + + +# --------------------------------------------------------------------------- # +# construction validation +# --------------------------------------------------------------------------- # + + +@pytest.mark.parametrize("bad_bucket", ["", None, 42]) +def test_bucket_must_be_a_non_empty_str(bad_bucket): + with pytest.raises(ConfigurationError): + mock_s3(bad_bucket) - s3_readonly = S3DolReadOnly()["environment variables"] - assert test_bucket_name in s3_readonly - assert test_key in s3_readonly[test_bucket_name] - assert test_key2 in s3_readonly[test_bucket_name] - assert s3_readonly[test_bucket_name][test_key] == test_value - with pytest.raises(TypeError): - s3_readonly[test_bucket_name][test_key] = test_value +def test_unknown_on_missing_bucket_value_rejected(): + with pytest.raises(ConfigurationError, match="on_missing_bucket"): + FakeBucketStore("b", backing=_FakeBackend(), on_missing_bucket="probe") - with pytest.raises(TypeError): - del s3_readonly[test_bucket_name][test_key] - del s3["environment variables"][test_bucket_name][test_key] - assert test_key not in s3_readonly[test_bucket_name] +# --------------------------------------------------------------------------- # +# moto tier 2: what only a real client shows +# --------------------------------------------------------------------------- # -if __name__ == "__main__": - pytest.main([__file__]) +def test_real_store_round_trip_and_presign_is_sigv4(monkeypatch): + moto = pytest.importorskip("moto") + for var in ("AWS_ENDPOINT_URL", "AWS_ENDPOINT_URL_S3", "AWS_PROFILE"): + monkeypatch.delenv(var, raising=False) + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "testing") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "testing") + monkeypatch.setenv("AWS_DEFAULT_REGION", "us-east-1") + from s3dol.connection import S3Connection + + with moto.mock_aws(): + connection = S3Connection(region_name="us-east-1") + connection.client.create_bucket(Bucket="real-bucket") + store = BucketStore("real-bucket", connection=connection, prefix="p") + store["a"] = b"alpha" + assert store["a"] == b"alpha" and sorted(store) == ["a"] + url = store.url_for("a") + assert "X-Amz-Algorithm=AWS4-HMAC-SHA256" in url and "/p/a" in url + assert "AWSAccessKeyId" not in url + # the object really is at the prefixed key + raw = connection.client.get_object(Bucket="real-bucket", Key="p/a") + assert raw["Body"].read() == b"alpha" + + +def test_real_store_pickles_across_a_process_boundary(monkeypatch): + pytest.importorskip("moto") + from s3dol.connection import S3Connection + + store = BucketStore( + "real-bucket", connection=S3Connection(region_name="us-east-1"), prefix="p" + ) + clone = pickle.loads(pickle.dumps(store)) + assert clone.bucket == store.bucket and clone.prefix == store.prefix + assert clone.connection == store.connection diff --git a/s3dol/tests/test_capabilities.py b/s3dol/tests/test_capabilities.py new file mode 100644 index 0000000..5f59413 --- /dev/null +++ b/s3dol/tests/test_capabilities.py @@ -0,0 +1,239 @@ +"""Sibling capability stores and free functions (ADR-0011 D2/D3/D4).""" + +import pytest +from dol import KeyCodecs, Pipe, cached_keys, filt_iter + +import s3dol +from s3dol.errors import ConfigurationError, CredentialsError, S3PartialFailure +from s3dol.testing import mock_s3 + +# --------------------------------------------------------------------------- # +# D2 — sibling stores are correct BY CONSTRUCTION (they route through +# __getitem__, the one thing dol maps correctly at every depth) +# --------------------------------------------------------------------------- # + + +def test_siblings_share_the_stores_key_space(): + store = mock_s3(data={"p/f": b"value"}, prefix="p") + assert s3dol.handles(store)["f"].key == "p/f" + assert "p/f" in s3dol.urls(store)["f"] + assert s3dol.info(store)["f"].size == 5 + + +@pytest.mark.parametrize( + "wrap", + [ + lambda s: KeyCodecs.prefixed("x/")(s), + lambda s: Pipe(filt_iter.prefixes("x/"), KeyCodecs.prefixed("x/"))(s), + lambda s: cached_keys(KeyCodecs.prefixed("x/")(s)), + ], + ids=["key-codec", "pipe", "cached_keys-over-codec"], +) +def test_parallel_wrapping_is_correct_at_any_depth_and_lifetime(wrap): + # The property no resolution primitive can offer: no inner_most_key, no + # wrapped_self, no live reference needed. + store = mock_s3(data={"p/x/f": b"D"}, prefix="p") + assert wrap(store)["f"] == b"D" + assert wrap(s3dol.handles(store))["f"].key == "p/x/f" + assert "p/x/f" in wrap(s3dol.urls(store))["f"] + # ...including when nothing holds the wrapper (a temporary in one expr): + assert KeyCodecs.prefixed("x/")(s3dol.handles(store))["f"].key == "p/x/f" + + +def test_accessors_refuse_a_wrapped_store_and_name_the_remedy(): + store = mock_s3(data={"p/f": b"v"}, prefix="p") + wrapped = KeyCodecs.prefixed("x/")(store) + for accessor in (s3dol.handles, s3dol.urls, s3dol.info): + with pytest.raises(ConfigurationError) as info: + accessor(wrapped) + message = str(info.value) + assert "unwrapped" in message and "codec(" in message + + +def test_accessors_reject_a_non_store(): + with pytest.raises(ConfigurationError, match="s3dol.handles"): + s3dol.handles({"a": b"1"}) + + +def test_url_sibling_takes_presign_options(): + store = mock_s3(data={"k": b"v"}) + assert "X-Amz-Expires=60" in s3dol.urls(store, expires_in=60)["k"] + + +def test_siblings_iterate_the_same_keys_as_the_data_store(): + store = mock_s3(data={"p/a": b"1", "p/b": b"2", "other": b"3"}, prefix="p") + assert sorted(s3dol.handles(store)) == sorted(store) == ["a", "b"] + + +def test_anonymous_urls_sibling_yields_none(): + assert s3dol.urls(mock_s3(data={"k": b"v"}, anon=True))["k"] is None + + +# --------------------------------------------------------------------------- # +# D3 — free functions +# --------------------------------------------------------------------------- # + + +def test_sub_on_an_unwrapped_store_extends_the_prefix_and_keeps_pushdown(): + store = mock_s3(data={"a/b/c": b"1", "a/d": b"2"}, prefix="a") + child = s3dol.sub(store, "b") + assert list(child) == ["c"] and child["c"] == b"1" + assert type(child) is type(store) # same class; the prefix moved + assert child.prefix == "a/b/" + + +def test_sub_on_a_wrapped_store_composes_filter_first(): + # Filter FIRST is the only safe composition (ADR-0006 §1): reversed, the + # store is silently empty — and bare prefixed() corrupts out-of-band keys. + store = mock_s3(data={"p/x/b/c": b"1", "p/x/d": b"2"}, prefix="p") + wrapped = KeyCodecs.prefixed("x/")(store) + child = s3dol.sub(wrapped, "b") + assert list(child) == ["c"] and child["c"] == b"1" + + +def test_sub_rejects_a_foreign_object(): + with pytest.raises(ConfigurationError, match="s3dol.sub"): + s3dol.sub({"a": 1}, "p") + + +def test_prefixes_lists_one_level_in_the_callers_key_space(): + store = mock_s3( + data={"a/b/c": b"1", "a/d": b"2", "a/e/f": b"3"}, prefix="a" + ) + assert sorted(s3dol.prefixes(store)) == ["b/", "e/"] + + +def test_prefixes_maps_outward_through_a_wrapper(): + store = mock_s3(data={"p/x/dir/a": b"1", "p/x/e": b"2", "p/other/z": b"3"}, prefix="p") + wrapped = KeyCodecs.prefixed("x/")(store) + # scoped to the caller's key space: 'other/' is NOT theirs to see + assert s3dol.prefixes(wrapped) == ["dir/"] + + +def test_prefixes_refuses_a_wrapper_whose_codec_has_no_directory_structure(): + store = mock_s3(data={"p/a/b": b"1"}, prefix="p") + wrapped = KeyCodecs.suffixed(".json")(store) # maps '' -> '.json' + with pytest.raises(ConfigurationError, match="directory structure"): + s3dol.prefixes(wrapped) + + +def test_free_functions_refuse_a_hand_rolled_non_store_layer(): + # D3a's documented hole: a passthrough __getattr__ layer breaks the walk. + # Refuse rather than return a plausible wrong key. + class Passthrough: + def __init__(self, store): + self.store = store + + def __getattr__(self, name): + return getattr(self.store, name) + + layered = Passthrough(mock_s3(data={"p/a": b"1"}, prefix="p")) + with pytest.raises(ConfigurationError, match="refuse rather than guess"): + s3dol.prefixes(layered) + + +# --------------------------------------------------------------------------- # +# D4 — destructive operations are free functions +# --------------------------------------------------------------------------- # + + +def test_delete_many_resolves_the_callers_keys_through_the_chain(): + store = mock_s3(data={"p/x/a": b"1", "p/x/b": b"2", "p/other": b"3"}, prefix="p") + wrapped = KeyCodecs.prefixed("x/")(store) + s3dol.delete_many(wrapped, ["a", "b"]) + assert sorted(store._backing.data) == ["p/other"] # nothing outside scope + + +def test_delete_many_on_an_unwrapped_store(): + store = mock_s3(data={"p/a": b"1", "p/b": b"2"}, prefix="p") + s3dol.delete_many(store, ["a"]) + assert sorted(store) == ["b"] + + +def test_delete_many_is_idempotent_for_absent_keys(): + # S3 reports absent keys as Deleted; delete_many does not pretend otherwise. + store = mock_s3(data={"a": b"1"}) + s3dol.delete_many(store, ["a", "never-existed"]) + assert list(store) == [] + + +def test_delete_many_chunks_at_a_thousand(): + from s3dol.capabilities import DELETE_MANY_CHUNK + + assert DELETE_MANY_CHUNK == 1000 # AWS's cap; moto accepts 1001 + calls = [] + + class Counting(type(mock_s3())): + def _op_bulk_delete(self, ids): + calls.append(len(ids)) + return super()._op_bulk_delete(ids) + + from s3dol.testing import _FakeBackend + + data = {f"k{i}": b"v" for i in range(2500)} + store = Counting("mock-bucket", backing=_FakeBackend(data)) + s3dol.delete_many(store, list(data)) + assert calls == [1000, 1000, 500] + + +def test_delete_many_partial_failure_carries_the_split(): + from s3dol.testing import FakeBucketStore, _FakeBackend + + class Failing(FakeBucketStore): + def _op_bulk_delete(self, ids): + return [ids[0]], [(ids[1], "AccessDenied", "nope")] + + store = Failing("b", backing=_FakeBackend({"a": b"1", "b": b"2"})) + with pytest.raises(S3PartialFailure) as info: + s3dol.delete_many(store, ["a", "b"]) + assert info.value.succeeded == ["a"] and set(info.value.failures) == {"b"} + + +def test_partial_failure_reports_keys_in_the_callers_key_space(): + # "Keys embedded in an exception payload" is one of the shapes a codec + # boundary cannot map (the raise site has no boundary frame) — but a free + # function holds the store, so the inverse walk IS available here. + from s3dol.testing import FakeBucketStore, _FakeBackend + + class Failing(FakeBucketStore): + def _op_bulk_delete(self, ids): + return [], [(ids[0], "AccessDenied", "nope")] + + store = Failing("b", backing=_FakeBackend({"p/x/a": b"1"}), prefix="p") + wrapped = KeyCodecs.prefixed("x/")(store) + with pytest.raises(S3PartialFailure) as info: + s3dol.delete_many(wrapped, ["a"]) + # 'a' — the key the caller passed — not the wire key 'p/x/a' + assert set(info.value.failures) == {"a"} + + +def test_delete_many_emulates_when_batch_delete_is_unsupported(): + # GCS and any provider that rejects the mandatory DeleteObjects checksum: + # loop DeleteObject; identical observable result, only cost differs. + from s3dol.connection import S3Connection + from s3dol.testing import FakeBucketStore, _FakeBackend + + store = FakeBucketStore( + "b", + backing=_FakeBackend({"a": b"1", "b": b"2"}), + connection=S3Connection(endpoint_url="https://storage.googleapis.com"), + ) + assert store._resolution().capabilities.batch_delete is False + s3dol.delete_many(store, ["a", "b"]) + assert list(store) == [] + + +def test_delete_many_refuses_a_foreign_first_argument(): + # The honest caveat: a free function accepts anything — so validate. + with pytest.raises(ConfigurationError): + s3dol.delete_many({"a": b"1"}, ["a"]) + + +def test_delete_many_refuses_anonymous_writes(): + with pytest.raises(CredentialsError): + s3dol.delete_many(mock_s3(data={"a": b"1"}, anon=True), ["a"]) + + +def test_delete_bucket_requires_an_endpoint_store(): + with pytest.raises(ConfigurationError, match="delete_bucket"): + s3dol.delete_bucket(mock_s3(), "b", force=True) diff --git a/s3dol/tests/test_conformance.py b/s3dol/tests/test_conformance.py new file mode 100644 index 0000000..3dab992 --- /dev/null +++ b/s3dol/tests/test_conformance.py @@ -0,0 +1,114 @@ +"""The exported conformance suite, run against the fake, the real store +(moto), and a shipped recipe — the ADR-0008 drift gate: the fake and the real +store pass the SAME laws, so divergence fails here.""" + +import json + +import pytest + +from s3dol.testing import FakeBucketStore, _FakeBackend, mock_s3, run_conformance + +moto = pytest.importorskip("moto") + + +@pytest.fixture +def _mock_aws_env(monkeypatch): + for var in ("AWS_ENDPOINT_URL", "AWS_ENDPOINT_URL_S3", "AWS_PROFILE"): + monkeypatch.delenv(var, raising=False) + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "testing") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "testing") + monkeypatch.setenv("AWS_DEFAULT_REGION", "us-east-1") + + +def test_conformance_fake(): + run_conformance(lambda: FakeBucketStore("mock-bucket", backing=_FakeBackend())) + + +def test_conformance_fake_with_prefix(): + run_conformance( + lambda: FakeBucketStore( + "mock-bucket", backing=_FakeBackend(), prefix="tenant-a" + ) + ) + + +def test_conformance_real_store(_mock_aws_env): + from s3dol.base import BucketStore + from s3dol.connection import S3Connection + + with moto.mock_aws(): + connection = S3Connection(region_name="us-east-1") + connection.client.create_bucket(Bucket="conf-bucket") + run_conformance( + lambda: BucketStore("conf-bucket", connection=connection), + check_pickle=False, # moto's patch does not survive a real + # process hop; pickling itself is covered in test_base_moto + ) + + +def test_conformance_real_store_with_prefix(_mock_aws_env): + from s3dol.base import BucketStore + from s3dol.connection import S3Connection + + with moto.mock_aws(): + connection = S3Connection(region_name="us-east-1") + connection.client.create_bucket(Bucket="conf-bucket") + run_conformance( + lambda: BucketStore("conf-bucket", connection=connection, prefix="p"), + check_pickle=False, + ) + + +def test_conformance_json_recipe(_mock_aws_env): + # Layer C: the value law's normalize is json.loads∘json.dumps (N3 — + # (1,2) -> [1,2]); values are py objects so the str-rejection and url + # sections do not apply verbatim. + from s3dol.connection import S3Connection + from s3dol.recipes import S3Jsons + + with moto.mock_aws(): + connection = S3Connection(region_name="us-east-1") + connection.client.create_bucket(Bucket="json-bucket") + store = S3Jsons("json-bucket", connection=connection) + store["a"] = {"x": 1, "t": (1, 2)} + assert store["a"] == json.loads(json.dumps({"x": 1, "t": (1, 2)})) + assert store["a"]["t"] == [1, 2] # the honest N3 invariant + + +def test_shipped_recipe_encoders_produce_bytes(): + # ADR-0008 assertion 9: every shipped recipe's encoder output is a + # BytesSource — this is what catches ValueCodecs.json() returning str. + import pickle as pickle_module + + encoders = { + "S3Texts": lambda s: s.encode(), + "S3Jsons": lambda obj: json.dumps(obj).encode(), + "S3Pickles": pickle_module.dumps, + } + samples = {"S3Texts": "text", "S3Jsons": {"a": 1}, "S3Pickles": {"a": 1}} + for name, encode in encoders.items(): + assert isinstance(encode(samples[name]), bytes), name + + +def test_recipe_wire_format_is_plain_bytes(_mock_aws_env): + # A codec must not change the wire format: re-read through a raw store. + from s3dol.base import BucketStore + from s3dol.connection import S3Connection + from s3dol.recipes import S3Jsons + + with moto.mock_aws(): + connection = S3Connection(region_name="us-east-1") + connection.client.create_bucket(Bucket="wire-bucket") + S3Jsons("wire-bucket", connection=connection)["k"] = {"n": 5} + raw = BucketStore("wire-bucket", connection=connection)["k"] + assert json.loads(raw) == {"n": 5} + + +def test_conformance_catches_a_broken_store(): + # The suite must actually bite: a store that lies about membership fails. + class Lying(FakeBucketStore): + def __contains__(self, k): + return False + + with pytest.raises(AssertionError): + run_conformance(lambda: Lying("mock-bucket", backing=_FakeBackend())) diff --git a/s3dol/tests/test_connection.py b/s3dol/tests/test_connection.py index 34fc75c..2f10262 100644 --- a/s3dol/tests/test_connection.py +++ b/s3dol/tests/test_connection.py @@ -25,6 +25,8 @@ AmbiguousResolution, ConfigurationError, MissingEndpoint, + MissingPresetParam, + PresetConflict, PresetHostMismatch, ) from s3dol.presets import get_preset @@ -307,9 +309,14 @@ def test_c1_requires_endpoint_raises_missing_endpoint(): def test_c1_unbound_template_raises_and_names_the_params(): - with pytest.raises(MissingEndpoint) as info: + # MissingPresetParam names this case exactly, and IS a MissingEndpoint — + # so either `except` works and neither class is dead (ADR-0004's own + # indictment of v0 was five never-raised exception classes). + with pytest.raises(MissingPresetParam) as info: resolve(S3Connection(preset="r2"), {}) assert "account_id" in str(info.value) + with pytest.raises(MissingEndpoint): + resolve(S3Connection(preset="r2"), {}) def test_c1_env_endpoint_satisfies_a_requiring_preset(): @@ -469,7 +476,12 @@ def test_load_aws_config_is_credential_scrubbed(): # tier 2 (moto): the built client — SigV4 presigning is issue #10's regression # --------------------------------------------------------------------------- # -moto = pytest.importorskip("moto") +@pytest.fixture +def moto(): + """moto, per-test — a module-level importorskip would skip this file's + ~60 pure tier-1 tests (credential shapes, pickling, the lock race, every + resolve() ladder) along with the two that actually need it.""" + return pytest.importorskip("moto") @pytest.fixture @@ -481,7 +493,7 @@ def _mock_aws_env(monkeypatch): monkeypatch.setenv("AWS_DEFAULT_REGION", "us-east-1") -def test_client_presigns_sigv4_even_in_us_east_1(_mock_aws_env): +def test_client_presigns_sigv4_even_in_us_east_1(moto, _mock_aws_env): # us-east-1 is precisely a region where botocore, left to itself, silently # downgrades presigning to SigV2 (ADR-0003 §4). Assert on the URL, never on # client.meta.config.signature_version — it reports 's3v4' while emitting @@ -498,7 +510,7 @@ def test_client_presigns_sigv4_even_in_us_east_1(_mock_aws_env): assert "AWSAccessKeyId" not in url # the SigV2 tell -def test_anon_client_is_unsigned_and_still_builds(_mock_aws_env): +def test_anon_client_is_unsigned_and_still_builds(moto, _mock_aws_env): from botocore import UNSIGNED with moto.mock_aws(): @@ -506,7 +518,7 @@ def test_anon_client_is_unsigned_and_still_builds(_mock_aws_env): assert spec.client.meta.config.signature_version is UNSIGNED -def test_static_credentials_reach_the_request(_mock_aws_env): +def test_static_credentials_reach_the_request(moto, _mock_aws_env): with moto.mock_aws(): spec = S3Connection( credentials=("AKIAINJECTED", "sEkR3t"), region_name="us-east-1" @@ -517,3 +529,174 @@ def test_static_credentials_reach_the_request(_mock_aws_env): "get_object", Params={"Bucket": "cred-bucket", "Key": "k"}, ExpiresIn=60 ) assert "AKIAINJECTED" in url # the injected identity signed, not the env one + + +# --------------------------------------------------------------------------- # +# branches the first pass left untested (found by adversarial review) +# --------------------------------------------------------------------------- # + + +def test_pinned_preset_conflict_raises(): + # The ratchet of ADR-0012 D4: pinning is dormant while rows are + # doc-sourced, and turns on with evidence. Exercise the branch that + # consumes it, or it can be broken before any row is ever verified. + from s3dol.presets import Preset, register, unregister + + register( + Preset( + name="verified-row", + addressing_style="path", + endpoint_url="https://verified.example", + region_name="eu-west-9", + verified=True, + ) + ) + try: + with pytest.raises(PresetConflict, match="eu-west-9"): + resolve(S3Connection(preset="verified-row", region_name="us-east-1"), {}) + # ...and agreeing is fine + assert ( + resolve( + S3Connection(preset="verified-row", region_name="eu-west-9"), {} + ).region_name.value + == "eu-west-9" + ) + finally: + unregister("verified-row") + + +def test_notes_are_actually_emitted_as_warnings_at_client_build(monkeypatch): + # resolve() only COLLECTS notes; _build_client is the sole user-visible + # channel, and filterwarnings over it is the documented strict mode. + # Without this test, deleting the warn loop leaves the suite green. + monkeypatch.setenv("AWS_ENDPOINT_URL_S3", "https://proxy.mycorp.internal") + with pytest.warns(AmbiguousResolution, match="host patterns"): + S3Connection(preset="r2").client + + +def test_filterwarnings_is_strict_mode(monkeypatch): + import warnings + + from s3dol.errors import S3DolWarning + + monkeypatch.setenv("AWS_ENDPOINT_URL_S3", "https://proxy.mycorp.internal") + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + warnings.filterwarnings("error", category=S3DolWarning) + with pytest.raises(AmbiguousResolution): + S3Connection(preset="r2").client + + +def test_verify_and_client_kwargs_reach_the_built_client(monkeypatch): + # verify= is a connection-security field: dropping it would silently lose + # a self-signed-MinIO user's CA setting, failing OPEN. + captured = {} + + class FakeSession: + def __init__(self, botocore_session=None): + pass + + def client(self, service, **kwargs): + captured.update(kwargs) + return object() + + import boto3 + + monkeypatch.setattr(boto3, "Session", FakeSession) + S3Connection( + endpoint_url="https://minio.internal:9000", + verify="/etc/ssl/corp-ca.pem", + client_kwargs={"use_ssl": True}, + ).client + assert captured["verify"] == "/etc/ssl/corp-ca.pem" + assert captured["use_ssl"] is True + assert captured["endpoint_url"] == "https://minio.internal:9000" + + +def test_signature_version_none_is_refused_not_silently_sigv2(): + # 'None == do not override' is the convention of its NEIGHBOURS; for this + # field it would re-enable the SigV2 downgrade (issue #10) silently. + with pytest.raises(ConfigurationError, match="SigV2"): + S3Connection(signature_version=None) + + +def test_client_kwargs_cannot_smuggle_credentials(): + with pytest.raises(ConfigurationError, match="credentials="): + S3Connection(client_kwargs={"aws_secret_access_key": "sEkR3t"}) + with pytest.raises(ConfigurationError): + S3Connection(client_kwargs={"config": object()}) + + +@pytest.mark.parametrize( + "shape", + [ + lambda: S3Connection(client_kwargs={"aws_secret_access_key": "sEkR3tVALUE"}), + lambda: S3Connection(credentials=("AKIAX", "sEkR3tVALUE")), + ], +) +def test_no_credential_shape_can_leak_through_repr(shape): + try: + spec = shape() + except ConfigurationError: + return # refused outright: even better + assert "sEkR3tVALUE" not in repr(spec) + + +def test_env_endpoint_userinfo_never_survives_into_messages_or_repr(): + secret_url = "https://user:sEkR3tVALUE@minio.internal:9000" + resolution = resolve(S3Connection(), {"AWS_ENDPOINT_URL_S3": secret_url}) + assert "sEkR3tVALUE" not in repr(resolution) + assert "minio.internal" in repr(resolution) # the host survives + with_note = resolve( + S3Connection(preset="wasabi"), {"AWS_ENDPOINT_URL_S3": secret_url} + ) + assert all("sEkR3tVALUE" not in message for _, message in with_note.notes) + + +def test_tuple_credentials_with_a_none_token_behave_like_the_pair(): + assert normalize_credentials(("AK", "SK", None)) == normalize_credentials( + ("AK", "SK") + ) + + +def test_callable_credentials_fetch_exactly_once_on_first_load(): + # A token vendor may be rate-limited or single-use. + calls = [] + + def counting_fetch(): + calls.append(1) + return { + "access_key": "a", + "secret_key": "b", + "expires_at": "2030-01-01T00:00:00Z", + } + + CallableCredentials(counting_fetch)._load_botocore_credentials() + assert len(calls) == 1 + + +def test_region_capture_does_not_hijack_an_explicit_env_region(): + for host in ( + "https://eu-storage.mycorp.com", + "https://srv1.corp.example", + "https://my-storage.example.com", + ): + resolution = resolve( + S3Connection(endpoint_url=host), {"AWS_DEFAULT_REGION": "eu-west-1"} + ) + assert resolution.region_name.value == "eu-west-1", host + # ...while the canonical shape is still captured + assert ( + resolve( + S3Connection(endpoint_url="https://s3.eu-central-2.amazonaws.com"), {} + ).region_name.value + == "eu-central-2" + ) + + +def test_c2_guard_applies_to_a_named_soft_row_too(): + with pytest.raises(PresetHostMismatch): + resolve( + S3Connection(preset="aws"), + {"AWS_ENDPOINT_URL_S3": "https://s3.eu-central-1.wasabisys.com"}, + ) diff --git a/s3dol/tests/test_diagnose.py b/s3dol/tests/test_diagnose.py index 402a156..ce57c2a 100644 --- a/s3dol/tests/test_diagnose.py +++ b/s3dol/tests/test_diagnose.py @@ -6,8 +6,12 @@ def run_diagnose(**kwargs): - """diagnose() against a controlled environment, capturing the report.""" + """diagnose() against a controlled environment AND a controlled aws + config, capturing the report. Injecting both is what makes these tier 1: + otherwise every test silently consults the developer's own ~/.aws/config + and goes red on a machine whose profile sets an endpoint_url.""" kwargs.setdefault("environ", {}) + kwargs.setdefault("aws_config", {}) return diagnose(file=io.StringIO(), **kwargs) @@ -53,10 +57,19 @@ def test_secret_values_never_echo(): def test_prints_to_the_given_file_and_returns_the_text(): sink = io.StringIO() - text = diagnose(endpoint_url="http://x:1", environ={}, file=sink) + text = diagnose(endpoint_url="http://x:1", environ={}, aws_config={}, file=sink) assert sink.getvalue() == text +def test_a_configured_aws_endpoint_does_not_bleed_into_a_what_if_run(): + # The injected config is the ONLY config consulted. + report = run_diagnose( + preset="minio", aws_config={"profiles": {"default": {"endpoint_url": "http://cfg:1"}}} + ) + assert "http://cfg:1" in report and "RESOLUTION FAILS" not in report + assert "RESOLUTION FAILS" in run_diagnose(preset="minio") # nothing supplies one + + # --------------------------------------------------------------------------- # # the v0 mangler (D7): ~20 frozen lines mapping 1:1 onto v0 # --------------------------------------------------------------------------- # @@ -129,6 +142,76 @@ def test_excluded_branch_prints_the_reason(): assert "process-global" in report +def test_explicit_credentials_overridden_by_env_is_reported_as_a_move(): + # THE case step 0 exists for: v0 let env credentials silently override + # explicit ones. Comparing provenance LABELS missed it (both truncate to + # 'AKIA…'); the identity predicate catches it. + report = run_diagnose( + bucket_name="b", + aws_access_key_id="AKIAEXPLICIT", + aws_secret_access_key="explicit-secret", + environ={ + "AWS_ACCESS_KEY_ID": "AKIAFROMENV", + "AWS_SECRET_ACCESS_KEY": "env-secret", + }, + ) + assert "credential identity" in report + assert "CHANGES ON UPGRADE" in report + + +def test_env_only_credentials_are_not_a_false_alarm(): + # The most common v0 configuration: env credentials, no explicit kwargs. + # Both sides resolve to the same keys, so the table must say so — a + # migration signal that cries wolf gets filtered away. + report = run_diagnose( + bucket_name="b", + endpoint_url="http://minio:9000", + environ={"AWS_ACCESS_KEY_ID": "AKIAENV", "AWS_SECRET_ACCESS_KEY": "s"}, + ) + assert "credential identity" not in report + + +def test_detection_supplied_settings_are_announced_as_new_in_v1(): + # Both sides run through v1's resolver, so pass-2 detection cancels out of + # any diff — it has to be reported explicitly or the announcement + # instrument cannot see the path->virtual flip it exists to announce. + report = run_diagnose( + bucket_name="b", endpoint_url="https://s3.us-west-004.backblazeb2.com" + ) + assert "NEW IN v1" in report + assert "backblaze" in report and "when_required" in report + + +def test_env_profile_without_env_credentials_names_the_v0_raise(): + report = run_diagnose(bucket_name="b", profile_name="environment variables") + assert "v0 RAISES" in report + assert "EXCLUDED" in report + + +def test_v1_credential_kwargs_are_never_echoed(): + # The allowlist, not a denylist of two v0 names: every v1 shape must be + # covered, in the one artifact users are told to run and paste into issues. + for kwargs in ( + {"credentials": ("AKIAX", "sEkR3tVALUE")}, + {"credentials": {"aws_secret_access_key": "sEkR3tVALUE"}}, + {"client_kwargs": {"aws_secret_access_key": "sEkR3tVALUE"}}, + ): + assert "sEkR3tVALUE" not in run_diagnose(**kwargs) + + +def test_the_access_key_id_is_shown_by_prefix_only(): + report = run_diagnose(bucket_name="b", aws_access_key_id="AKIAFULLKEYVALUE") + assert "AKIAFULLKEYVALUE" not in report + assert "AKIA" in report + + +def test_an_endpoint_with_userinfo_is_redacted_in_the_echo(): + report = run_diagnose( + bucket_name="b", endpoint_url="https://user:sEkR3tVALUE@host:9000" + ) + assert "sEkR3tVALUE" not in report + + def test_credential_identity_divergence_is_a_predicate_not_a_value(): report = run_diagnose( bucket_name="b", diff --git a/s3dol/tests/test_keyed_surface.py b/s3dol/tests/test_keyed_surface.py new file mode 100644 index 0000000..5ae1e94 --- /dev/null +++ b/s3dol/tests/test_keyed_surface.py @@ -0,0 +1,153 @@ +"""The ADR-0011 D5 reflective conformance test: no keyed methods, enforced. + +Not a registry — a registry fails silently when an author forgets to declare a +method. This test enumerates **every public attribute of every Layer B class** +against an explicit inventory and fails on any *new* one, forcing the +"does this take anything that reaches the wire as a key?" judgement at review +time. (A signature predicate cannot make that judgement: ``delete(name)``, +``delete_many(keys)``, ``batch(operations)`` and ``sync_to(target)`` all hide +keys in innocently-named parameters.) + +Allowlist of keyed methods: ``url_for`` (the guarded ``dol.SupportsUrlFor`` +shim, ADR-0011 D3b) — and NOTHING else. To add a capability: +sibling store (D2), ``ObjectHandle`` (key bound at construction), or a free +function taking the store first (D3). Do not grow this inventory with a keyed +method; the comment you are reading is the review gate. +""" + +import pytest + +from s3dol.base import ( + BucketCollection, + BucketReader, + BucketStore, + EndpointCollection, + EndpointReader, + EndpointStore, + S3Profiles, +) +from s3dol.capabilities import BucketHandles, BucketInfo, BucketUrls + +#: The ONLY method allowed to take a key (guarded; see ADR-0011 D3b). +KEYED_METHOD_ALLOWLIST = {"url_for"} + +#: Mapping-protocol members `dol` maps correctly at every wrapper depth — +#: the sanctioned keyed surface. +MAPPING_PROTOCOL = { + "__getitem__", + "__setitem__", + "__delitem__", + "__contains__", + "__iter__", + "__len__", + "__eq__", + "__ne__", + "get", + "keys", + "values", + "items", + "pop", + "popitem", + "clear", + "update", + "setdefault", + "head", # dol.KvReader convenience (first item; not key-taking) +} + +#: The full public (non-dunder, non-Mapping) inventory, per class. A test +#: failure here means the public surface changed: review the new attribute +#: against ADR-0011 D1 before extending the inventory. +PUBLIC_INVENTORY = { + BucketCollection: {"bucket", "connection", "delimiter", "prefix", "reads", + "writes", "on_missing_bucket", "strict_delete", "client", + "KeysView", "ValuesView", "ItemsView"}, + BucketReader: {"url_for"}, + BucketStore: set(), + BucketHandles: set(), + BucketUrls: {"expires_in", "client_method"}, + BucketInfo: set(), + EndpointCollection: {"connection", "bucket_kwargs", "client", + "KeysView", "ValuesView", "ItemsView"}, + EndpointReader: set(), + EndpointStore: set(), + S3Profiles: {"bucket_kwargs", "readonly", "KeysView", "ValuesView", + "ItemsView"}, +} + +#: Methods among the inventory that take a key-like argument. Must stay equal +#: to the allowlist. +KEYED_IN_INVENTORY = {"url_for"} + + +def _public_surface(cls) -> set: + """Every public attribute the s3dol classes in ``cls``'s MRO define, + minus the Mapping protocol.""" + names = set() + for klass in cls.__mro__: + if klass.__module__.startswith("s3dol"): + names |= set(vars(klass)) + return { + name + for name in names + if not name.startswith("_") and name not in MAPPING_PROTOCOL + } + + +@pytest.mark.parametrize("cls", list(PUBLIC_INVENTORY), ids=lambda c: c.__name__) +def test_public_surface_matches_the_inventory(cls): + surface = _public_surface(cls) + all_declared = set() + for klass, names in PUBLIC_INVENTORY.items(): + if klass in cls.__mro__: + all_declared |= names + unexpected = surface - all_declared + assert not unexpected, ( + f"{cls.__name__} grew public attribute(s) {sorted(unexpected)} not in " + f"the ADR-0011 D5 inventory. If any takes something that reaches the " + f"wire as a key — at any argument position, or inside a structure — " + f"it must become a sibling store, an ObjectHandle method, or a free " + f"function. Otherwise, add it to PUBLIC_INVENTORY with a reviewer's " + f"eyes on this message." + ) + + +def test_the_keyed_allowlist_is_exactly_url_for(): + assert KEYED_IN_INVENTORY == KEYED_METHOD_ALLOWLIST == {"url_for"} + + +def test_url_for_exists_only_where_declared(): + # BucketReader (and its subclasses) only — never the endpoint level. + assert "url_for" in vars(BucketReader) + for cls in (BucketCollection, EndpointCollection, EndpointReader, EndpointStore): + assert "url_for" not in vars(cls) + + +def test_object_handle_binds_its_key_at_construction(): + # The handle may be method-rich precisely because no method takes a key. + import inspect + + from s3dol.base import ObjectHandle + + for name, member in vars(ObjectHandle).items(): + if name.startswith("_") or not callable(member): + continue + params = list(inspect.signature(member).parameters)[1:] # drop self + assert "k" not in params and "key" not in params, ( + f"ObjectHandle.{name} takes a key parameter — the handle's " + f"immunity to the delegation trap rests on the key being bound " + f"at construction (ADR-0011 D1)." + ) + + +def test_free_functions_take_the_store_first(): + import inspect + + import s3dol.capabilities as capabilities + + for name in ("sub", "prefixes", "delete_many", "handles", "urls", "info"): + first = next(iter(inspect.signature(getattr(capabilities, name)).parameters)) + assert first in ("store", "endpoint"), (name, first) + first = next( + iter(inspect.signature(capabilities.delete_bucket).parameters) + ) + assert first == "endpoint" diff --git a/s3dol/tests/test_store.py b/s3dol/tests/test_store.py deleted file mode 100644 index ad3ac65..0000000 --- a/s3dol/tests/test_store.py +++ /dev/null @@ -1,47 +0,0 @@ -import pytest -from s3dol.base import S3ClientDol - -from s3dol.store import S3Store -from s3dol.tests.test_base import assert_bucket_key_value -from s3dol.utility import S3DolException - - -def setup_test_bucket(aws_access_key_id, aws_secret_access_key, endpoint_url): - s3_client = S3ClientDol( - aws_access_key_id=aws_access_key_id, - aws_secret_access_key=aws_secret_access_key, - endpoint_url=endpoint_url, - ) - if "test-bucket" not in s3_client: - s3_client["test-bucket"] = {} - - if "level1/level2/test-key" in s3_client["test-bucket"]: - del s3_client["test-bucket"]["level1/level2/test-key"] - - -@pytest.mark.parametrize( - "aws_access_key_id, aws_secret_access_key, endpoint_url", - [("localstack", "localstack", "http://localhost:4566")], -) -def test_s3_store_crud(aws_access_key_id, aws_secret_access_key, endpoint_url): - setup_test_bucket(aws_access_key_id, aws_secret_access_key, endpoint_url) - - s3_store = S3Store( - bucket_name="test-bucket", - aws_access_key_id=aws_access_key_id, - aws_secret_access_key=aws_secret_access_key, - endpoint_url=endpoint_url, - ) - test_value = b"test-store-value" - s3_store["level1/level2/test-key"] = test_value - assert_bucket_key_value(s3_store, "level1/level2/test-key", test_value) - assert_bucket_key_value(s3_store["level1/level2/"], "test-key", test_value) - assert_bucket_key_value(s3_store["level1/"]["level2/"], "test-key", test_value) - assert_bucket_key_value(s3_store["level1/"], "level2/test-key", test_value) - - del s3_store["level1/"]["level2/"]["test-key"] - assert "level1/level2/test-key" not in s3_store - - -if __name__ == "__main__": - pytest.main([__file__]) diff --git a/s3dol/tests/test_store_shim.py b/s3dol/tests/test_store_shim.py new file mode 100644 index 0000000..41c251f --- /dev/null +++ b/s3dol/tests/test_store_shim.py @@ -0,0 +1,237 @@ +"""The v0 ``S3Store`` compat shim (ADR-0007 §3/§4, ADR-0012 D7). + +The shim's justification is "existing users keep working", so these tests are +written from the dependents' *actual* usage: `lacing` calls +``S3Store(bucket, path=prefix, **kw)`` positionally and passes +``make_bucket=True``; `http_cosmo_prep` passes the bucket by keyword with an +explicit ``endpoint_url``, writes ``str`` values, uses ``store['folder/']`` as +a sub-store, and relies on ``del`` being a silent no-op. +""" + +import inspect +import warnings + +import pytest + +from s3dol.errors import S3DolResolutionChanged +from s3dol.store import ( + S3Store, + _V0CompatStore, + dechunk_aws_chunked, + detect_chunked_framing, +) +from s3dol.testing import _FakeBackend, mock_s3 + + +def compat(data=None, prefix="", dechunk=False): + """A shim store over the in-memory fake (no network).""" + return _V0CompatStore(mock_s3(data=data, prefix=prefix), dechunk=dechunk) + + +# --------------------------------------------------------------------------- # +# the signature must not move (both dependents import s3dol.store.S3Store) +# --------------------------------------------------------------------------- # + + +def test_module_survives_as_an_importable_path(): + import s3dol.store # noqa: F401 — the fully-qualified import both use + + assert callable(S3Store) + + +def test_signature_is_v0s_exactly(): + parameters = inspect.signature(S3Store).parameters + assert list(parameters)[0] == "bucket_name" + assert parameters["bucket_name"].kind is inspect.Parameter.POSITIONAL_OR_KEYWORD + for name in ( + "make_bucket", + "path", # NOT renamed to prefix on the shim + "aws_access_key_id", + "aws_secret_access_key", + "aws_session_token", + "endpoint_url", + "region_name", + "profile_name", + "skip_bucket_check", + "is_supabase_endpoint", # v0 accepts it; omitting it = TypeError on upgrade + ): + assert name in parameters, name + + +def test_test_utils_still_importable_for_py2store(): + # py2store imports these under suppress(ImportError) — breakage is SILENT. + from s3dol.tests.util import ( # noqa: F401 + extract_s3_access_info, + get_s3_test_access_info_from_env_vars, + ) + + +def test_calling_it_warns_about_the_v2_removal(): + with pytest.warns(DeprecationWarning, match="s3_store"): + with pytest.raises(Exception): # no credentials/bucket here; the warning is the point + S3Store("some-bucket", make_bucket=False) + + +# --------------------------------------------------------------------------- # +# the three behaviours a dependent's PASSING tests rely on (ADR-0007 §4) +# --------------------------------------------------------------------------- # + + +def test_str_values_are_accepted_with_a_deprecation_warning(): + store = compat() + with pytest.warns(DeprecationWarning, match="utf-8"): + store["k"] = "a string" + assert store["k"] == b"a string" + + +def test_trailing_slash_returns_a_sub_store(): + store = compat(data={"level1/level2/k": b"v"}) + assert store["level1/level2/"]["k"] == b"v" + assert store["level1/"]["level2/"]["k"] == b"v" + assert store["level1/"]["level2/k"] == b"v" + + +def test_delete_of_an_absent_key_is_a_silent_no_op(): + store = compat(data={"a": b"1"}) + del store["a"] + del store["a"] # v0 relied on this; v1 deletes are idempotent anyway + + +def test_len_still_counts_on_the_shim(): + # The v1 store raises TypeError; the shim keeps v0's behaviour (and cost). + assert len(compat(data={"a": b"1", "b": b"2"})) == 2 + + +def test_url_for_is_forwarded_and_prefix_aware(): + store = compat(data={"blobs/abc": b"v"}, prefix="blobs") + url = store.url_for("abc") + assert "blobs/abc" in url + assert "X-Amz-Algorithm=AWS4-HMAC-SHA256" in url # the #10 fix + assert "AWSAccessKeyId" not in url # the SigV2 tell + + +def test_shim_iterates_and_contains_like_a_mapping(): + store = compat(data={"a": b"1", "b": b"2"}) + assert sorted(store) == ["a", "b"] + assert "a" in store and "zz" not in store + + +# --------------------------------------------------------------------------- # +# the corrected de-chunker (ADR-0003 §3) — v0's truncated multi-chunk payloads +# --------------------------------------------------------------------------- # + + +def test_dechunk_single_chunk(): + assert dechunk_aws_chunked(b"5\r\nhello\r\n0\r\n\r\n") == b"hello" + + +def test_dechunk_multi_chunk_where_v0_truncated(): + framed = b"3\r\nabc\r\n2\r\nde\r\n0\r\nx-amz-checksum-crc32:AAAA\r\n\r\n" + assert dechunk_aws_chunked(framed) == b"abcde" + + +@pytest.mark.parametrize( + "payload", + [ + b"cafe is hex-ish but not framed", # v0 misfired on this shape + b"0123456789", + b"", + b"deadbeef", + b"5\r\nhello", # truncated framing: passthrough, never a partial parse + b"5\r\ntoo-long-for-its-header\r\n0\r\n\r\n", + ], +) +def test_dechunk_is_full_parse_or_passthrough(payload): + assert dechunk_aws_chunked(payload) == payload + + +def test_detect_chunked_framing_finds_the_corrupted_keys(): + store = mock_s3( + data={"good": b"plain bytes", "bad": b"5\r\nhello\r\n0\r\n\r\n"} + ) + assert detect_chunked_framing(store) == ["bad"] + + +def test_dechunking_store_repairs_reads_only_when_enabled(): + framed = b"5\r\nhello\r\n0\r\n\r\n" + assert compat(data={"k": framed}, dechunk=True)["k"] == b"hello" + assert compat(data={"k": framed})["k"] == framed # never a default + + +# --------------------------------------------------------------------------- # +# the D7 migration warning +# --------------------------------------------------------------------------- # + + +def test_warns_when_v1_resolves_differently_than_v0(monkeypatch): + # v0 DROPPED an explicit endpoint_url whenever env credentials existed — + # the store then talked to AWS. v1 honours it; that moves the data target, + # so it must announce itself. + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "AKIAENV") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "envsecret") + monkeypatch.delenv("AWS_ENDPOINT_URL", raising=False) + monkeypatch.delenv("AWS_ENDPOINT_URL_S3", raising=False) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + S3Store( + "b", + endpoint_url="https://minio.example:9000", + aws_access_key_id="AKIAEXPLICIT", + aws_secret_access_key="explicitsecret", + ) + messages = [str(w.message) for w in caught if w.category is S3DolResolutionChanged] + assert messages and "diagnose" in messages[0] + + +def test_no_warning_when_nothing_moves(monkeypatch): + for var in ("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_ENDPOINT_URL", + "AWS_ENDPOINT_URL_S3"): + monkeypatch.delenv(var, raising=False) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + S3Store("b", endpoint_url="https://minio.example:9000") + assert not [w for w in caught if w.category is S3DolResolutionChanged] + + +def test_strict_mode_is_filterwarnings_not_a_kwarg(monkeypatch): + # ADR-0012 D1: no strict= kwarg; filterwarnings('error', S3DolWarning) is + # strict mode, with zero API surface. + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "AKIAENV") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "envsecret") + monkeypatch.delenv("AWS_ENDPOINT_URL", raising=False) + monkeypatch.delenv("AWS_ENDPOINT_URL_S3", raising=False) + from s3dol.errors import S3DolWarning + + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + warnings.filterwarnings("error", category=S3DolWarning) + with pytest.raises(S3DolResolutionChanged): + S3Store( + "b", + endpoint_url="https://minio.example:9000", + aws_access_key_id="AKIAEXPLICIT", + aws_secret_access_key="explicitsecret", + ) + + +def test_make_bucket_tristate_maps_to_on_missing_bucket(): + from s3dol.base import BucketStore + + captured = {} + original = BucketStore.__init__ + + def spy(self, *args, **kwargs): + captured["on_missing_bucket"] = kwargs.get("on_missing_bucket") + captured["prefix"] = kwargs.get("prefix") + original(self, *args, **kwargs) + + BucketStore.__init__ = spy + try: + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + S3Store("b", make_bucket=True, path="logs") + assert captured == {"on_missing_bucket": "create", "prefix": "logs"} + S3Store("b", make_bucket=None) + assert captured["on_missing_bucket"] == "assume" + finally: + BucketStore.__init__ = original diff --git a/s3dol/tests/test_url_for.py b/s3dol/tests/test_url_for.py deleted file mode 100644 index 422f953..0000000 --- a/s3dol/tests/test_url_for.py +++ /dev/null @@ -1,61 +0,0 @@ -"""Presigned-URL (``url_for``) tests, mocked with moto. See issue #7. - -Unlike the env-gated real-S3 tests, these run anywhere — moto fully mocks the -S3 API including ``generate_presigned_url``. -""" - -import pytest - -pytest.importorskip("moto") -from moto import mock_aws - -from s3dol.store import S3Store - -_CREDS = dict( - aws_access_key_id="testing", - aws_secret_access_key="testing", - region_name="us-east-1", -) - - -@pytest.fixture(autouse=True) -def _isolate_aws_env(monkeypatch): - """Make these moto tests hermetic. The localstack tests (test_base) set - ``AWS_ENDPOINT_URL=http://localhost:4566`` via ``os.environ`` with no - cleanup, which would route boto3 past moto to a (down) localstack. Clear any - leaked endpoint + pin fake creds so ``@mock_aws`` always intercepts.""" - for var in ("AWS_ENDPOINT_URL", "AWS_ENDPOINT_URL_S3", "AWS_PROFILE"): - monkeypatch.delenv(var, raising=False) - monkeypatch.setenv("AWS_ACCESS_KEY_ID", "testing") - monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "testing") - monkeypatch.setenv("AWS_DEFAULT_REGION", "us-east-1") - - -@mock_aws -def test_url_for_returns_presigned_get_url(): - store = S3Store("test-bucket", make_bucket=True, **_CREDS) - store["hello.txt"] = b"hi" - url = store.url_for("hello.txt") - assert isinstance(url, str) - assert "test-bucket" in url - assert "hello.txt" in url - # Presigned marker (SigV4 X-Amz-Signature / SigV2 Signature). - assert "Signature" in url - - -@mock_aws -def test_url_for_is_prefix_aware(): - # path= sets a key prefix; url_for must point at the prefixed id, matching - # __getitem__ (else the URL 404s). - store = S3Store("test-bucket", make_bucket=True, path="blobs", **_CREDS) - store["abc"] = b"x" - url = store.url_for("abc") - assert "blobs" in url and "abc" in url - - -@mock_aws -def test_url_for_honors_expires_in(): - store = S3Store("test-bucket", make_bucket=True, **_CREDS) - store["k"] = b"v" - url = store.url_for("k", expires_in=60) - assert "X-Amz-Expires=60" in url or "Expires=" in url diff --git a/s3dol/tests/test_values.py b/s3dol/tests/test_values.py new file mode 100644 index 0000000..84a599d --- /dev/null +++ b/s3dol/tests/test_values.py @@ -0,0 +1,90 @@ +"""Tier-1 tests for the value refs and write domain (ADR-0005).""" + +import io +import pickle + +import pytest + +from s3dol.values import ( + Chunks, + Filepath, + Streamable, + as_fileobj, + is_bytes_like, + reject_str_value, +) + + +def test_str_is_rejected_with_the_two_option_message(): + with pytest.raises(TypeError) as info: + reject_str_value("/tmp/big.mp4") + message = str(info.value) + assert "encode()" in message and "Filepath" in message + + +def test_bare_pathlike_is_rejected_pointing_at_filepath(): + from pathlib import Path + + with pytest.raises(TypeError, match="Filepath"): + as_fileobj(Path("/tmp/x")) + + +def test_alien_type_names_the_extension_point(): + with pytest.raises(TypeError, match="register"): + as_fileobj(12345) + + +@pytest.mark.parametrize( + "value", [b"bytes", bytearray(b"ba"), memoryview(b"mv")] +) +def test_bytes_like_members(value): + assert is_bytes_like(value) + assert as_fileobj(value).read() == bytes(value) + + +def test_open_binary_files_dispatch_via_io_iobase(): + # The trap: @register(typing.BinaryIO) is accepted and never fires + # (io.BytesIO is not in its MRO — ADR-0005 §1). io.IOBase is the base + # that actually covers real file objects. + assert as_fileobj(io.BytesIO(b"data")).read() == b"data" + + +def test_filepath_normalises_pathlike_and_reads(tmp_path): + target = tmp_path / "payload.bin" + target.write_bytes(b"CONTENT") + ref = Filepath(target) # a Path is fine; normalised to str + assert isinstance(ref.path, str) + with as_fileobj(ref) as fileobj: + assert fileobj.read() == b"CONTENT" + # re-readable (N1-total): a second consumption works + with as_fileobj(ref) as fileobj: + assert fileobj.read() == b"CONTENT" + assert pickle.loads(pickle.dumps(ref)) == ref + + +def test_chunks_concatenates_and_is_one_shot(): + ref = Chunks(iter([b"ab", b"cd", b"e"])) + assert as_fileobj(ref).read() == b"abcde" + with pytest.raises(ValueError, match="one-shot"): + as_fileobj(ref) # the documented error, not a silent b'' write + + +def test_streamable_reopens_per_write(): + ref = Streamable(lambda: iter([b"xy", b"z"])) + assert as_fileobj(ref).read() == b"xyz" + assert as_fileobj(ref).read() == b"xyz" # fresh stream each time + + +def test_iterable_fileobj_partial_reads(): + fileobj = as_fileobj(Chunks(iter([b"abc", b"defg"]))) + assert fileobj.read(2) == b"ab" + assert fileobj.read(3) == b"cde" + assert fileobj.read() == b"fg" + + +def test_write_strategies_are_picklable_values(): + from s3dol.reads import BytesReads, StreamReads + from s3dol.writes import SimpleWrites, TransferWrites + + for strategy in (TransferWrites(), SimpleWrites(), BytesReads(), StreamReads()): + assert pickle.loads(pickle.dumps(strategy)) == strategy diff --git a/s3dol/utility.py b/s3dol/utility.py deleted file mode 100644 index dad5362..0000000 --- a/s3dol/utility.py +++ /dev/null @@ -1,127 +0,0 @@ -""" -This module provides utility functions and custom exceptions for the `s3dol` package. - -Classes: -- S3DolException: Base exception class for `s3dol` package. -- S3KeyError: Exception raised when a key error occurs. -- NoSuchKeyError: Exception raised when a key is not found. -- KeyNotValidError: Exception raised when a key is not valid. -- GetItemForKeyError: Exception raised when an error occurs while getting an item for a key. -- S3HttpError: Exception raised when an HTTP error occurs. -- S3Not200StatusCodeError: Exception raised when an HTTP status code other than 200 is returned. - -Functions: -- raise_on_error: Function that raises an error. -- return_none_on_error: Function that returns None. -- return_empty_tuple_on_error: Function that returns an empty tuple. -- Resp.status_code: Function that returns the HTTP status code from a dictionary. -- Resp.contents: Function that returns the contents from a dictionary. -- Resp.key: Function that returns the key from a dictionary. -- Resp.common_prefixes: Function that returns the common prefixes from a dictionary. -- Resp.prefix: Function that returns the prefix from a dictionary. -- Resp.buckets: Function that returns the buckets from a dictionary. -- Resp.body: Function that returns the body from a dictionary. -- Resp.ascertain_status_code: Function that raises an error if the HTTP status code is not 200. -- Resp.ascertain_200_status_code: Function that raises an error if the HTTP status code is not 200 and the response is not a dictionary with a 'ResponseMetadata' key. -""" - -"""""" -from dol.paths import path_get, OnErrorType - - -class S3DolException(Exception): ... - - -class S3KeyError(KeyError): ... - - -class NoSuchKeyError(S3KeyError): ... - - -class KeyNotValidError(S3KeyError): ... - - -class GetItemForKeyError(S3KeyError): ... - - -class S3HttpError(RuntimeError): ... - - -class S3Not200StatusCodeError(S3HttpError): ... - - -raise_on_error: OnErrorType -return_none_on_error: OnErrorType -return_empty_tuple_on_error: OnErrorType - - -def raise_on_error(d: dict): - raise - - -def return_none_on_error(d: dict): - return None - - -def return_empty_tuple_on_error(d: dict): - return () - - -class Resp: - @staticmethod - def status_code(d, on_error: OnErrorType = raise_on_error): - return path_get(d, ["ResponseMetadata", "HTTPStatusCode"], on_error) - - @staticmethod - def contents(d, on_error: OnErrorType = return_empty_tuple_on_error): - return path_get(d, ["Contents"], on_error) - - @staticmethod - def key(d, on_error: OnErrorType = raise_on_error): - return path_get(d, ["Key"], on_error) - - @staticmethod - def common_prefixes(d, on_error: OnErrorType = return_empty_tuple_on_error): - return path_get(d, ["CommonPrefixes"], on_error) - - @staticmethod - def prefix(d, on_error: OnErrorType = raise_on_error): - return path_get(d, ["Prefix"], on_error) - - @staticmethod - def buckets(d, on_error: OnErrorType = return_empty_tuple_on_error): - return path_get(d, ["Buckets"], on_error) - - @staticmethod - def body(d, on_error: OnErrorType = return_none_on_error): - return path_get(d, ["Body"], on_error) - - @staticmethod - def ascertain_status_code( - d, - status_code=200, - raise_error=S3HttpError, - *error_args, - **error_kwargs, - ): - if Resp.status_code(d) != status_code: - raise raise_error(*error_args, **error_kwargs) - - @staticmethod - def ascertain_200_status_code(d): - status_code = Resp.status_code(d, return_none_on_error) - if status_code != 200: - if not isinstance(d, dict): - raise S3Not200StatusCodeError( - "Yeah, that's not even a dict, so doubt it's even a response." - f"I'm expecting a response over here. Instead I got a {type(d)}" - ) - elif "ResponseMetadata" in d: - raise S3Not200StatusCodeError( - f"Status code was not 200. Was {d['ResponseMetadata']}. " - f"ResponseMetadata is {d['ResponseMetadata']}" - ) - else: - raise S3Not200StatusCodeError( - f"Status code was not 200. In fact, the response dict didn't even have a ResponseMetadata key" - ) diff --git a/s3dol/values.py b/s3dol/values.py new file mode 100644 index 0000000..7a7a45d --- /dev/null +++ b/s3dol/values.py @@ -0,0 +1,204 @@ +"""Value refs and the write domain (ADR-0005). + +The write domain is a small, closed, explicitly-named union:: + + BytesSource = bytes | bytearray | memoryview | BinaryIO | Filepath | Chunks | Streamable + +``Filepath``/``Chunks``/``Streamable`` are tiny frozen dataclasses — value +*refs*, not values (h5py's ``SoftLink``/``ExternalLink`` are the precedent: +purpose-built objects meaning "the value is a reference to content elsewhere", +assigned through an ordinary ``__setitem__``). + +The law that makes the asymmetric read/write types safe (ADR-0005 §3): + +- **N1 (canonical form)**: ``normalize: WriteDomain -> bytes`` is total on + ``bytes | bytearray | memoryview | Filepath`` and **one-shot** on + ``BinaryIO | Chunks | Streamable`` — a stream ref is consumed by its first + write; assigning the same ref twice is a documented error, not a second copy. +- **N3 (the honest invariant)**: ``s[k] = v => s[k] == normalize(v)`` — for + the re-readable half. + +``str`` is rejected, loudly, on **decidability** (§2): ``s['config'] = '{"a":1}'`` +and ``s['video'] = '/tmp/big.mp4'`` are both overwhelmingly plausible, and any +rule distinguishing them is a latent data-corruption bug. One word — +``Filepath(p)`` — removes the ambiguity. Bare ``os.PathLike`` is rejected too: +accepting it while rejecting ``str`` would be a confusing half-rule. + +>>> reject_str_value('{"a": 1}') +Traceback (most recent call last): +... +TypeError: ... +""" + +from __future__ import annotations + +import io +import os +from dataclasses import dataclass, field +from functools import singledispatch +from typing import BinaryIO, Iterable, Union + +# --------------------------------------------------------------------------- # +# The refs +# --------------------------------------------------------------------------- # + + +@dataclass(frozen=True) +class Filepath: + """The value *is the content of this file*. Re-readable (N1-total).""" + + path: str + + def __post_init__(self): + # A Path is fine; normalise it to str so the ref stays picklable/hashable. + object.__setattr__(self, "path", os.fspath(self.path)) + + +@dataclass(frozen=True) +class Chunks: + """The value is the concatenation of an iterable of ``bytes`` chunks. + **One-shot** unless the iterable is re-iterable (a list is; a generator is + not — and a consumed generator writes ``b''``, which is why assigning the + same one-shot ref twice is a documented error).""" + + chunks: Iterable[bytes] + consumed: list = field(default_factory=list, compare=False, repr=False) + + def _consume_once(self): + if self.consumed: + raise ValueError( + "This Chunks ref was already written once. A stream ref is " + "one-shot (ADR-0005 N1): re-create the ref (or pass a list of " + "chunks, which is re-readable)." + ) + self.consumed.append(True) + return self.chunks + + +@dataclass(frozen=True) +class Streamable: + """The value is whatever a zero-arg factory's stream yields — for sources + that must be (re)opened at write time, e.g. an HTTP download. The factory + is called once per write, so unlike :class:`Chunks` this ref IS safely + re-writable (each write gets a fresh stream).""" + + open_stream: object # zero-arg callable -> BinaryIO | Iterable[bytes] + + +#: The static write domain (``BinaryIO`` is for mypy; runtime dispatch uses +#: ``io.IOBase`` — see :func:`as_fileobj`). +BytesSource = Union[ + bytes, bytearray, memoryview, BinaryIO, Filepath, Chunks, Streamable +] + + +def reject_str_value(v): + """The one error message for the ``str`` ambiguity (ADR-0005 §2).""" + raise TypeError( + "A str value is ambiguous here: did you mean its utf-8 bytes, or a " + "filepath?\n" + " content : s[k] = v.encode() (or wrap the store: " + "dol.wrap_kvs(store, data_of_obj=lambda s: s.encode(), " + "obj_of_data=lambda b: b.decode()))\n" + " filepath: s[k] = s3dol.Filepath(v)" + ) + + +class _IterableFileObj(io.RawIOBase): + """A minimal read-only file object over an iterable of bytes chunks.""" + + def __init__(self, chunks: Iterable[bytes]): + self._iterator = iter(chunks) + self._buffer = b"" + + def readable(self): + return True + + def readinto(self, target): + while len(self._buffer) < len(target): + try: + self._buffer += next(self._iterator) + except StopIteration: + break + n = min(len(target), len(self._buffer)) + target[:n] = self._buffer[:n] + self._buffer = self._buffer[n:] + return n + + +# --------------------------------------------------------------------------- # +# as_fileobj — the open extension point (ADR-0005 §1) +# --------------------------------------------------------------------------- # + + +@singledispatch +def as_fileobj(v) -> BinaryIO: + """A binary file object for any member of the write domain. + + ``functools.singledispatch``: the union is open for extension (users + register their own ref types) and closed for modification. Everything + non-``bytes`` routes through ``upload_fileobj`` downstream, so seekability + is s3transfer's problem. + """ + if isinstance(v, str): + reject_str_value(v) + if isinstance(v, os.PathLike): + raise TypeError( + f"Got a bare path object ({type(v).__name__}). Wrap it: " + f"s3dol.Filepath({v!r}) — one word, zero ambiguity (ADR-0005 §2)." + ) + raise TypeError( + f"Cannot write a {type(v).__name__} to an s3dol store. Accepted: " + f"bytes, bytearray, memoryview, an open binary file, " + f"s3dol.Filepath(path), s3dol.Chunks(iterable_of_bytes), or " + f"s3dol.Streamable(open_stream). Extend with " + f"s3dol.values.as_fileobj.register(YourType)." + ) + + +@as_fileobj.register(bytes) +def _(v: bytes): + return io.BytesIO(v) + + +@as_fileobj.register(bytearray) +def _(v: bytearray): + return io.BytesIO(bytes(v)) + + +@as_fileobj.register(memoryview) +def _(v: memoryview): + return io.BytesIO(bytes(v)) + + +# Register on io.IOBase, NEVER typing.BinaryIO: @register(BinaryIO) is accepted +# at definition time and then never fires — io.BytesIO is not in +# typing.BinaryIO's MRO (measured, ADR-0005 §1). io.IOBase covers BytesIO, +# BufferedReader, botocore.response.StreamingBody, SpooledTemporaryFile and +# urllib3 responses. +@as_fileobj.register(io.IOBase) +def _(v: io.IOBase): + return v + + +@as_fileobj.register(Filepath) +def _(v: Filepath): + return open(v.path, "rb") + + +@as_fileobj.register(Chunks) +def _(v: Chunks): + return _IterableFileObj(v._consume_once()) + + +@as_fileobj.register(Streamable) +def _(v: Streamable): + stream = v.open_stream() + if isinstance(stream, io.IOBase): + return stream + return _IterableFileObj(stream) + + +def is_bytes_like(v) -> bool: + """Whether ``v`` is an in-memory bytes value (the ``PutObject`` fast path).""" + return isinstance(v, (bytes, bytearray, memoryview)) diff --git a/s3dol/writes.py b/s3dol/writes.py new file mode 100644 index 0000000..509f0a3 --- /dev/null +++ b/s3dol/writes.py @@ -0,0 +1,99 @@ +"""Write strategies (ADR-0005 §5): how ``s[k] = v`` reaches the wire. + +A strategy is a callable ``(client, bucket, key, value) -> None``, injected at +store construction — ``__setitem__`` stays ``__setitem__``; *how* it uploads is +a constructor parameter. This is the answer to "infra-specific optimisation +without polluting the interface": multipart is *transport*, not storage-model. + +Strategies are frozen dataclasses, not closures, because every s3dol store +must pickle (ADR-0008 §6 / ADR-0012 D1) and a store carries its strategy. + +Every non-``bytes`` source routes through ``upload_fileobj``/``TransferManager``, +never ``PutObject`` — so seekability is s3transfer's problem, and the size +threshold is never consulted for a non-seekable source (where it is undecidable +anyway). Note ``io.UnsupportedOperation`` and botocore's +``UnseekableStreamError`` raise *before the request is built*, so they reach the +caller directly (they are not ``ClientError``s and the error seam does not — +and must not — convert them to ``KeyError``). + +>>> strategy = transfer_writes(multipart_threshold=16 * 2**20) +>>> import pickle; pickle.loads(pickle.dumps(strategy)) == strategy +True +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Protocol + +from s3dol.values import as_fileobj, is_bytes_like, reject_str_value + + +class WriteStrategy(Protocol): + def __call__(self, client: Any, bucket: str, key: str, value: Any) -> None: ... + + +#: boto3's own default multipart threshold (8 MiB). +DEFAULT_MULTIPART_THRESHOLD = 8 * 2**20 + + +@dataclass(frozen=True) +class TransferWrites: + """The default: small in-memory values take a single ``PutObject``; file + objects and refs go through ``upload_fileobj`` (which goes multipart above + the threshold, transparently). The overhead on small objects is one branch. + + Providers that cap multipart parts (Scaleway: 1000 — see the preset's + ``Capabilities.max_multipart_parts``) may need a larger + ``multipart_chunksize`` for very large objects: parts ≈ size / chunksize. + """ + + multipart_threshold: int = DEFAULT_MULTIPART_THRESHOLD + multipart_chunksize: int = DEFAULT_MULTIPART_THRESHOLD + max_concurrency: int = 10 + use_threads: bool = True + + def __call__(self, client, bucket: str, key: str, value) -> None: + if isinstance(value, str): + reject_str_value(value) + if is_bytes_like(value) and len(value) < self.multipart_threshold: + client.put_object(Bucket=bucket, Key=key, Body=bytes(value)) + return + from boto3.s3.transfer import TransferConfig + + config = TransferConfig( + multipart_threshold=self.multipart_threshold, + multipart_chunksize=self.multipart_chunksize, + max_concurrency=self.max_concurrency, + use_threads=self.use_threads, + ) + with as_fileobj(value) as fileobj: + client.upload_fileobj(fileobj, bucket, key, Config=config) + + +@dataclass(frozen=True) +class SimpleWrites: + """Everything through a single ``PutObject`` — refs are materialised in + memory first. For small-object workloads and providers with no multipart + at all (``Capabilities.multipart=False``).""" + + def __call__(self, client, bucket: str, key: str, value) -> None: + if isinstance(value, str): + reject_str_value(value) + if not is_bytes_like(value): + with as_fileobj(value) as fileobj: + value = fileobj.read() + client.put_object(Bucket=bucket, Key=key, Body=bytes(value)) + + +def transfer_writes(**kwargs) -> TransferWrites: + """Factory spelling of :class:`TransferWrites` (the constructor-injection + idiom of ADR-0005 §5: ``BucketStore(..., writes=transfer_writes(...))``).""" + return TransferWrites(**kwargs) + + +def simple_writes() -> SimpleWrites: + return SimpleWrites() + + +DEFAULT_WRITES = TransferWrites()