ENH: two-tier runtime config (permanent set_config + scoped config_context)

benchmarks/test_io_benchmarks.py

@@ -8,7 +8,7 @@
 import pytest
 
 import dascore as dc
-from dascore.config import set_config
+from dascore.config import get_config, set_config
 from dascore.exceptions import DependencyError
 from dascore.utils.downloader import fetch, get_registry_df
 
@@ -31,9 +31,17 @@ def test_file_paths():
 
 @pytest.fixture(scope="module", autouse=True)
 def allow_legacy_dasdae_coord_unpickle():
-    """Benchmarks include trusted historical DASDAE fixtures from the registry."""
-    with set_config(allow_dasdae_format_unpickle=True):
+    """Benchmarks include trusted historical DASDAE fixtures from the registry.
+
+    Uses the permanent config base (not a scoped ``config_context``) because a
+    module-scoped fixture spans many benchmarks.
+    """
+    previous = get_config()
+    set_config(allow_dasdae_format_unpickle=True)
+    try:
         yield
+    finally:
+        set_config(previous)
 
 
 class TestIOBenchmarks:

dascore/__init__.py

@@ -11,7 +11,13 @@
 from dascore.core.spool import BaseSpool, Spool, spool
 from dascore.core.coordmanager import get_coord_manager, CoordManager
 from dascore.core.coords import get_coord
-from dascore.config import DascoreConfig, get_config, reset_config, set_config
+from dascore.config import (
+    DascoreConfig,
+    config_context,
+    get_config,
+    reset_config,
+    set_config,
+)
 from dascore.examples import get_example_patch, get_example_spool
 from dascore.io.core import get_format, read, scan, scan_payloads, scan_to_df, write
 from dascore.units import get_quantity, get_unit

dascore/config.py

@@ -2,9 +2,13 @@
 
 from __future__ import annotations
 
+import os
+from collections.abc import Iterator
 from contextlib import contextmanager
+from contextvars import ContextVar
 from pathlib import Path
 from tempfile import gettempdir
+from threading import Lock
 from typing import Literal
 
 import pooch
@@ -27,6 +31,9 @@ class DascoreConfig(BaseModel):
     model_config = ConfigDict(
         frozen=True,
         validate_default=True,
+        # Reject unknown fields so misspelled overrides raise rather than
+        # silently doing nothing.
+        extra="forbid",
     )
 
     # General behavior.
@@ -139,9 +146,30 @@ def _coerce_path(cls, value):
         return Path(value).expanduser()
 
 
-# The active runtime config is a process-global singleton. A scoped override
-# via `set_config(...)` is applied globally and restored on context exit.
-_CONFIG = DascoreConfig()
+# Runtime configuration has two tiers. `_GLOBAL_CONFIG` is the process-wide
+# base, visible from every thread and task; `set_config(...)` swaps it. Scoped
+# overrides from `config_context(...)` live in a ContextVar so concurrent
+# blocks stay isolated per thread/task and never clobber one another.
+_GLOBAL_CONFIG: DascoreConfig = DascoreConfig()
+_GLOBAL_CONFIG_LOCK = Lock()
+_CONFIG_OVERRIDE: ContextVar[DascoreConfig | None] = ContextVar(
+    "dascore_config_override", default=None
+)
+
+
+def _reinit_config_lock():
+    """Install a fresh config lock, used after a fork.
+
+    A fork can copy the lock while another thread holds it. That thread does
+    not exist in the child, so the inherited copy would never be released and
+    any config change in the child would hang.
+    """
+    global _GLOBAL_CONFIG_LOCK
+    _GLOBAL_CONFIG_LOCK = Lock()
+
+
+if hasattr(os, "register_at_fork"):  # not available on windows
+    os.register_at_fork(after_in_child=_reinit_config_lock)
 
 
 class _ConfigDescriptor:
@@ -161,69 +189,106 @@ def config_attr(attr_name: str):
 
 
 def get_config() -> DascoreConfig:
-    """Return the active runtime configuration."""
-    return _CONFIG
+    """Return the active runtime configuration.
 
+    A scoped override from [`config_context`](`dascore.config.config_context`)
+    in the current thread/task takes precedence over the process-wide base set
+    by [`set_config`](`dascore.config.set_config`).
+    """
+    override = _CONFIG_OVERRIDE.get()
+    return override if override is not None else _GLOBAL_CONFIG
 
-@contextmanager
-def _restore_config(previous: DascoreConfig):
-    """Restore the previous config when exiting a context manager."""
-    global _CONFIG
-    try:
-        yield _CONFIG
-    finally:
-        _CONFIG = previous
+
+def _build_config(base: DascoreConfig, new_config, kwargs) -> DascoreConfig:
+    """Validate and build a config from a full replacement or field overrides."""
+    if new_config is not None and kwargs:
+        msg = "Cannot supply both new_config and keyword overrides."
+        raise ValueError(msg)
+    if new_config is None:
+        payload = base.model_dump()
+        payload.update(kwargs)
+        return DascoreConfig(**payload)
+    if not isinstance(new_config, DascoreConfig):
+        msg = "new_config must be an instance of DascoreConfig."
+        raise TypeError(msg)
+    return new_config
 
 
-def set_config(new_config: DascoreConfig | None = None, **kwargs):
+def set_config(new_config: DascoreConfig | None = None, **kwargs) -> DascoreConfig:
     """
-    Set the active runtime config and return a restoring context manager.
+    Set the process-wide runtime config, visible from every thread and task.
 
     Parameters
     ----------
     new_config
         A complete [`DascoreConfig`](`dascore.config.DascoreConfig`) to install.
         Mutually exclusive with keyword overrides.
     **kwargs
-        Individual field overrides applied on top of the current config.
+        Individual field overrides applied on top of the current base config.
 
     Notes
     -----
-    The config is a process-global singleton. An override is applied immediately
-    and also returns a context manager which restores the previous config on
-    exit. Overrides are not thread-scoped.
+    This is a permanent change to the process-wide base (it is not restored
+    automatically). For a temporary, thread/task-local override that restores
+    on exit, use [`config_context`](`dascore.config.config_context`) instead.
 
     Examples
     --------
     >>> import dascore as dc
-    >>> # Scoped override (restored on block exit) -- the common case.
-    >>> with dc.set_config(debug=True):
-    ...     assert dc.get_config().debug
-    >>> assert not dc.get_config().debug
-    >>>
-    >>> # Bare call: apply until reset.
     >>> _ = dc.set_config(display_float_precision=5)
     >>> assert dc.get_config().display_float_precision == 5
     >>> _ = dc.reset_config()
     """
-    global _CONFIG
-    previous = _CONFIG
-    if new_config is not None and kwargs:
-        msg = "Cannot supply both new_config and keyword overrides."
-        raise ValueError(msg)
-    if new_config is None:
-        payload = previous.model_dump()
-        payload.update(kwargs)
-        new_config = DascoreConfig(**payload)
-    elif not isinstance(new_config, DascoreConfig):
-        msg = "new_config must be an instance of DascoreConfig."
-        raise TypeError(msg)
-    _CONFIG = new_config
-    return _restore_config(previous)
+    global _GLOBAL_CONFIG
+    # Serialize the read-modify-write so concurrent keyword updates cannot lose
+    # each other's fields or return a config another thread just installed.
+    with _GLOBAL_CONFIG_LOCK:
+        _GLOBAL_CONFIG = _build_config(_GLOBAL_CONFIG, new_config, kwargs)
+        return _GLOBAL_CONFIG
+
+
+@contextmanager
+def config_context(
+    new_config: DascoreConfig | None = None, **kwargs
+) -> Iterator[DascoreConfig]:
+    """
+    Temporarily override the runtime config for the current thread/task.
+
+    Parameters
+    ----------
+    new_config
+        A complete [`DascoreConfig`](`dascore.config.DascoreConfig`) to install.
+        Mutually exclusive with keyword overrides.
+    **kwargs
+        Individual field overrides applied on top of the active config.
+
+    Notes
+    -----
+    The override is stored in a ``ContextVar``, so it is isolated per thread and
+    task and restored when the block exits. Whether a newly started OS thread
+    inherits a copy of the override is runtime-dependent
+    (``sys.flags.thread_inherit_context`` -- normally enabled on free-threaded
+    builds and disabled otherwise); an inherited copy is not undone by this
+    block's exit. For deterministic propagation, capture
+    ``contextvars.copy_context()`` and run the worker with it, or rely on APIs
+    that bind the config for you such as
+    [`Spool.map`](`dascore.core.spool.BaseSpool.map`).
+
+    Examples
+    --------
+    >>> import dascore as dc
+    >>> with dc.config_context(debug=True):
+    ...     assert dc.get_config().debug
+    >>> assert not dc.get_config().debug
+    """
+    config = _build_config(get_config(), new_config, kwargs)
+    token = _CONFIG_OVERRIDE.set(config)
+    try:
+        yield config
+    finally:
+        _CONFIG_OVERRIDE.reset(token)
 
 
 def reset_config() -> DascoreConfig:
-    """Reset the active runtime config to defaults."""
-    global _CONFIG
-    _CONFIG = DascoreConfig()
-    return _CONFIG
+    """Reset the process-wide runtime config base to defaults."""
+    return set_config(DascoreConfig())

dascore/examples.py

@@ -13,7 +13,7 @@
 import dascore as dc
 import dascore.core
 from dascore.compat import random_state
-from dascore.config import set_config
+from dascore.config import config_context
 from dascore.exceptions import UnknownExampleError
 from dascore.utils.downloader import fetch
 from dascore.utils.imports import lazy_import
@@ -29,7 +29,7 @@
 
 def _load_example_patch_from_file(path: str | Path) -> dc.Patch:
     """Load the first patch from an example file without spool indirection."""
-    with set_config(allow_dasdae_format_unpickle=True):
+    with config_context(allow_dasdae_format_unpickle=True):
         return dc.read(path)[0]
 
 

dascore/io/dasdae/_compat.py

@@ -63,8 +63,10 @@ def translate_legacy_attrs(attrs):
             msg = (
                 "This DASDAE file contains legacy pickled coordinate metadata. "
                 "Unpickling DASDAE format metadata is disabled by default for "
-                "security. If you trust this file, enable legacy compatibility "
-                "with dc.set_config(allow_dasdae_format_unpickle=True)."
+                "security. If you trust this file, read it inside a "
+                "'with dc.config_context(allow_dasdae_format_unpickle=True):' "
+                "block, or enable it permanently with "
+                "dc.set_config(allow_dasdae_format_unpickle=True)."
             )
             raise InvalidFiberFileError(msg)
         with contextlib.suppress(

dascore/utils/misc.py

@@ -23,6 +23,7 @@
 from scipy.special import factorial
 
 from dascore.compat import UPath, is_array
+from dascore.config import config_context, get_config
 from dascore.constants import WARN_LEVELS
 from dascore.exceptions import (
     FilterValueError,
@@ -663,20 +664,26 @@ def wrapper(self, *args, **kwargs):
 class _MapFuncWrapper:
     """A class for unwrapping spools to base applies."""
 
-    def __init__(self, func, kwargs, progress=True):
+    def __init__(self, func, kwargs, config, progress=True):
         self._func = func
         self._kwargs = kwargs
         self._progress = progress
+        # Bind the config active at map() call time so workers (threads or
+        # pickled into processes) apply the same config the caller had, rather
+        # than a fresh default or a scoped override that would not propagate.
+        self._config = config
 
     def __call__(self, spool):
-        iterable = spool
-        # in order to handle multiprocessing, we apply a secret tag of "_progress"
-        # to the first spool. This way only the first spool displays the
-        # the progress bar. A huge hack, maybe there is a better way? See #265.
-        if not getattr(spool, "_no_progress", False):
-            desc = f"Applying {self._func.__name__} to spool"
-            iterable = track(spool, desc) if self._progress else spool
-        return [self._func(x, **self._kwargs) for x in iterable]
+        with config_context(self._config):
+            iterable = spool
+            # in order to handle multiprocessing, we apply a secret tag of
+            # "_progress" to the first spool. This way only the first spool
+            # displays the progress bar. A huge hack, maybe there is a better
+            # way? See #265.
+            if not getattr(spool, "_no_progress", False):
+                desc = f"Applying {self._func.__name__} to spool"
+                iterable = track(spool, desc) if self._progress else spool
+            return [self._func(x, **self._kwargs) for x in iterable]
 
 
 def _spool_map(spool, func, size=None, client=None, progress=True, **kwargs):
@@ -711,7 +718,7 @@ def _spool_map(spool, func, size=None, client=None, progress=True, **kwargs):
     # displayed in one thread/process.
     for sub_spool in spools[1:]:
         sub_spool._no_progress = True
-    new_func = _MapFuncWrapper(func, kwargs, progress=progress)
+    new_func = _MapFuncWrapper(func, kwargs, get_config(), progress=progress)
     return [x for y in client.map(new_func, spools) for x in y]
 
 

docs/changelog.qmd

@@ -4,6 +4,7 @@ The [releases page](https://github.com/DASDAE/dascore/releases) tracks changes f
 
 ## Unreleased API Changes
 
+- **Runtime configuration is now two-tier.** `dc.set_config(...)` sets the process-wide base permanently (visible from every thread and task) and returns the new config; it is no longer a context manager. Temporary, thread/task-local overrides use the new `dc.config_context(...)` context manager, which restores on exit and isolates concurrent overrides via a `ContextVar`. `Spool.map(...)` captures the config active at the call and re-applies it in each worker, so thread- and process-pool workers observe the caller's config regardless of the pool's start method. Unknown config field names now raise instead of being silently ignored. Migrate `with dc.set_config(...)` to `with dc.config_context(...)`.
 - PRODML 2.1 now supports writing one raw time-by-distance patch with `dc.write` as a standalone HDF5 file. Full PRODML 2.3 conformance requires EPC/XML packaging, which DASCore does not write.
 - **The `dascore.io.sintela_binary` module is removed (no alias).** Both Sintela readers now live in `dascore.io.sintela`, which also provides the new protobuf reader; use `from dascore.io.sintela import SintelaBinaryV3`. Reading Sintela binary files through `dc.read`/`dc.spool`/`dc.scan` is unaffected — only the direct module import path changed.
 - **Removed the unused `PatchFileSummary` model** (`dascore.io.PatchFileSummary`), superseded by [`PatchSummary`](`dascore.PatchSummary`), along with the internal helpers `coord_summary_from_data` and `_normalize_coord_summary_dtype`. Build a coord first (`get_coord(...)`) and call `.to_summary()` to summarize raw array data. The unused `index_query_buffer` config option is also removed.

docs/tutorial/configuration.qmd

@@ -5,25 +5,45 @@ DASCore exposes a small runtime configuration surface through `dascore.config`.
 ```python
 from pathlib import Path
 
-from dascore.config import get_config, set_config
+from dascore.config import get_config, config_context
 
 print(get_config().remote_cache_dir)
 
-with set_config(remote_cache_dir=Path("/tmp/dascore-remote-cache")):
+with config_context(remote_cache_dir=Path("/tmp/dascore-remote-cache")):
     ...
 ```
 
 Configuration changes affect subsequent operations only. For example, changing `remote_cache_dir` changes where future remote-file materializations are cached.
 
+## Two configuration tiers
+
+Config can be changed in two ways:
+
+- `set_config(...)` changes the process-wide base permanently. The change is visible from every thread and task and is not restored automatically; `reset_config()` returns to defaults. Use it for application-level settings applied once at startup.
+- `config_context(...)` overrides the config only for the current thread or task. The override is restored when the block exits, and concurrent blocks in different threads never clobber one another.
+
+```python
+import dascore as dc
+
+dc.set_config(display_float_precision=5)  # permanent
+
+with dc.config_context(display_float_precision=8):  # scoped to this block
+    ...
+
+dc.reset_config()  # drop the permanent change
+```
+
+[`Spool.map`](`dascore.core.spool.BaseSpool.map`) binds the config active when `map` is called and re-applies it in each worker, so overrides also reach thread- and process-pool workers.
+
 History recording is also configurable:
 
 - `patch_history="standard"` preserves the default behavior and appends new entries to `Patch.attrs.history`.
 - `patch_history="disabled"` preserves any existing history but stops DASCore from appending new entries inside that config context.
 
 ```python
-from dascore.config import set_config
+from dascore.config import config_context
 
-with set_config(patch_history="disabled"):
+with config_context(patch_history="disabled"):
     ...
 ```