From 3b2c3a0962a09dc6ac8224db67eadc2ac8e3ffd6 Mon Sep 17 00:00:00 2001 From: Thor Whalen <1906276+thorwhalen@users.noreply.github.com> Date: Sat, 26 Sep 2026 09:17:38 +0000 Subject: [PATCH 1/6] docs: document #16 risks (pickle decode, broad fallback, silent empties); drop dead code Documentation-only half of the #16 audit omnibus. No behaviour change. - codecs: warn that .pkl/.pickle decode with pickle.loads (arbitrary code execution on untrusted bytes); replace the eval() in register_codec's example. - base: get_config docstring warns about the broad (Exception,) default; remove the dead commented-out OPENAI_API_KEY/getpass block. - tools/s_configparser: state the current silent-empty behaviour of extract_exports and ConfigReader for missing paths, the os.path.sep-based path detection, and the import-time folder creation. The behaviour changes are split into #25, #26, #27, #28 and #29. Fixes #16 --- config2py/base.py | 20 ++++++++------------ config2py/codecs.py | 16 +++++++++++++++- config2py/s_configparser.py | 6 +++++- config2py/tools.py | 18 +++++++++++++++++- 4 files changed, 45 insertions(+), 15 deletions(-) diff --git a/config2py/base.py b/config2py/base.py index 6d43c4a..897e3d3 100644 --- a/config2py/base.py +++ b/config2py/base.py @@ -78,17 +78,6 @@ def __contains__(self, k: KT) -> bool: Sources = Iterable[Union[GettableContainer, Getter]] GetConfigEgress = Callable[[KT, VT], VT] -# # TODO: Refactor into reusable function that can look (and write) in multiple stores -# _open_api_key_env_name = 'OPENAI_API_KEY' -# _api_key = os.environ.get(_open_api_key_env_name, None) -# if _api_key is None: -# # TODO: Figure out a way to make input response invisible (using * or something) -# _api_key = getpass.getpass( -# f"Please set your OpenAI API key and press enter to continue. " -# f"I will put it in the environment variable {_open_api_key_env_name} " -# ) -# openai.api_key = _api_key - def is_not_none_nor_empty(x): """True unless ``x`` is ``None`` or the empty string. @@ -171,7 +160,14 @@ def get_config( you want in some situations, since you'd be hiding some errors that you might want to be aware of. This is why allow you to specify what exceptions should actually be considered as "config not found" exceptions, through the - ``config_not_found_exceptions`` argument, which defaults to ``Exception``. + ``config_not_found_exceptions`` argument, which defaults to ``(Exception,)``. + + Beware that this broad default treats *any* error raised by a callable source + (a network blip, a bug, a missing import, rejected credentials) as "not found", + and silently moves on to the next, possibly less trusted, source. When the + sources are known, prefer passing something narrower, such as + ``config_not_found_exceptions=(KeyError, LookupError, FileNotFoundError)`` + (see https://github.com/i2mint/config2py/issues/25). Further, your sources may return a value, but not one that you consider valid: For example, a sentinel like ``None``. In this case you may want the search to diff --git a/config2py/codecs.py b/config2py/codecs.py index 3a49029..1b91f50 100644 --- a/config2py/codecs.py +++ b/config2py/codecs.py @@ -27,6 +27,12 @@ The module automatically registers codecs for standard formats (json, toml, ini, etc.) and conditionally registers codecs that require third-party libraries (yaml, json5, etc.). + +Security warning: the ``.pkl`` and ``.pickle`` extensions decode with +``pickle.loads``, which can execute arbitrary code. Never call +``decode_by_extension`` with those extensions on bytes you do not fully trust (for +example, bytes fetched from a remote or shared store). +See https://github.com/i2mint/config2py/issues/29. """ from typing import Callable, TypeVar, Any, Optional @@ -116,6 +122,11 @@ def decode_by_extension(key: str, data: bytes) -> Any: Raises: ValueError: If no decoder registered for extension + Warning: + The decoder is chosen by the key's extension alone. ``.pkl`` and + ``.pickle`` map to ``pickle.loads``, which can execute arbitrary code, so + never decode untrusted bytes under those extensions. + Examples: >>> data = b'{"key": "value"}' @@ -193,7 +204,7 @@ def register_codec( Examples: >>> def my_encoder(obj): return str(obj).encode() - >>> def my_decoder(data): return eval(data.decode()) + >>> def my_decoder(data): return data.decode() >>> register_codec('.custom', encoder=my_encoder, decoder=my_decoder, overwrite=True) """ if not extension.startswith("."): @@ -356,6 +367,9 @@ def get_codec_info(extension: str) -> dict[str, Any]: ) # Pickle - Python object serialization (not text-based, but useful) +# SECURITY: ``pickle.loads`` executes arbitrary code embedded in the bytes it reads. +# Only decode ``.pkl``/``.pickle`` data you produced yourself or otherwise fully +# trust. Making this decoder opt-in is tracked in i2mint/config2py#29. register_codec( ".pkl", encoder=pickle.dumps, diff --git a/config2py/s_configparser.py b/config2py/s_configparser.py index 21a53ae..78ee6a3 100644 --- a/config2py/s_configparser.py +++ b/config2py/s_configparser.py @@ -239,7 +239,11 @@ def __init__( ): """See the class docstring: ``source`` may be a filepath, a config string, bytes, a dict, or a readable stream; ``defaults``, ``dict_type`` and - ``allow_no_value`` are passed on to ``ConfigParser``.""" + ``allow_no_value`` are passed on to ``ConfigParser``. + + Note: a filepath that doesn't exist is silently skipped (the semantics of + ``ConfigParser.read``), giving an empty config rather than an error + (see https://github.com/i2mint/config2py/issues/27).""" super().__init__( defaults, dict_type, allow_no_value, **more_config_parser_kwargs ) diff --git a/config2py/tools.py b/config2py/tools.py index dfcb627..2145fd9 100644 --- a/config2py/tools.py +++ b/config2py/tools.py @@ -27,6 +27,12 @@ def get_configs_local_store( If it's a directory, it's assumed to be a folder of text files. If it's a file, it's assumed to be an ini or cfg file. If it's a string, it's assumed to be an app name, from which to create a folder + + Note: a directory is only recognized as such if ``config_src`` contains + ``os.path.sep``. A bare name (e.g. ``"configs"``) is always treated as an app + name, even if a directory of that name exists in the current working directory; + pass ``"./configs"`` (or an absolute path) to use that directory + (see https://github.com/i2mint/config2py/issues/28). """ if os.path.sep in config_src and os.path.isdir(config_src): # TODO: This was a quick fix to avoid unknowingly making directories in the @@ -103,7 +109,10 @@ def simple_config_getter( return config_getter -# Make a ready-to-use config getter, using the defaults +# Make a ready-to-use config getter, using the defaults. +# Note: this (and ``local_configs`` below) runs at import time, so ``import config2py`` +# creates the default configs folder (``~/.config/config2py/configs`` on Linux) if it +# doesn't exist yet. Making this lazy is tracked in i2mint/config2py#26. config_getter = simple_config_getter() @@ -147,6 +156,13 @@ def extract_exports(exports: str) -> dict: >>> extract_exports('export KEY="secret"\nexport TOKEN="arbitrary"') {'KEY': 'secret', 'TOKEN': 'arbitrary'} + Note that a single-line argument that is not an existing file is parsed as + content, not as a path. A mistyped path therefore silently gives an empty dict + (see https://github.com/i2mint/config2py/issues/27): + + >>> extract_exports('no/such/path/.env') + {} + Use case: --------- From b03b89af2e630aca9d2e0771e792112c24ac0aea Mon Sep 17 00:00:00 2001 From: Thor Whalen <1906276+thorwhalen@users.noreply.github.com> Date: Sun, 27 Sep 2026 08:22:26 +0000 Subject: [PATCH 2/6] fix: mask secret-looking prompts by default; read piped stdin when masking ask_user_for_input (and so the simple_config_getter/config_getter prompt-for-missing-key flow) echoed every typed value, secrets included. - DFLT_MASKING_INPUT is now looks_like_secret: prompts mentioning secret/token/pass/pwd/api/key/credential/auth/private are masked, others (file paths, names) still echo. mask_input accepts a bool or a prompt -> bool predicate; explicit True/False behave as before. - Masked reads use input() when stdin is not a terminal and getpass is the stdlib one: stdlib getpass reads /dev/tty, not stdin, so piped input was ignored (or the call hung). A frontend's replacement getpass (Jupyter's masked widget) is always used. Tests: config2py/tests/test_masking.py (17). Verified under a real pty that typed secrets no longer appear in terminal output, and that piped input with a controlling tty is read instead of hanging. Dependents py2store, xdol and oa match their baselines. Fixes #13 --- .claude/CLAUDE.md | 16 ++- config2py/tests/test_masking.py | 175 +++++++++++++++++++++++++++ config2py/tests/test_tools.py | 3 +- config2py/tests/utils_for_testing.py | 8 +- config2py/util.py | 65 +++++++++- 5 files changed, 252 insertions(+), 15 deletions(-) create mode 100644 config2py/tests/test_masking.py diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index 9eab1f1..fd824f8 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -39,12 +39,16 @@ command above (matching CI) to catch that before pushing. ## Invariants / known gaps (see open issues before "fixing" these) -- **`DFLT_MASKING_INPUT = False`** in `util.py` — `simple_config_getter`'s - prompt-for-missing-key flow echoes typed input (including secrets) to the - terminal by default; masking (`getpass`) is wired in but off by default. - Flipping the default is blocked on 3 fleet dependents not present on every - box (see [i2mint/config2py#13](https://github.com/i2mint/config2py/issues/13)) — - don't change it without re-running the dependents check. +- **`DFLT_MASKING_INPUT = looks_like_secret`** in `util.py` (since + [#13](https://github.com/i2mint/config2py/issues/13)) — `ask_user_for_input` + masks prompts that mention something secret-looking (`API_KEY`, `TOKEN`, + `PASSWORD`, ...) and echoes the others. `mask_input` accepts a bool or a + `prompt -> bool` predicate. When stdin is not a terminal and `getpass.getpass` + is the stdlib one, masked reads fall back to `input` (stdlib `getpass` reads + `/dev/tty`, not stdin, so it would ignore piped input or hang). A frontend's + replacement `getpass` (Jupyter) is always used. Tests in + `config2py/tests/test_masking.py`; don't change the default without re-running + the dependents check. - [i2mint/config2py#16](https://github.com/i2mint/config2py/issues/16) — an omnibus of smaller audit findings (docstring overclaims, a broad fallback, the pickle codec, import-time side effects) — read before touching those areas. diff --git a/config2py/tests/test_masking.py b/config2py/tests/test_masking.py new file mode 100644 index 0000000..05e5c35 --- /dev/null +++ b/config2py/tests/test_masking.py @@ -0,0 +1,175 @@ +"""Tests for input masking in ``ask_user_for_input`` (i2mint/config2py#13). + +Two prompt functions are faked throughout: + +- a *stdlib-like* ``getpass`` (its ``__module__`` is ``"getpass"``), which, like the + real one, reads the controlling terminal rather than ``sys.stdin``; +- ``builtins.input``, which reads ``sys.stdin``. + +Whether ``sys.stdin`` is a terminal is controlled by swapping in stand-ins. +""" + +import builtins +import getpass +import io +import sys + +import pytest + +from config2py.util import ask_user_for_input, looks_like_secret +from config2py.tools import simple_config_getter + + +class _TtyStdin(io.StringIO): + """A stdin stand-in that claims to be an interactive terminal.""" + + def isatty(self): + return True + + +def _fake_getpass(returns, module="getpass"): + calls = [] + + def fake(prompt="", stream=None): + calls.append(prompt) + if isinstance(returns, BaseException): + raise returns + return returns + + fake.__module__ = module + fake.calls = calls + return fake + + +def _fake_input(returns): + calls = [] + + def fake(prompt=""): + calls.append(prompt) + if isinstance(returns, BaseException): + raise returns + return returns + + fake.calls = calls + return fake + + +@pytest.fixture +def interactive_terminal(monkeypatch): + """Make ``sys.stdin`` look like a terminal; return the prompt fakes to configure.""" + monkeypatch.setattr(sys, "stdin", _TtyStdin()) + + def install(*, getpass_returns="masked value", input_returns="echoed value"): + fake_getpass = _fake_getpass(getpass_returns) + fake_input = _fake_input(input_returns) + monkeypatch.setattr(getpass, "getpass", fake_getpass) + monkeypatch.setattr(builtins, "input", fake_input) + return fake_getpass, fake_input + + return install + + +@pytest.mark.parametrize( + "key", + ["OPENAI_API_KEY", "github_token", "DB_PASSWORD", "client_secret", "MY_PWD"], +) +def test_secret_looking_keys_are_masked_by_default(interactive_terminal, key): + fake_getpass, fake_input = interactive_terminal() + assert ask_user_for_input(f"Enter a value for {key}: ") == "masked value" + assert len(fake_getpass.calls) == 1 + assert fake_input.calls == [] + + +@pytest.mark.parametrize("key", ["DATA_DIR", "display_name", "OA_DFLT_MODEL"]) +def test_other_keys_are_echoed_by_default(interactive_terminal, key): + fake_getpass, fake_input = interactive_terminal() + assert ask_user_for_input(f"Enter a value for {key}: ") == "echoed value" + assert fake_getpass.calls == [] + assert len(fake_input.calls) == 1 + + +def test_default_value_does_not_decide_masking(interactive_terminal): + """Only the caller's prompt is inspected, not the ``[default]`` decoration.""" + fake_getpass, fake_input = interactive_terminal(input_returns="") + assert ask_user_for_input("Enter DATA_DIR", default="~/my_keys") == "~/my_keys" + assert fake_getpass.calls == [] + + +def test_explicit_mask_input_overrides_the_heuristic(interactive_terminal): + fake_getpass, fake_input = interactive_terminal() + assert ask_user_for_input("Enter API_KEY", mask_input=False) == "echoed value" + assert ask_user_for_input("Enter DATA_DIR", mask_input=True) == "masked value" + assert len(fake_getpass.calls) == len(fake_input.calls) == 1 + + +def test_mask_input_can_be_a_custom_predicate(interactive_terminal): + fake_getpass, _ = interactive_terminal() + always = ask_user_for_input("Enter DATA_DIR", mask_input=lambda prompt: True) + assert always == "masked value" + assert len(fake_getpass.calls) == 1 + + +def test_masking_toggle_starts_from_the_resolved_default(interactive_terminal, monkeypatch): + """With the toggle, the user sees (and flips) the state the heuristic chose.""" + answers = iter(["", "typed"]) # "" toggles masking, then the real value + prompts = [] + + def respond(prompt=""): + prompts.append(prompt) + return next(answers) + + respond.__module__ = "getpass" + monkeypatch.setattr(getpass, "getpass", respond) + monkeypatch.setattr(builtins, "input", _fake_input(AssertionError("not masked"))) + with pytest.raises(AssertionError, match="not masked"): + # secret-looking: starts ENABLED, toggling switches to (the failing) input + ask_user_for_input("Enter API_KEY", masking_toggle_str="") + assert "Input masking is ENABLED" in prompts[0] + + +@pytest.mark.parametrize("mask_input", [True, looks_like_secret]) +def test_piped_stdin_is_read_even_when_masking(monkeypatch, mask_input): + """Stdlib ``getpass`` reads the terminal, not stdin, so it would ignore piped + input (or block waiting on the terminal). Without a terminal there is nothing to + echo to, so the value must be read from stdin. + """ + monkeypatch.setattr(sys, "stdin", io.StringIO("piped value\n")) + monkeypatch.setattr( + getpass, "getpass", _fake_getpass(AssertionError("read the terminal")) + ) + # Note: the real builtins.input is used here, reading the replaced sys.stdin + assert ask_user_for_input("Enter API_KEY", mask_input=mask_input) == "piped value" + + +def test_frontend_getpass_is_used_without_a_terminal(monkeypatch): + """Frontends such as Jupyter replace ``getpass.getpass`` with their own masked + widget while ``sys.stdin`` is not a terminal. That replacement must be used. + """ + monkeypatch.setattr(sys, "stdin", io.StringIO("")) + frontend_getpass = _fake_getpass("from widget", module="ipykernel.kernelbase") + monkeypatch.setattr(getpass, "getpass", frontend_getpass) + monkeypatch.setattr(builtins, "input", _fake_input(AssertionError("echoed"))) + assert ask_user_for_input("Enter API_KEY") == "from widget" + + +def test_no_stdin_at_all_does_not_crash(monkeypatch): + """``sys.stdin`` is ``None`` under pythonw and some embedded interpreters.""" + monkeypatch.setattr(sys, "stdin", None) + monkeypatch.setattr(getpass, "getpass", _fake_getpass(AssertionError("tty"))) + monkeypatch.setattr(builtins, "input", _fake_input("fallback")) + assert ask_user_for_input("Enter API_KEY") == "fallback" + + +def test_simple_config_getter_masks_secret_keys_end_to_end( + interactive_terminal, tmp_path +): + fake_getpass, fake_input = interactive_terminal( + getpass_returns="sk-not-shown", input_returns="/some/dir" + ) + get = simple_config_getter( + str(tmp_path) + "/", first_look_in_env_vars=False, ask_user_if_key_not_found=True + ) + assert get("_C2P_TEST_API_TOKEN_") == "sk-not-shown" + assert get("_C2P_TEST_DATA_DIR_") == "/some/dir" + assert len(fake_getpass.calls) == len(fake_input.calls) == 1 + assert (tmp_path / "_C2P_TEST_API_TOKEN_").read_text() == "sk-not-shown" diff --git a/config2py/tests/test_tools.py b/config2py/tests/test_tools.py index e9d562a..b222687 100644 --- a/config2py/tests/test_tools.py +++ b/config2py/tests/test_tools.py @@ -35,7 +35,8 @@ def test_simple_config_getter(mock_config_store_factory): # Test getting config with ask_user_if_key_not_found=True # (patch both prompt functions ask_user_for_input can dispatch to -- which one is - # used depends on mask_input, default False today, see i2mint/config2py#13) + # used depends on mask_input, which by default masks secret-looking prompts only, + # see i2mint/config2py#13) with ( patch("builtins.input", return_value="from user"), patch("getpass.getpass", return_value="from user"), diff --git a/config2py/tests/utils_for_testing.py b/config2py/tests/utils_for_testing.py index dff9fd5..cc6d1e0 100644 --- a/config2py/tests/utils_for_testing.py +++ b/config2py/tests/utils_for_testing.py @@ -6,10 +6,10 @@ def user_input_patch(monkeypatch, user_input_string: str): """Patch both prompt functions ``ask_user_for_input`` can dispatch to. - Which one is actually called depends on ``mask_input`` (currently defaults to - ``DFLT_MASKING_INPUT = False``, i.e. ``input`` -- but see the still-open - i2mint/config2py#13, which proposes flipping that default), so tests that don't - care about masking specifically should patch both rather than assume one. + Which one is actually called depends on ``mask_input`` (by default + ``DFLT_MASKING_INPUT = looks_like_secret``: secret-looking prompts are masked, the + others echoed, see i2mint/config2py#13), so tests that don't care about masking + specifically should patch both rather than assume one. """ monkeypatch.setattr("builtins.input", lambda _: user_input_string) monkeypatch.setattr("getpass.getpass", lambda _: user_input_string) diff --git a/config2py/util.py b/config2py/util.py index 4cd2371..23e9826 100644 --- a/config2py/util.py +++ b/config2py/util.py @@ -5,6 +5,7 @@ import re import os +import sys import ast from collections import ChainMap, namedtuple from pathlib import Path @@ -22,7 +23,33 @@ # return type(name, (), {'__repr__': lambda self: name})() DFLT_APP_NAME = "config2py" -DFLT_MASKING_INPUT = False + +SECRET_LOOKING_PATTERN = re.compile( + r"secret|token|pass|pwd|api|key|credential|auth|private", re.IGNORECASE +) + + +def looks_like_secret(text: str) -> bool: + """True if ``text`` (typically a prompt naming a config key) looks secret. + + It errs on the side of masking: a false positive only means the user doesn't see + what they type, while a false negative echoes a secret to the terminal. + + >>> looks_like_secret("Enter a value for OPENAI_API_KEY: ") + True + >>> looks_like_secret("Enter a value for github_token: ") + True + >>> looks_like_secret("Enter a value for DATA_DIR: ") + False + """ + return bool(SECRET_LOOKING_PATTERN.search(text)) + + +# The default for ``ask_user_for_input``'s ``mask_input``: either a bool, or a +# ``prompt -> bool`` predicate deciding per prompt. Masking only secret-looking prompts +# (rather than all of them) keeps non-secret values, such as the file paths the +# README suggests storing, visible while being typed (see i2mint/config2py#13). +DFLT_MASKING_INPUT = looks_like_secret not_found = mk_sentinel("not_found") no_default = mk_sentinel("no_default") @@ -118,12 +145,35 @@ def __repr__(self): envvar = EnvironmentVariables() +def _stdin_is_a_terminal() -> bool: + try: + return sys.stdin is not None and sys.stdin.isatty() + except (AttributeError, ValueError): # exotic or closed stdin + return False + + +def _masked_prompt_func() -> Callable[[str], str]: + """The function to read a masked response with. + + The stdlib's ``getpass.getpass`` reads the controlling terminal, not ``sys.stdin``: + with piped input it would ignore the pipe (or block waiting on the terminal). + When stdin isn't a terminal there's nothing to echo to, so we read stdin with + ``input`` instead. A ``getpass.getpass`` replaced by a frontend (e.g. Jupyter's + masked widget, where stdin is never a terminal) is always used. + """ + getpass_func = getpass.getpass + is_stdlib_getpass = getattr(getpass_func, "__module__", None) == "getpass" + if is_stdlib_getpass and not _stdin_is_a_terminal(): + return input + return getpass_func + + # TODO: Make this into an open-closed mini-framework def ask_user_for_input( prompt: str, default: str = "", *, - mask_input=DFLT_MASKING_INPUT, + mask_input: bool | Callable[[str], bool] = DFLT_MASKING_INPUT, masking_toggle_str: str = None, egress: Callable = identity, ) -> str: @@ -132,7 +182,12 @@ def ask_user_for_input( :param prompt: Prompt to display to the user :param default: Default value to return if the user enters nothing - :param mask_input: Whether to mask the user's input + :param mask_input: Whether to mask the user's input: a bool, or a + ``prompt -> bool`` predicate. The default, ``looks_like_secret``, masks + prompts that mention something secret-looking (``API_KEY``, ``TOKEN``, + ``PASSWORD``, ...) and echoes the others. Masking needs a terminal (or a + frontend such as Jupyter): when stdin is piped, the response is read from + stdin, where nothing is echoed anyway. :param masking_toggle_str: String to toggle input masking. If ``None``, no toggle is available. If not ``None`` (a common choice is the empty string) the user can enter this string to toggle input masking. @@ -140,6 +195,8 @@ def ask_user_for_input( This can be used to validate the response, for example. :return: The user's response (or the default value if the user entered nothing) """ + if callable(mask_input): + mask_input = bool(mask_input(prompt)) _original_prompt = prompt if prompt[-1] != " ": # pragma: no cover prompt = prompt + " " @@ -152,7 +209,7 @@ def ask_user_for_input( if default not in {""}: prompt = prompt + f" [{default}]: " if mask_input: - _prompt_func = getpass.getpass + _prompt_func = _masked_prompt_func() else: _prompt_func = input From c1cb644fd24f3e4c6d6f293b10923f17fb701a87 Mon Sep 17 00:00:00 2001 From: Thor Whalen <1906276+thorwhalen@users.noreply.github.com> Date: Sun, 27 Sep 2026 08:25:37 +0000 Subject: [PATCH 3/6] packaging: SPDX license, classifiers, keywords, author and project URLs - license = "Apache-2.0" (PEP 639) plus license-files, replacing the deprecated [project.license] table; no License :: classifier. - Classifiers for Python 3.10-3.13 (suite verified on each), keywords, author, and Documentation/Repository/Issues URLs (docs site checked live). - build-system floor hatchling>=1.27, the first release with PEP 639 support. Dependencies, version and CI are unchanged; setup.cfg removal and the CI stub migration stay in #22. twine check passes on the sdist and wheel, and the CI version-bump regex still targets [project].version only. --- pyproject.toml | 36 ++++++++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f7878a6..57e9f4f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["hatchling"] +requires = ["hatchling>=1.27"] # PEP 639: SPDX `license` string + `license-files` build-backend = "hatchling.build" [project] @@ -8,15 +8,39 @@ version = "0.1.54" description = "Simplified reading and writing configurations from various sources and formats" readme = "README.md" requires-python = ">=3.10" -keywords = [] -authors = [] +license = "Apache-2.0" +license-files = ["LICENSE"] +keywords = [ + "config", + "configuration", + "settings", + "environment-variables", + "secrets", + "xdg", + "configparser", + "key-value-store", +] +authors = [{ name = "Thor Whalen" }] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: Utilities", +] dependencies = ["dol", "i2", "importlib_resources; python_version < '3.9'"] -[project.license] -text = "Apache-2.0" - [project.urls] Homepage = "https://github.com/i2mint/config2py" +Documentation = "https://i2mint.github.io/config2py/" +Repository = "https://github.com/i2mint/config2py" +Issues = "https://github.com/i2mint/config2py/issues" [project.optional-dependencies] dev = ["pytest>=7.0", "pytest-cov>=4.0", "ruff>=0.1.0"] From 3e3625df83d6850a702068f95c57910f77fe9557 Mon Sep 17 00:00:00 2001 From: Thor Whalen <1906276+thorwhalen@users.noreply.github.com> Date: Sun, 27 Sep 2026 08:58:59 +0000 Subject: [PATCH 4/6] agent layer: consumer and dev skills, .claude/skills links, sdist fix - config2py/data/skills/config2py-quickstart: how to use the package (entry points, get_config, simple_config_getter, user_gettable, app folders, FileStore, ConfigStore, codecs, gotchas). Ships in the wheel. Every Python snippet was run. - skills/config2py-dev: how to work on it (module map, CI-matching test command, test isolation, dependents gate, open design issues, release flow). - .claude/skills/: relative symlinks so Claude Code loads both. - pyproject: [tool.hatch.build.targets.sdist] excludes .claude/skills with skip-excluded-dirs. hatchling walks with followlinks=True and skips inodes it has seen, so the symlinks made it drop config2py/data/skills from the sdist, and so from the wheel CI builds from it. test_packaging.py checks the sdist file list whenever hatchling is importable. - .claude/CLAUDE.md: agent-layer notes, the #16 follow-up issues, the dependents gate, and current test counts. Both skills pass skill.validate with no issues. --- .claude/CLAUDE.md | 69 +++---- .claude/skills/config2py-dev | 1 + .claude/skills/config2py-quickstart | 1 + .../data/skills/config2py-quickstart/SKILL.md | 184 ++++++++++++++++++ config2py/tests/test_packaging.py | 42 ++++ pyproject.toml | 9 + skills/config2py-dev/SKILL.md | 65 +++++++ 7 files changed, 325 insertions(+), 46 deletions(-) create mode 120000 .claude/skills/config2py-dev create mode 120000 .claude/skills/config2py-quickstart create mode 100644 config2py/data/skills/config2py-quickstart/SKILL.md create mode 100644 config2py/tests/test_packaging.py create mode 100644 skills/config2py-dev/SKILL.md diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index fd824f8..f8d3630 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -1,65 +1,42 @@ # config2py -Tools to read and write configurations from various sources and formats: layered -lookup (env vars, files, user prompt), a `ConfigStore`/`ConfigReader` data-object -layer over `configparser`, extension-based codecs, and synchronized key-value -stores with automatic persistence. +Tools to read and write configurations from various sources and formats: layered lookup (env vars, files, user prompt), a `ConfigStore`/`ConfigReader` data-object layer over `configparser`, extension-based codecs, per-app XDG/Windows folders, and synchronized key-value stores with automatic persistence. ## Module map (`config2py/`) -- `base.py` — `get_config`, `user_gettable`, `sources_chainmap`: the layered - lookup chain a config value is resolved through. -- `tools.py` — `config_getter`/`simple_config_getter` (the headline "ask user for - missing key -> save to disk" flow), `get_configs_local_store`, `local_configs`, - `configs`, `Configs`, `extract_exports`. -- `util.py` — `envvar` (like `os.environ` but hides secrets on display), - `ask_user_for_input`, `get_app_config_folder`/`get_app_data_folder`/`get_app_folder`. -- `s_configparser.py` — `ConfigStore`/`ConfigReader`: a data-object layer over - stdlib `configparser`. -- `codecs.py` — extension-based codec registry (bytes <-> JSON-friendly Python - types), keyed by file extension. -- `sync_store.py` — `MutableMapping`s that auto-sync changes to a backing store. -- `errors.py` — `Config2PyError` and subclasses. +- `base.py`: `get_config`, `user_gettable`, `sources_chainmap`. This is the layered lookup chain a config value is resolved through. +- `tools.py`: `config_getter`/`simple_config_getter` (the headline "ask user for missing key, then save to disk" flow), `get_configs_local_store`, `local_configs`, `configs`, `Configs`, `extract_exports`. `config_getter` and `local_configs` are built at import time. +- `util.py`: `envvar` (like `os.environ` but hides values from `repr`), `ask_user_for_input` and `looks_like_secret` (masking), the app folders (`app_folder_standards`, `get_app_folder`, `get_app_config_folder`, `get_app_data_folder`, `AppData`, `ensure_seeded`), and `secure_open`/`secure_makedirs`. +- `s_configparser.py`: `ConfigStore`/`ConfigReader`, a data-object layer over stdlib `configparser`. +- `codecs.py`: an extension-based codec registry (bytes to and from JSON-friendly Python types). +- `sync_store.py`: `MutableMapping`s that auto-sync changes to a backing store. It deliberately has no intra-package imports. +- `errors.py`: `Config2PyError` and `ConfigNotFound`. +- `data/skills/config2py-quickstart/`: the consumer skill. It ships in the wheel. + +## Agent layer + +- Skills: the consumer skill is `config2py/data/skills/config2py-quickstart/SKILL.md` (pip-shipped). The dev skill is `skills/config2py-dev/SKILL.md` (repo only). `.claude/skills/` are relative symlinks to them. +- Because of those symlinks, `pyproject.toml` has `[tool.hatch.build.targets.sdist] exclude = [".claude/skills"]` with `skip-excluded-dirs = true`. Without it, hatchling drops the real skill folders from the sdist. `config2py/tests/test_packaging.py` checks this when hatchling is installed. +- `config2py/tests/test_docs_examples.py` runs every ```` ```python ```` block and every `>>>` example of `README.md` and of the consumer skill, in a subprocess with a sandboxed HOME and closed stdin. Mark a block that must prompt with `` on the line before its fence. +- The README section between the `epythet:agentic-readme` markers is generated by `epythet ai-readme-check . --write`. Don't hand-edit inside the markers. ## Tests & lint (verified) ```bash uv venv .venv && uv pip install -e . pytest ruff .venv/bin/pytest config2py --doctest-modules \ - -o doctest_optionflags='ELLIPSIS IGNORE_EXCEPTION_DETAIL' -q # 132 passed, 4 skipped + -o doctest_optionflags='ELLIPSIS IGNORE_EXCEPTION_DETAIL' -q # 154 passed, 5 skipped .venv/bin/ruff check . ``` -Or `wads ci-local`. **Gotchas already documented in `pyproject.toml` comments:** -`testpaths = ["config2py"]`, not `"tests"` — there is no top-level `tests/` dir, -only `config2py/tests/`. `doctest_optionflags` here is kept in sync with what CI's -`run-tests-uv` action passes on the command line (which *overrides* this -setting) — notably CI does **not** set `NORMALIZE_WHITESPACE`, so a doctest that -passes locally with the bare `pytest` config can still fail in CI; use the -command above (matching CI) to catch that before pushing. + +Or `wads ci-local`. **Gotchas already documented in `pyproject.toml` comments:** `testpaths = ["config2py"]`, not `"tests"`. There is no top-level `tests/` dir, only `config2py/tests/`. `doctest_optionflags` there is kept in sync with what CI's `run-tests-uv` action passes on the command line, which *overrides* the setting. Notably, CI does **not** set `NORMALIZE_WHITESPACE`, so a doctest that passes locally with the bare `pytest` config can still fail in CI. Use the command above, which matches CI, to catch that before pushing. ## Invariants / known gaps (see open issues before "fixing" these) -- **`DFLT_MASKING_INPUT = looks_like_secret`** in `util.py` (since - [#13](https://github.com/i2mint/config2py/issues/13)) — `ask_user_for_input` - masks prompts that mention something secret-looking (`API_KEY`, `TOKEN`, - `PASSWORD`, ...) and echoes the others. `mask_input` accepts a bool or a - `prompt -> bool` predicate. When stdin is not a terminal and `getpass.getpass` - is the stdlib one, masked reads fall back to `input` (stdlib `getpass` reads - `/dev/tty`, not stdin, so it would ignore piped input or hang). A frontend's - replacement `getpass` (Jupyter) is always used. Tests in - `config2py/tests/test_masking.py`; don't change the default without re-running - the dependents check. -- [i2mint/config2py#16](https://github.com/i2mint/config2py/issues/16) — an - omnibus of smaller audit findings (docstring overclaims, a broad fallback, - the pickle codec, import-time side effects) — read before touching those areas. -- [i2mint/config2py#22](https://github.com/i2mint/config2py/pull/22) (open) — - removes the vestigial `setup.cfg`, adds `[tool.wads.ci]`, migrates CI to the - reusable-workflow stub; currently blocked on a hosted-CI `action_required` - anomaly (see [#23](https://github.com/i2mint/config2py/issues/23)). Until it - lands, CI here is the inline uv workflow using discrete `i2mint/wads` actions. +- **`DFLT_MASKING_INPUT = looks_like_secret`** in `util.py` (since [#13](https://github.com/i2mint/config2py/issues/13)). `ask_user_for_input` masks prompts that mention something secret-looking (`API_KEY`, `TOKEN`, `PASSWORD`, ...) and echoes the others. `mask_input` accepts a bool or a `prompt -> bool` predicate. When stdin is not a terminal and `getpass.getpass` is the stdlib one, masked reads fall back to `input`: stdlib `getpass` reads `/dev/tty`, not stdin, so it would ignore piped input or hang. A frontend's replacement `getpass` (Jupyter) is always used. The tests are in `config2py/tests/test_masking.py`. Don't change the default without re-running the dependents check. +- The #16 audit was split into follow-ups, each with a plan. Read the matching one before touching an area: [#25](https://github.com/i2mint/config2py/issues/25) (broad `(Exception,)` fallback), [#26](https://github.com/i2mint/config2py/issues/26) (import-time folder creation), [#27](https://github.com/i2mint/config2py/issues/27) (typo'd paths give silent empty configs), [#28](https://github.com/i2mint/config2py/issues/28) (`os.path.sep` path sniffing), [#29](https://github.com/i2mint/config2py/issues/29) (pickle decoder opt-in). Also [#30](https://github.com/i2mint/config2py/issues/30) (the masking toggle drops `egress`, which interacts with `oa`) and [#33](https://github.com/i2mint/config2py/issues/33) (configs-store value files are written `0o644`). +- [#22](https://github.com/i2mint/config2py/pull/22) (open) removes the vestigial `setup.cfg`, adds `[tool.wads.ci]`, and migrates CI to the reusable-workflow stub. It is blocked on a hosted-CI `action_required` anomaly (see [#23](https://github.com/i2mint/config2py/issues/23)). Until it lands, CI here is the inline uv workflow using discrete `i2mint/wads` actions. ## Dependents -`accompy`, `aix`, `arioso`, `aw`, `brand`, `mood`, `py2store`, `tonal`, and 25 -others (see `fleet_dependents.json`) import this package — check their tests -before changing `get_config`, `simple_config_getter`, or any default. +`accompy`, `aix`, `arioso`, `aw`, `brand`, `mood`, `py2store`, `tonal`, and about 25 others import this package. The full list is the maintainer's local `fleet_dependents.json`, which is not in this repo. Check their tests before changing `get_config`, `simple_config_getter`, or any default. The publicly clonable ones checked in cloud sessions are `i2mint/py2store`, `i2mint/xdol` and `thorwhalen/oa`. At baseline, `xdol` has 2 doctest-format failures and `oa` has 3 tests that need a real OpenAI key. diff --git a/.claude/skills/config2py-dev b/.claude/skills/config2py-dev new file mode 120000 index 0000000..972f192 --- /dev/null +++ b/.claude/skills/config2py-dev @@ -0,0 +1 @@ +../../skills/config2py-dev \ No newline at end of file diff --git a/.claude/skills/config2py-quickstart b/.claude/skills/config2py-quickstart new file mode 120000 index 0000000..da738c8 --- /dev/null +++ b/.claude/skills/config2py-quickstart @@ -0,0 +1 @@ +../../config2py/data/skills/config2py-quickstart \ No newline at end of file diff --git a/config2py/data/skills/config2py-quickstart/SKILL.md b/config2py/data/skills/config2py-quickstart/SKILL.md new file mode 100644 index 0000000..59c9c2c --- /dev/null +++ b/config2py/data/skills/config2py-quickstart/SKILL.md @@ -0,0 +1,184 @@ +--- +name: config2py-quickstart +description: >- + Use config2py to get configuration values and secrets into Python code: + layered lookup (environment variables, then a local config folder, then an + interactive prompt whose answer is saved), per-app config/data/cache/state + folders that follow XDG and Windows conventions, JSON/INI/YAML/TOML files + exposed as dicts that save on write, and extension-based codecs. Use when + code needs an API key or setting "from the environment or a config file", + when prompting a user once for a value and remembering it, when deciding + where an app should store its files, or when reading and writing config + files as mappings. Triggers: config2py, config_getter, simple_config_getter, + get_config, user_gettable, ask_user_for_input, get_app_folder, AppData, + FileStore, JsonStore, ConfigStore, ConfigReader, decode_by_extension. +metadata: + audience: users +--- + +# config2py quickstart + +`pip install config2py`. Optional format extras: `config2py[yaml]`, `[toml]`, `[env]`, `[json5]`, `[properties]`, or `[all-codecs]`. + +## Pick the entry point + +| You need | Use | +|---|---| +| A value from an env var, else a saved local config, else ask once and save | `config_getter`, or your own via `simple_config_getter(...)` | +| Your own ordered sources (dicts, callables, stores) | `get_config(key, sources, default=...)` | +| Ask the user for a value and store the answer | `user_gettable(save_to=store)` | +| Where app `X` should keep config, data, cache or state files | `get_app_folder("X", folder_kind=...)`, or the `AppData("X")` facade | +| A JSON/INI/YAML/TOML file as a dict that is saved on every write | `FileStore`, `JsonStore` | +| An `.ini` file as a mapping of sections | `ConfigStore` (read-write), `ConfigReader` | +| Bytes to Python objects (and back) chosen by file extension | `config2py.codecs` | + +## Layered lookup: `get_config` + +Sources are tried in order and the first one that has the key wins. A source is any mapping, or any callable `key -> value` that raises when it doesn't have the key. Pass `sources` alone to get a reusable getter. + +```python +import os +from config2py import get_config + +defaults = {"MODEL": "small", "TIMEOUT": "30"} +get = get_config(sources=[{"MODEL": "large"}, defaults]) +assert get("MODEL") == "large" +assert get("TIMEOUT") == "30" +assert get("NOT_SET", default=None) is None + + +def from_vault(key): # any callable works as a source + raise KeyError(key) # "not here": move on to the next source + + +get = get_config( + sources=[from_vault, os.environ, defaults], + config_not_found_exceptions=(KeyError, LookupError), # see the gotchas +) +assert get("TIMEOUT") == "30" +``` + +`get_config` also takes `egress=lambda key, value: ...` to post-process (or cache) what it found, and `val_is_valid=` to skip values such as `None` or `""`. + +## The ready-made getter: `simple_config_getter` + +`simple_config_getter(src)` looks in environment variables, then in a local store, and optionally asks the user and saves the answer in that store. `src` is an app name (store in `~/.config//configs/`), a directory path containing a separator (a folder of one text file per key), or an `.ini`/`.cfg` file. The package-level `config_getter` is `simple_config_getter()` with the defaults (app name `config2py`). + +```python +import os +import tempfile +from config2py import simple_config_getter + +folder = tempfile.mkdtemp() + os.sep # a trailing separator marks a folder path +get = simple_config_getter(folder, ask_user_if_key_not_found=False) +get.configs["DB_URL"] = "sqlite:///app.db" # writes the file /DB_URL +assert get("DB_URL") == "sqlite:///app.db" + +os.environ["DB_URL"] = "postgres://prod" # env vars are consulted first +assert get("DB_URL") == "postgres://prod" +del os.environ["DB_URL"] +``` + +## Ask the user once, remember the answer + +`user_gettable(save_to=...)` is a mapping that prompts for any key and saves non-empty answers into `save_to` (any `MutableMapping`, or a `(key, value)` function). Put it last in a source list, after the store it saves to. `user_asker` is swapped here so the example runs unattended. + +```python +from config2py import get_config, user_gettable + +saved = {} +ask = user_gettable(save_to=saved, user_asker=lambda prompt: "typed value") +get = get_config(sources=[saved, ask]) +assert get("API_TOKEN") == "typed value" # asked, then saved +assert saved == {"API_TOKEN": "typed value"} +assert get("API_TOKEN") == "typed value" # now found in `saved`, not asked again +``` + +The default asker is `ask_user_for_input`. It masks what the user types when the prompt looks secret (it mentions `key`, `token`, `pass`, `pwd`, `secret`, `api`, `auth`, `credential` or `private`) and echoes it otherwise. Force a choice with `ask_user_for_input(prompt, mask_input=True)` (or `False`), or pass your own `prompt -> bool` predicate. When stdin is piped, the answer is read from stdin. + +## App folders + +```python +import os +from config2py import get_app_folder + +config_dir = get_app_folder("myapp", folder_kind="config") +data_dir = get_app_folder("myapp", folder_kind="data", ensure_exists=True) +assert os.path.basename(config_dir) == "myapp" and os.path.isdir(data_dir) +``` + +- Kinds are `config` (settings, API keys), `data` (files users would miss), `cache` (disposable), `state` (logs, history) and `runtime` (sockets, PID files). +- On Linux and macOS: `~/.config`, `~/.local/share`, `~/.cache`, `~/.local/state`, then `$XDG_RUNTIME_DIR` or `/tmp`. macOS uses these XDG paths, not `~/Library`. On Windows: `%APPDATA%`, `%LOCALAPPDATA%`, `%LOCALAPPDATA%\Temp`, `%LOCALAPPDATA%`, `%TEMP%`. +- Precedence: `CONFIG2PY__DIR` (for example `CONFIG2PY_DATA_DIR`), then the platform variable (`XDG_DATA_HOME`, `LOCALAPPDATA`, ...), then the default. These name the *root*; the app name is appended. +- Folders config2py creates are owner-only (`0o700`). `FileStore`, `ConfigStore` and `AppData` write files as `0o600`. + +`AppData("myapp")` wraps this: `.app_folder(folder_kind=...)`, `.get_artifact_dir("runs")`, and `.get_resource(name)` / `.get_config(name)`, which copy a default file shipped in `myapp/_seed_data/{resources,config}/` on first access and never overwrite user edits. + +## Files as dicts: `FileStore`, `JsonStore` + +The format comes from the extension: `.json`, `.ini`/`.cfg`, `.yaml`/`.yml` (needs `pyyaml`) and `.toml` (needs `tomli-w` to write). Every write saves the file, and a `with` block batches the writes into one save. + +```python +import json +import os +import tempfile +from config2py import FileStore + +path = os.path.join(tempfile.mkdtemp(), "settings.json") +# create_file_content makes a missing file (without it, a missing file raises) +settings = FileStore(path, create_file_content=dict) +settings["theme"] = "dark" # saved immediately +with settings: # one save, on exit + settings["a"] = 1 + settings["b"] = 2 + +db = FileStore(path, key_path="database", create_key_path_content=dict) +db["host"] = "localhost" # only touches the "database" section +assert json.load(open(path)) == { + "theme": "dark", + "a": 1, + "b": 2, + "database": {"host": "localhost"}, +} +``` + +`key_path` also takes a dotted path (`"app.settings"`) or a tuple. Use `register_extension(".ext", loader, dumper)` to add a format, and `SyncStore(loader, dumper)` to back the mapping with anything else. + +## INI files: `ConfigStore` + +```python +import os +import tempfile +from config2py import ConfigStore + +path = os.path.join(tempfile.mkdtemp(), "app.ini") +with open(path, "w") as f: + f.write("[db]\nhost = localhost\n") + +store = ConfigStore(path) +store["cache"] = {"ttl": "60"} # assigning a section saves the file +with store: # edits *inside* a section are saved when the block exits + store["db"]["port"] = "5432" +assert "port = 5432" in open(path).read() +``` + +## Codecs by extension + +```python +from config2py.codecs import decode_by_extension, encode_by_extension, register_codec + +data = encode_by_extension("settings.json", {"a": 1}) +assert decode_by_extension("settings.json", data) == {"a": 1} +register_codec(".upper", encoder=lambda s: s.upper().encode(), decoder=bytes.decode) +assert decode_by_extension("x.upper", encode_by_extension("x.upper", "hi")) == "HI" +``` + +## Gotchas + +- **Prompts in production.** `ask_user_if_key_not_found=None` (the default) prompts whenever `is_repl()` is true, and that includes `python -c` and notebooks. Pass `ask_user_if_key_not_found=False` in services, CI and libraries. +- **Import-time side effect.** `import config2py` creates `~/.config/config2py/configs/`, and default folders are resolved at import. In tests, point `HOME`/`XDG_CONFIG_HOME` (or `CONFIG2PY_CONFIG_DIR`) at a temp dir *before* the first import. +- **Broad fallback.** `config_not_found_exceptions` defaults to `(Exception,)`, so a bug or network error in a callable source silently falls through to the next source. Narrow it when you can. +- **Plain text, not a vault.** Values saved by the prompt flow are plain one-file-per-key text files, currently written with the umask default and protected only by their `0o700` folder when config2py created it (i2mint/config2py#33). `envvar` hides values from `repr()` only, not from iteration or pickling. +- **Silent empties.** `ConfigReader("typo.ini")` and `extract_exports("typo/.env")` return empty results instead of raising, so check that the path exists first. +- **Pickle.** `.pkl`/`.pickle` decode with `pickle.loads`, so never decode untrusted bytes under those extensions. +- **Bare names are app names.** `get_configs_local_store("configs")` means the app `configs`, even if `./configs` exists. Pass `"./configs"` to use the directory. diff --git a/config2py/tests/test_packaging.py b/config2py/tests/test_packaging.py new file mode 100644 index 0000000..90588fa --- /dev/null +++ b/config2py/tests/test_packaging.py @@ -0,0 +1,42 @@ +"""Packaging checks: the consumer skill must ship in the sdist (and so in the wheel). + +``.claude/skills/`` are relative symlinks to the real skill folders. hatchling +walks the project with ``followlinks=True`` and skips folders it has already seen, so +unless ``.claude/skills`` is pruned (see ``[tool.hatch.build.targets.sdist]`` in +``pyproject.toml``) the real folders are silently dropped from the sdist. +""" + +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] +CONSUMER_SKILL = "config2py/data/skills/config2py-quickstart/SKILL.md" +DEV_SKILL = "skills/config2py-dev/SKILL.md" + + +def _require_source_checkout(): + if not (REPO_ROOT / "pyproject.toml").exists(): + pytest.skip("not running from a source checkout") + + +def test_claude_skill_links_resolve_to_the_real_skills(): + _require_source_checkout() + links = { + "config2py-quickstart": CONSUMER_SKILL, + "config2py-dev": DEV_SKILL, + } + for name, target in links.items(): + link = REPO_ROOT / ".claude" / "skills" / name + if not link.is_symlink(): + pytest.skip("symlinks not materialized (e.g. Windows checkout)") + assert (link / "SKILL.md").resolve() == (REPO_ROOT / target).resolve() + + +def test_sdist_includes_the_skills(): + _require_source_checkout() + sdist = pytest.importorskip("hatchling.builders.sdist") + builder = sdist.SdistBuilder(str(REPO_ROOT)) + included = {f.relative_path.replace("\\", "/") for f in builder.recurse_included_files()} + assert CONSUMER_SKILL in included + assert DEV_SKILL in included diff --git a/pyproject.toml b/pyproject.toml index 57e9f4f..e7925fe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,6 +69,15 @@ all = [ "jproperties>=2.1.0", ] +[tool.hatch.build.targets.sdist] +# .claude/skills/ are relative symlinks to the real skill folders +# (config2py/data/skills/, skills/). hatchling walks with followlinks=True and skips +# any folder whose inode it has already seen, so without this it would visit the +# symlinks first and then drop the real folders: config2py/data/skills would be +# missing from the sdist, and so from the wheel CI builds from it. +exclude = [".claude/skills"] +skip-excluded-dirs = true + [tool.ruff] line-length = 88 target-version = "py310" diff --git a/skills/config2py-dev/SKILL.md b/skills/config2py-dev/SKILL.md new file mode 100644 index 0000000..202f6c9 --- /dev/null +++ b/skills/config2py-dev/SKILL.md @@ -0,0 +1,65 @@ +--- +name: config2py-dev +description: >- + Work on the config2py codebase itself: module map, how to run the tests the + way CI does (doctests included), the dependents gate before changing a + default, the docs-examples test that runs README and skill snippets, the + release flow (merging to master publishes to PyPI), and the invariants and + open design issues to read before "fixing" something. Use when contributing + to, reviewing, debugging or releasing config2py (i2mint/config2py), or when + a change touches get_config, simple_config_getter, ask_user_for_input, + app folders or any public default. +metadata: + audience: developers +--- + +# Working on config2py + +## Module map (`config2py/`) + +| Module | What lives there | +|---|---| +| `base.py` | `get_config` (the layered lookup), `sources_chainmap`, `FuncBasedGettableContainer` (callable to `KeyError`-raising mapping), `user_gettable` and `ask_user_for_key` (prompt, then save) | +| `tools.py` | `simple_config_getter`, `get_configs_local_store`, the import-time instances `config_getter`, `local_configs` and `configs`, plus `extract_exports` and `source_config_params` | +| `util.py` | `ask_user_for_input` (masking), `looks_like_secret`, `envvar`, app folders (`app_folder_standards`, `get_app_rootdir`, `get_app_folder`, `AppData`, `ensure_seeded`), `secure_open`/`secure_makedirs`, `is_repl` | +| `s_configparser.py` | `ConfigStore`/`ConfigReader`, mapping views over `configparser` | +| `sync_store.py` | `SyncStore`, `FileStore`, `JsonStore`, `register_extension`. It deliberately imports nothing from the package | +| `codecs.py` | Extension-keyed encoder and decoder registries (`encode_by_extension`, `decode_by_extension`, `register_codec`) | +| `errors.py` | `ConfigNotFound` and friends | +| `data/skills/` | Consumer skills shipped in the wheel. Dev skills like this one live in the top-level `skills/` | + +All OS branching for folders is in `app_folder_standards(os_name)`, which takes the OS as an argument so both platforms are testable anywhere. + +## Run the tests like CI + +```bash +uv venv .venv && . .venv/bin/activate +uv pip install -e . pytest ruff +python -m pytest config2py --doctest-modules -o doctest_optionflags='ELLIPSIS IGNORE_EXCEPTION_DETAIL' -q +ruff check . && ruff format --check . +``` + +- Doctests are a large part of the suite. CI passes `doctest_optionflags` on the command line, which overrides `pyproject.toml`, and it does not set `NORMALIZE_WHITESPACE`, so always use the flags above. +- `testpaths = ["config2py"]`: the tests live in `config2py/tests/`, and there is no top-level `tests/`. +- `config2py/tests/test_docs_examples.py` runs every ```` ```python ```` block of `README.md` and of `config2py/data/skills/config2py-quickstart/SKILL.md` in a subprocess with a sandboxed HOME and closed stdin. When you edit either document, keep its blocks runnable, or put `` on the line before a fence that has to prompt or needs placeholders. +- `wads ci-local` (from `pip install wads`) replays the whole CI job: ruff, pytest on each configured Python, then `uv build`. It is the pre-merge gate while hosted CI on the stub workflow is blocked (#23). + +## Test isolation + +`import config2py` creates `~/.config/config2py/configs/`, and folder defaults such as `DFLT_CONFIG_FOLDER` are computed at import. A test that needs a clean home must point `HOME` and the `XDG_*` variables (or `CONFIG2PY_*_DIR`) at a temp dir in a subprocess, as `test_docs_examples.py` does. Setting them after import has no effect on the module-level defaults. Prompt tests patch both `builtins.input` and `getpass.getpass`; `config2py/tests/utils_for_testing.py` has `user_input_patch` for that. + +## Before changing a default or public behaviour + +About 30 packages depend on config2py (for example `py2store`, `xdol`, `oa`, `aix`, `tonal`). A merge to master publishes to PyPI, so: + +1. Grep the dependents for the symbol you are changing. +2. Run their test suites against your working tree (`uv pip install -e && uv pip install -e ` in a scratch venv), before and after your change, and compare pass counts. Known pre-existing failures: `xdol` has 2 doctest-format failures and `oa` has 3 tests that need a real OpenAI key. +3. Prefer additive, keyword-only parameters. For example, `mask_input` accepts a bool or a `prompt -> bool` predicate, and explicit booleans keep their old meaning. + +Read these open issues before touching the matching area: #25 (the broad `(Exception,)` fallback), #26 (import-time folder creation), #27 (typo'd paths give silent empty configs), #28 (`os.path.sep` path sniffing), #29 (the pickle decoder), #30 (the masking toggle drops `egress`), #33 (file modes of the configs store), #12 and #17 (folder layout and platformdirs). + +## Release flow + +- Never edit `version`. CI's publish job on master bumps it, builds, uploads to PyPI, then commits the bump back and tags it. +- Packaging is `pyproject.toml` with hatchling (`>=1.27`, for the PEP 639 SPDX `license`). The vestigial `setup.cfg` and the move to the reusable wads CI stub are in #22. +- Commit messages and PR text must not contain CI marker strings (the skip-CI and publish markers), because squash merges can carry them into master. From 913d2e04deec1128d330fc31cb5a4488004032b4 Mon Sep 17 00:00:00 2001 From: Thor Whalen <1906276+thorwhalen@users.noreply.github.com> Date: Sun, 27 Sep 2026 08:59:06 +0000 Subject: [PATCH 5/6] docs: AI-first README with runnable examples; fix broken SyncStore examples - Top: what the package does, a link to the human section at the end, then "What an agent can do" with a minimal runnable example and the pip route to the bundled skill. The epythet agentic section was regenerated with `epythet ai-readme-check . --write` and now lists the skills and CLAUDE.md. - Fixed examples: FileStore on a fresh file and nested key_path need create_file_content / create_key_path_content, the register_extension import is now config2py.sync_store, placeholder functions are replaced by runnable ones, and a duplicated paragraph is removed. The prompt now documents the masking behaviour. - New section at the end for human developers: dev setup, design rationale, contributing, where to ask. - config2py/tests/test_docs_examples.py runs every python block and >>> example of README.md and of the consumer skill, in a subprocess with a sandboxed HOME and closed stdin. Blocks that must prompt carry a marker. The old README examples fail this test. Fixes #32 --- README.md | 276 ++++++++++++++------------ config2py/tests/test_docs_examples.py | 129 ++++++++++++ 2 files changed, 276 insertions(+), 129 deletions(-) create mode 100644 config2py/tests/test_docs_examples.py diff --git a/README.md b/README.md index 469b2ff..ac01487 100644 --- a/README.md +++ b/README.md @@ -1,22 +1,52 @@ # config2py -Simplified reading and writing configurations from various sources and formats. +Get configuration values and secrets into Python code through layered sources: environment variables, then a local config folder, then a one-time prompt whose answer is saved for next time. It also gives you per-app config/data/cache folders that follow XDG and Windows conventions, config files you edit as dicts that save on every write, and extension-based codecs. -To install: ```pip install config2py``` +Still typing code yourself? Skip to [For humans who still hand-edit their dotfiles](#for-humans-who-still-hand-edit-their-dotfiles). -[Documentation](https://i2mint.github.io/config2py/) +To install: `pip install config2py` · [Documentation](https://i2mint.github.io/config2py/) + +# What an agent can do with config2py + +- Resolve an API key or setting from environment variables, then a saved local config, and in interactive sessions only, a one-time prompt that masks secret-looking keys and saves the answer. +- Compose any lookup chain over dicts, callables and [dol](https://github.com/i2mint/dol) stores with `get_config`. +- Find the right per-user folder for an app's config, data, cache or state files, overridable by environment variables. +- Edit JSON, INI, YAML or TOML files as dicts that save on write (`FileStore`, `ConfigStore`), and encode or decode bytes by file extension. + +A minimal, non-interactive example: + +```python +from config2py import simple_config_getter + +get = simple_config_getter("myapp", ask_user_if_key_not_found=False) +get.configs["MYAPP_MODEL"] = "small" # saved as a file in ~/.config/myapp/configs/ +assert get("MYAPP_MODEL") == "small" # an env var named MYAPP_MODEL would win +``` + +The consumer skill below also ships inside the wheel. After `pip install config2py`, its folder is `python -c "import config2py, os; print(os.path.join(config2py.__path__[0], 'data', 'skills'))"`, ready to link into your agent's skills directory (for example with `skill link-skills `). -## For AI agents +# For AI agents -`config2py` ships no skills or subagents of its own — it's the thing that reads -*your* agent's config, not the other way around. If you're an agent that needs to -fetch a value from an environment variable, a local file, or a user prompt without -three different codecs and a `configparser` incantation, this is your package. +`config2py` ships tooling for coding agents. If you are one, start here. -**The documentation, machine-readable**: [`llms.txt`](https://i2mint.github.io/config2py/llms.txt) indexes every page; [`config2py.md`](https://i2mint.github.io/config2py/config2py.md) is the whole documentation in one file; every page has a `.md` twin; [`objects.inv`](https://i2mint.github.io/config2py/objects.inv) maps symbols to URLs. +**Skills** ([Agent Skills](https://agentskills.io) format), for any agent host. Install one with `gh skill`: +```bash +gh skill install i2mint/config2py config2py-quickstart --agent claude-code # or copilot, cursor, codex, gemini +``` + +| Skill | Use it to | +|---|---| +| `config2py-quickstart` | use config2py to get configuration values and secrets into Python code | +| `config2py-dev` | work on the config2py codebase itself | + +The same skills are inside the wheel, under `config2py/data/skills/`. + +**Instruction files**: `.claude/CLAUDE.md` (Claude Code). -If you are a control freak (human or otherwise), the rest of this README is written for you, starting at [The cherry on top: config_getter](#the-cherry-on-top-config_getter). +**The documentation, machine-readable**: [`llms.txt`](https://i2mint.github.io/config2py/llms.txt) indexes every page; [`config2py.md`](https://i2mint.github.io/config2py/config2py.md) is the whole documentation in one file; every page has a `.md` twin; [`objects.inv`](https://i2mint.github.io/config2py/objects.inv) maps symbols to URLs. The full list, with install lines, is on the site's [For AI agents](https://i2mint.github.io/config2py/ai-agents.html) page. + +If you are a human, the rest of this README is written for you, starting at [The cherry on top: config_getter](#the-cherry-on-top-config_getter). # The cherry on top: config_getter @@ -25,8 +55,7 @@ If you are a control freak (human or otherwise), the rest of this README is writ from config2py import config_getter ``` -Let's start with an extremely convenient, no questions asked, object. -Later, we'll look under the hood to show the many tools that support it, and can be shaped to fit many desired behaviors. +Let's start with an extremely convenient, no questions asked, object. Later, we'll look under the hood to show the many tools that support it, and can be shaped to fit many desired behaviors. What `config2py.config_getter(key)` will do is: * search for `key` in your environment variables, and if not found... @@ -44,43 +73,31 @@ config_getter("HOME") # if you are using Linux/MacOS '/Users/thorwhalen' +Now, normally all systems come with a `HOME` environment variable (or a `USERPROFILE` on windows), so the above should always work fine. But see what happens if you ask for a key that is not an environment variable: -Now, normally all systems come with a `HOME` environment variable (or a `USERPROFILE` on windows), so the above should always work fine. -But see what happens if you ask for a key that is not an environment variable: - - + ```python my_config_val = config_getter("_TEST_NON_EXISTING_KEY_") # triggers a user input dialog # ... I enter 'my config value' in the dialog, and then... -``` - - -```python my_config_val ``` 'my config value' - +When the key looks like a secret (its name mentions `key`, `token`, `password`, `secret`, `api` and the like), what you type is masked, as with a password prompt. Other values, such as file paths, are echoed so you can see them. You can force either behaviour with `ask_user_for_input(..., mask_input=True)` (or `False`). But if I do that again (even on a different day, somewhere else (on my same computer), in a different session), it will get me the value I entered in the user input dialog. - + ```python -my_config_val = config_getter( - "_TEST_NON_EXISTING_KEY_" -) # does not trigger input dialog +# This time, no input dialog: +my_config_val = config_getter("_TEST_NON_EXISTING_KEY_") my_config_val ``` 'my config value' - - -And of course, we give you a means to delete that value, since `config_getter` has a `local_configs` mapping (think `dict`) to the local files where it has been stored. -You can do all the usual stuff you do with a `dict` (except the effects will be on local files), -like list the keys (with `list(.)`), get values for a key (with `.[key]`), ask for the number of keys (`len(.)`), and, well, delete stuff: - +And of course, we give you a means to delete that value, since `config_getter` has a `configs` mapping (think `dict`) to the local files where it has been stored. You can do all the usual stuff you do with a `dict` (except the effects will be on local files), like list the keys (with `list(.)`), get values for a key (with `.[key]`), ask for the number of keys (`len(.)`), and, well, delete stuff: ```python if "_TEST_NON_EXISTING_KEY_" in config_getter.configs: @@ -94,23 +111,19 @@ This tool allows you to: This is very convenient situation where user input (via things like `__builtins__.input` or `getpass.getpass` etc) is available. But **you should not use this to manage configurations/resources anywhere were there's not a user to see and respond to the builtin user input dialog** -Don't fret though, this `config_getter` is just our no-BS entry point to much more. -Let's have a slight look under its hood to see what else we can do with it. +Don't fret though, this `config_getter` is just our no-BS entry point to much more. Let's have a slight look under its hood to see what else we can do with it. And of course, if you're that type, you can already have a look at [the documentation](https://i2mint.github.io/config2py/) - ## `simple_config_getter`: Controlling your config_getter a bit more -If you look up for the definition of the `config_getter` function you imported above, you'll find this: `config_getter = simple_config_getter()`. -That is, it was created by `simple_config_getter` with its default arguments. -Let's have a look at what these are. +If you look up for the definition of the `config_getter` function you imported above, you'll find this: `config_getter = simple_config_getter()`. That is, it was created by `simple_config_getter` with its default arguments. Let's have a look at what these are. In fact, `simple_config_getter` is a function to make configuration getters that ressemble the one we've seen above: image -But where you can control what the central store (by default a local configuration files store) is, and whether to first search in environment variables or not, and whether to ask the user for the value, if not found before, or not. +But where you can control what the central store (by default a local configuration files store) is, and whether to first search in environment variables or not, and whether to ask the user for the value, if not found before, or not. ```python from config2py import simple_config_getter, get_configs_local_store @@ -120,31 +133,29 @@ print(*str(Sig(simple_config_getter)).split(","), sep="\n") ``` (configs_src: str = '.../.config/config2py/configs' - * - first_look_in_env_vars: bool = True - ask_user_if_key_not_found: bool = None - config_store_factory: Callable = ) + * + first_look_in_env_vars: bool = True + ask_user_if_key_not_found: bool = None + config_store_factory: collections.abc.Callable = ) `first_look_in_env_vars` specifies whether to look into environment variables first, or not. -`ask_user_if_key_not_found` specifies whether to ask the user if a configuration key is not found. The default is `None`, which will result in checking if you're running in an interactive environment or not. -When you use `config2py` in production though, you should definitely specify `ask_user_if_key_not_found=False` to make that choice explicit. +`ask_user_if_key_not_found` specifies whether to ask the user if a configuration key is not found. The default is `None`, which will result in checking if you're running in an interactive environment or not. When you use `config2py` in production though, you should definitely specify `ask_user_if_key_not_found=False` to make that choice explicit. -The `configs_src` default is automatically set to be the `config2py/configs` folder of your system's config directory (following XDG standards on Unix/Linux/macOS). You can override this with environment variables like `CONFIG2PY_CONFIG_DIR`, `CONFIG2PY_DATA_DIR`, etc., or the standard XDG variables. +The `configs_src` default is automatically set to be the `config2py/configs` folder of your system's config directory (following XDG standards on Unix/Linux/macOS). You can override this with environment variables like `CONFIG2PY_CONFIG_DIR`, `CONFIG2PY_DATA_DIR`, etc., or the standard XDG variables. Your central store will be `config_store_factory(configs_src)`, and since you can also specify `config_store_factory`, you have total control over the store. The default `config_store_factory` is `get_configs_local_store` which will give you a locally persisted store where if `configs_src`: -* is a directory, it's assumed to be a folder of text files. +* is a directory (a path containing a separator), it's assumed to be a folder of text files. * is a file, it's assumed to be an ini or cfg file. * is a string, it's assumed to be an app name, from which to create a config folder for with the default method - # Setting the config key search path -If you check out the code for `simple_config_getter`, you'll find that all it it is simply setting the `sources` argument for the `get_config` function. -Something more or less like: +If you check out the code for `simple_config_getter`, you'll find that all it it is simply setting the `sources` argument for the `get_config` function. Something more or less like: + ```python configs = config_store_factory(configs_src) source = [ @@ -155,21 +166,19 @@ source = [ config_getter = get_config(sources=source) ``` -So you see that you can easily define your own sources for configs, and in what order they should be searched. If you don't want that "ask the user for the value" thing, you can just remove the `user_gettable(local_configs)` part. If you wanted instead to add a place to look before the environment variables -- say, you want to look in to local variables of the scope the config getter is **defined** (not called), you can stick `locals()` in front of the `os.environ`. - -So you see that you can easily define your own sources for configs, and in what order they should be searched. If you don't want that "ask the user for the value" thing, you can just remove the `user_gettable(local_configs)` part. If you wanted instead to add a place to look before the environment variables -- say, you want to look in to local variables of the scope the config getter is **defined** (not called), you can stick `locals()` in front of the `os.environ`. +So you see that you can easily define your own sources for configs, and in what order they should be searched. If you don't want that "ask the user for the value" thing, you can just remove the `user_gettable(configs)` part. If you wanted instead to add a place to look before the environment variables -- say, you want to look in to local variables of the scope the config getter is **defined** (not called), you can stick `locals()` in front of the `os.environ`. Let's work through a custom-made `config_getter`. ```python +import os +import tempfile from config2py import get_config, user_gettable from dol import TextFiles -import os -my_configs = TextFiles( - "~/.my_configs/" -) # Note, to run this, you'd need to have such a directory! -# (But you can also use my_configs = dict() if you want.) +# A folder of text files: TextFiles("~/.my_configs/") would do too, if that directory +# exists (and a plain dict() works as well). Here, a fresh temporary folder: +my_configs = TextFiles(tempfile.mkdtemp() + os.sep) config_getter = get_config( sources=[locals(), os.environ, my_configs, user_gettable(my_configs)] ) @@ -177,27 +186,22 @@ config_getter = get_config( Now let's see what happens when we do: + ```python config_getter("SOME_CONFIG_KEY") ``` -Well, it will first look in `locals()`, which is a dictionary containing local variables -where the `config_getter` was **defined** (careful -- not called!!). -This is desirable sometimes when you define your `config_getter` in a module that has other python variables you'd like to use. +Well, it will first look in `locals()`, which is a dictionary containing local variables where the `config_getter` was **defined** (careful -- not called!!). This is desirable sometimes when you define your `config_getter` in a module that has other python variables you'd like to use. -Assuming it doesn't find such a key in `locals()` it goes on to try to find it in -`os.environ`, which is a dict containing system environment variables. +Assuming it doesn't find such a key in `locals()` it goes on to try to find it in `os.environ`, which is a dict containing system environment variables. -Assuming it doesn't find it there either (that is, doesn't find a file with that name in -the directory `~/.my_configs/`), it will prompt the user to enter the value of that key. -The function finally returns with the value that the user entered. +Assuming it doesn't find it there either (that is, doesn't find a file with that name in the `my_configs` directory), it will prompt the user to enter the value of that key. The function finally returns with the value that the user entered. But there's more! -Now look at what's in `my_configs`! -If you've used `TextFiles`, look in the folder to see that there's a new file. -Either way, if you do: +Now look at what's in `my_configs`! If you've used `TextFiles`, look in the folder to see that there's a new file. Either way, if you do: + ```python my_configs["SOME_CONFIG_KEY"] ``` @@ -206,13 +210,12 @@ You'll now see the value the user entered. This means what? This means that the next time you try to get the config: + ```python config_getter("SOME_CONFIG_KEY") ``` -It will return the value that the user entered last time, without prompting the -user again. - +It will return the value that the user entered last time, without prompting the user again. ## SyncStore: Auto-Syncing Key-Value Stores @@ -225,8 +228,9 @@ user again. ```python from config2py.sync_store import FileStore, JsonStore -# Auto-detected from .json extension -config = FileStore("config.json") +# Auto-detected from .json extension. create_file_content makes the file if it's +# missing (without it, a missing file raises FileNotFoundError). +config = FileStore("config.json", create_file_content=dict) config["api_key"] = "secret" # Syncs immediately # Batch operations (deferred sync) @@ -240,12 +244,15 @@ with config: ### Nested Sections ```python -# Work with specific section via key_path -db_config = FileStore("config.json", key_path="database") +# Work with specific section via key_path (create_key_path_content creates the +# section if it's missing; without it, a missing section raises KeyError) +db_config = FileStore("config.json", key_path="database", create_key_path_content=dict) db_config["host"] = "localhost" # Only affects database section # Dotted notation for deep nesting -items = FileStore("config.json", key_path="app.settings.items") +items = FileStore( + "config.json", key_path="app.settings.items", create_key_path_content=dict +) items["item1"] = "value" ``` @@ -259,10 +266,20 @@ Auto-detected by extension: Register custom formats: ```python -from sync_store import register_extension +from config2py.sync_store import FileStore, register_extension + + +def load_kv(text: str) -> dict: + return dict(line.split("=", 1) for line in text.splitlines() if line) -register_extension(".custom", my_loader, my_dumper) -store = FileStore("data.custom") + +def dump_kv(data: dict) -> str: + return "\n".join(f"{k}={v}" for k, v in data.items()) + + +register_extension(".kv", load_kv, dump_kv) +store = FileStore("data.kv", create_file_content=dict) +store["answer"] = "42" ``` ### Custom Backing Storage @@ -270,18 +287,22 @@ store = FileStore("data.custom") ```python from config2py.sync_store import SyncStore +database = {"key": "old value"} # stands in for any backing storage + # Any backing storage via loader/dumper def my_loader(): - return fetch_from_database() + return dict(database) # e.g. fetch_from_database() def my_dumper(data): - save_to_database(data) + database.clear() # e.g. save_to_database(data) + database.update(data) store = SyncStore(my_loader, my_dumper) store["key"] = "value" # Calls my_dumper +assert database == {"key": "value"} ``` ### Key Classes @@ -290,12 +311,11 @@ store["key"] = "value" # Calls my_dumper - **`FileStore`** - File-based with extension detection and key_path - **`JsonStore`** - Explicit JSON with sensible defaults - # A few notable tools you can import from config2py * `get_config`: Get a config value from a list of sources. See more below. * `user_gettable`: Create a ``GettableContainer`` that asks the user for a value, optionally saving it. -* `ask_user_for_input`: Ask the user for input, optionally masking, validating and transforming the input. +* `ask_user_for_input`: Ask the user for input, optionally masking, validating and transforming the input. By default it masks prompts that look like they ask for a secret. * `get_app_folder`: Returns the full path of a directory suitable for storing application-specific data for a given app name and folder kind (config, data, cache, state, runtime). * `get_app_config_folder`: Specialized version of `get_app_folder` for configuration files. * `get_app_data_folder`: Specialized version of `get_app_folder` for application data. @@ -306,15 +326,13 @@ store["key"] = "value" # Calls my_dumper Get a config value from a list of sources. -This function acts as a mini-framework to construct config accessors including defining -multiple sources of where to find these configs, +This function acts as a mini-framework to construct config accessors including defining multiple sources of where to find these configs. -A source can be a function or a ``GettableContainer``. -(A ``GettableContainer`` is anything that can be indexed with brackets: ``obj[k]``, -like ``dict``, ``list``, ``str``, etc..). +A source can be a function or a ``GettableContainer``. (A ``GettableContainer`` is anything that can be indexed with brackets: ``obj[k]``, like ``dict``, ``list``, ``str``, etc..). Let's take two sources: a ``dict`` and a ``Callable``. + >>> from config2py import get_config >>> def func(k): ... if k == 'foo': ... return 'quux' @@ -325,23 +343,19 @@ Let's take two sources: a ``dict`` and a ``Callable``. >>> dict_ = {'foo': 'bar', 'baz': 'qux'} >>> sources = [func, dict_] - -See that ``get_config`` go through the sources in the order they were listed, -and returns the first value it finds (or manages to compute) for the key: +See that ``get_config`` go through the sources in the order they were listed, and returns the first value it finds (or manages to compute) for the key: ``get_config`` finds ``'foo'`` in the very first source (``func``): >>> get_config('foo', sources) 'quux' -But ``baz`` makes ``func`` raise an error, so it goes to the next source: ``dict_``. -There, it finds ``'baz'`` and returns its value: +But ``baz`` makes ``func`` raise an error, so it goes to the next source: ``dict_``. There, it finds ``'baz'`` and returns its value: >>> get_config('baz', sources) 'qux' -On the other hand, no one manages to find a config value for ``'no_a_key'``, so -``get_config`` raises an error: +On the other hand, no one manages to find a config value for ``'no_a_key'``, so ``get_config`` raises an error: >>> get_config('no_a_key', sources) Traceback (most recent call last): @@ -353,29 +367,11 @@ But if you provide a default value, it will return that instead: >>> get_config('no_a_key', sources, default='default') 'default' -You can also provide a function that will be called on the value before it is -returned. This is useful if you want to do some post-processing on the value, -or if you want to make sure that the value is of a certain type: - -This "search the next source if the previous one fails" behavior may not be what -you want in some situations, since you'd be hiding some errors that you might -want to be aware of. This is why allow you to specify what exceptions should -actually be considered as "config not found" exceptions, through the -``config_not_found_exceptions`` argument, which defaults to ``Exception``. - -Further, your sources may return a value, but not one that you consider valid: -For example, a sentinel like ``None``. In this case you may want the search to -continue. This is what the ``val_is_valid`` argument is for. It is a function -that takes a value and returns a boolean. If it returns ``False``, the search -will continue. If it returns ``True``, the search will stop and the value will -be returned. - -Finally, we have ``egress : Callable[[KT, TT], VT]``. -This is a function that takes a key and a value, and -returns a value. It is called after the value has been found, and its return -value is the one that is returned by ``get_config``. This is useful if you want -to do some post-processing on the value, or before you return the value, or if you -want to do some caching. +This "search the next source if the previous one fails" behavior may not be what you want in some situations, since you'd be hiding some errors that you might want to be aware of. This is why allow you to specify what exceptions should actually be considered as "config not found" exceptions, through the ``config_not_found_exceptions`` argument, which defaults to ``(Exception,)``, that is, *any* error. Narrow it, for instance to ``(KeyError, LookupError)``, when a failing source should raise rather than be skipped. + +Further, your sources may return a value, but not one that you consider valid: For example, a sentinel like ``None``. In this case you may want the search to continue. This is what the ``val_is_valid`` argument is for. It is a function that takes a value and returns a boolean. If it returns ``False``, the search will continue. If it returns ``True``, the search will stop and the value will be returned. + +Finally, we have ``egress : Callable[[KT, TT], VT]``. This is a function that takes a key and a value, and returns a value. It is called after the value has been found, and its return value is the one that is returned by ``get_config``. This is useful if you want to do some post-processing on the value, or before you return the value, or if you want to do some caching. >>> config_store = dict() >>> def store_before_returning(k, v): @@ -386,24 +382,17 @@ want to do some caching. >>> config_store {'foo': 'quux'} - Note that a source can be a callable or a ``GettableContainer`` (most of the - time, a ``Mapping`` (e.g. ``dict``)). - Here, you should be compelled to use the resources of ``dol`` - (https://pypi.org/project/dol/) which will allow you to make ``Mapping``s for all - sorts of data sources. +Note that a source can be a callable or a ``GettableContainer`` (most of the time, a ``Mapping`` (e.g. ``dict``)). Here, you should be compelled to use the resources of ``dol`` (https://pypi.org/project/dol/) which will allow you to make ``Mapping``s for all sorts of data sources. For more info, see: https://github.com/i2mint/config2py/issues/4 - - - # user_gettable -So, what's that `user_gettable`? +So, what's that `user_gettable`? It's a way for you to specify that the system should ask the user for a key, and optionally save it somewhere, plus many other parameters (like what to ask the user, etc.) - + ```python from config2py.base import user_gettable @@ -419,5 +408,34 @@ s = user_gettable(save_to=d) s["SOME_KEY"] ``` -More on that another day... +The asking function is pluggable (`user_asker`), which also makes `user_gettable` easy to use in tests and scripts: + +```python +from config2py import user_gettable + +d = dict(some="store") +s = user_gettable(save_to=d, user_asker=lambda prompt: "SOME_VAL") +assert s["SOME_KEY"] == "SOME_VAL" +assert d == {"some": "store", "SOME_KEY": "SOME_VAL"} +``` + +# For humans who still hand-edit their dotfiles + +Welcome, fellow keyboard enthusiast. Here is what the sections above don't already cover. + +**Set up and run the tests** the way CI does. Doctests are a big part of the suite, and CI passes its own doctest flags, so use these: + +```bash +git clone https://github.com/i2mint/config2py && cd config2py +uv venv .venv && . .venv/bin/activate && uv pip install -e . pytest ruff +python -m pytest config2py --doctest-modules -o doctest_optionflags='ELLIPSIS IGNORE_EXCEPTION_DETAIL' -q +ruff check . +``` + +The Python examples in this README, and in the bundled skill, are run by `config2py/tests/test_docs_examples.py` in a sandboxed home directory. If you add an example that prompts, put `` on the line before its fence. + +**Design, in three sentences.** `get_config` is a tiny framework: a source is anything you can index (`dict`, a [dol](https://github.com/i2mint/dol) store) or call, and the first source that yields a valid value wins. Everything else (`simple_config_getter`, `user_gettable`, the app folders) is a preset built on that one idea, so you can take any layer apart and recompose it. All platform-specific folder logic sits in one table, `config2py.util.app_folder_standards`, which takes the OS as an argument so both Windows and POSIX behaviour are tested on any machine. + +**Contributing.** Issues and pull requests are welcome on [GitHub](https://github.com/i2mint/config2py/issues). About thirty packages depend on config2py, and a merge to `master` publishes a release to PyPI, so changes to defaults need a run of the dependents' tests. Never edit the version number, because CI bumps it. The architecture notes and conventions a coding agent follows, in [`.claude/CLAUDE.md`](.claude/CLAUDE.md) and the [`config2py-dev`](skills/config2py-dev/SKILL.md) skill, are just as useful to humans. +**Questions** go to [GitHub issues](https://github.com/i2mint/config2py/issues). diff --git a/config2py/tests/test_docs_examples.py b/config2py/tests/test_docs_examples.py new file mode 100644 index 0000000..cb717c4 --- /dev/null +++ b/config2py/tests/test_docs_examples.py @@ -0,0 +1,129 @@ +"""Run the Python examples in ``README.md`` and the bundled skills, so they can't rot. + +Each document's ```` ```python ```` blocks run top to bottom in one namespace (like a +notebook), in a subprocess whose HOME, XDG and CONFIG2PY_* folders all point into a +temporary directory: importing config2py creates its default configs folder, so the +examples must never touch the real user's home. stdin is closed, so a block that would +prompt the user fails loudly instead of hanging. + +``>>>`` examples in the document are also run, as a doctest (with the same flags +as CI). + +A block that can't run unattended (it prompts, or needs a placeholder filled in) is +excluded by putting ```` on the line right before its fence. +""" + +import os +import re +import subprocess +import sys +from pathlib import Path + +import pytest + +PKG_ROOT = Path(__file__).resolve().parents[1] +REPO_ROOT = PKG_ROOT.parent + +DOCUMENTS = { + "README.md": REPO_ROOT / "README.md", + "config2py-quickstart": PKG_ROOT + / "data" + / "skills" + / "config2py-quickstart" + / "SKILL.md", +} + +SKIP_MARKER = "" +_FENCE_OPEN = re.compile(r"^```python\s*$") +_FENCE_CLOSE = re.compile(r"^```\s*$") + +_RUNNER = """ +import sys, traceback +blocks = {blocks!r} +namespace = {{"__name__": "__main__"}} +for line, code in blocks: + try: + exec(compile(code, f"<{doc} block at line {{line}}>", "exec"), namespace) + except BaseException: + traceback.print_exc() + print(f"FAILED: {doc} block starting at line {{line}}", file=sys.stderr) + sys.exit(1) +print(f"ran {{len(blocks)}} blocks") +if {has_doctests!r}: + import doctest + flags = doctest.ELLIPSIS | doctest.IGNORE_EXCEPTION_DETAIL + failed, attempted = doctest.testfile( + {path!r}, module_relative=False, optionflags=flags, encoding="utf-8" + ) + print(f"doctests: {{attempted - failed}}/{{attempted}} passed") + sys.exit(1 if failed else 0) +""" + + +def python_blocks(text: str) -> list[tuple[int, str]]: + """Return ``(line_number, code)`` for each runnable ```python block of ``text``. + + >>> doc = "x\\n```python\\na = 1\\n```\\n\\n```python\\ninput()\\n```\\n" + >>> python_blocks(doc) + [(3, 'a = 1')] + """ + lines = text.splitlines() + blocks, i = [], 0 + while i < len(lines): + if _FENCE_OPEN.match(lines[i]): + skip = i > 0 and lines[i - 1].strip() == SKIP_MARKER + start = i + 1 + j = start + while j < len(lines) and not _FENCE_CLOSE.match(lines[j]): + j += 1 + if not skip: + blocks.append((start + 1, "\n".join(lines[start:j]))) + i = j + 1 + else: + i += 1 + return blocks + + +def _sandboxed_env(root: Path) -> dict: + env = {k: v for k, v in os.environ.items() if not k.startswith("CONFIG2PY_")} + home = root / "home" + home.mkdir() + env.update( + HOME=str(home), + USERPROFILE=str(home), + XDG_CONFIG_HOME=str(home / ".config"), + XDG_DATA_HOME=str(home / ".local" / "share"), + XDG_CACHE_HOME=str(home / ".cache"), + XDG_STATE_HOME=str(home / ".local" / "state"), + APPDATA=str(home / "AppData" / "Roaming"), + LOCALAPPDATA=str(home / "AppData" / "Local"), + ) + return env + + +@pytest.mark.parametrize("doc", sorted(DOCUMENTS)) +def test_document_examples_run(doc, tmp_path): + path = DOCUMENTS[doc] + if not path.exists(): + pytest.skip(f"{path} not present (not running from a source checkout)") + text = path.read_text(encoding="utf-8") + blocks = python_blocks(text) + assert blocks, f"no runnable python blocks found in {path}" + workdir = tmp_path / "cwd" + workdir.mkdir() + result = subprocess.run( + [ + sys.executable, + "-c", + _RUNNER.format( + blocks=blocks, doc=doc, path=str(path), has_doctests=">>> " in text + ), + ], + cwd=workdir, + env=_sandboxed_env(tmp_path), + stdin=subprocess.DEVNULL, + capture_output=True, + text=True, + timeout=120, + ) + assert result.returncode == 0, result.stdout + result.stderr From 3c54bef4f3a988eeff56fad524ed1fbfc68c0a83 Mon Sep 17 00:00:00 2001 From: Thor Whalen <1906276+thorwhalen@users.noreply.github.com> Date: Sun, 27 Sep 2026 09:05:12 +0000 Subject: [PATCH 6/6] fix: keep explicit mask_input=True on getpass when stdin is piped The adversarial review of #31 found a regression: the stdin fallback applied to every masked read, so an explicit mask_input=True with a terminal present and stdin piped read the first piped line as the secret. Stdlib getpass reads /dev/tty on purpose there, as sudo does. The fallback now applies only when masking was inferred by a predicate (the new default), which keeps both paths identical to master: explicit True reads the terminal, and a defaulted prompt reads stdin as the old echoing default did. Verified under a real pty. The docstrings now also say that looks_like_secret is a substring match on the whole prompt. Refs #13 --- .claude/CLAUDE.md | 4 +-- .../data/skills/config2py-quickstart/SKILL.md | 2 +- config2py/tests/test_masking.py | 25 +++++++++++--- config2py/util.py | 33 ++++++++++++------- skills/config2py-dev/SKILL.md | 2 +- 5 files changed, 46 insertions(+), 20 deletions(-) diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index f8d3630..63617f4 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -25,7 +25,7 @@ Tools to read and write configurations from various sources and formats: layered ```bash uv venv .venv && uv pip install -e . pytest ruff .venv/bin/pytest config2py --doctest-modules \ - -o doctest_optionflags='ELLIPSIS IGNORE_EXCEPTION_DETAIL' -q # 154 passed, 5 skipped + -o doctest_optionflags='ELLIPSIS IGNORE_EXCEPTION_DETAIL' -q # 155 passed, 5 skipped .venv/bin/ruff check . ``` @@ -33,7 +33,7 @@ Or `wads ci-local`. **Gotchas already documented in `pyproject.toml` comments:** ## Invariants / known gaps (see open issues before "fixing" these) -- **`DFLT_MASKING_INPUT = looks_like_secret`** in `util.py` (since [#13](https://github.com/i2mint/config2py/issues/13)). `ask_user_for_input` masks prompts that mention something secret-looking (`API_KEY`, `TOKEN`, `PASSWORD`, ...) and echoes the others. `mask_input` accepts a bool or a `prompt -> bool` predicate. When stdin is not a terminal and `getpass.getpass` is the stdlib one, masked reads fall back to `input`: stdlib `getpass` reads `/dev/tty`, not stdin, so it would ignore piped input or hang. A frontend's replacement `getpass` (Jupyter) is always used. The tests are in `config2py/tests/test_masking.py`. Don't change the default without re-running the dependents check. +- **`DFLT_MASKING_INPUT = looks_like_secret`** in `util.py` (since [#13](https://github.com/i2mint/config2py/issues/13)). `ask_user_for_input` masks prompts that mention something secret-looking (`API_KEY`, `TOKEN`, `PASSWORD`, ...) and echoes the others. `mask_input` accepts a bool or a `prompt -> bool` predicate. When masking was inferred by the predicate, stdin is not a terminal, and `getpass.getpass` is the stdlib one, the read falls back to `input`, as before the default existed (stdlib `getpass` reads `/dev/tty`, not stdin). An explicit `mask_input=True` always uses `getpass.getpass`. A frontend's replacement `getpass` (Jupyter) is always used. The tests are in `config2py/tests/test_masking.py`. Don't change the default without re-running the dependents check. - The #16 audit was split into follow-ups, each with a plan. Read the matching one before touching an area: [#25](https://github.com/i2mint/config2py/issues/25) (broad `(Exception,)` fallback), [#26](https://github.com/i2mint/config2py/issues/26) (import-time folder creation), [#27](https://github.com/i2mint/config2py/issues/27) (typo'd paths give silent empty configs), [#28](https://github.com/i2mint/config2py/issues/28) (`os.path.sep` path sniffing), [#29](https://github.com/i2mint/config2py/issues/29) (pickle decoder opt-in). Also [#30](https://github.com/i2mint/config2py/issues/30) (the masking toggle drops `egress`, which interacts with `oa`) and [#33](https://github.com/i2mint/config2py/issues/33) (configs-store value files are written `0o644`). - [#22](https://github.com/i2mint/config2py/pull/22) (open) removes the vestigial `setup.cfg`, adds `[tool.wads.ci]`, and migrates CI to the reusable-workflow stub. It is blocked on a hosted-CI `action_required` anomaly (see [#23](https://github.com/i2mint/config2py/issues/23)). Until it lands, CI here is the inline uv workflow using discrete `i2mint/wads` actions. diff --git a/config2py/data/skills/config2py-quickstart/SKILL.md b/config2py/data/skills/config2py-quickstart/SKILL.md index 59c9c2c..cd27643 100644 --- a/config2py/data/skills/config2py-quickstart/SKILL.md +++ b/config2py/data/skills/config2py-quickstart/SKILL.md @@ -94,7 +94,7 @@ assert saved == {"API_TOKEN": "typed value"} assert get("API_TOKEN") == "typed value" # now found in `saved`, not asked again ``` -The default asker is `ask_user_for_input`. It masks what the user types when the prompt looks secret (it mentions `key`, `token`, `pass`, `pwd`, `secret`, `api`, `auth`, `credential` or `private`) and echoes it otherwise. Force a choice with `ask_user_for_input(prompt, mask_input=True)` (or `False`), or pass your own `prompt -> bool` predicate. When stdin is piped, the answer is read from stdin. +The default asker is `ask_user_for_input`. It masks what the user types when the prompt looks secret (it mentions `key`, `token`, `pass`, `pwd`, `secret`, `api`, `auth`, `credential` or `private`) and echoes it otherwise. Force a choice with `ask_user_for_input(prompt, mask_input=True)` (or `False`), or pass your own `prompt -> bool` predicate. The match is a plain substring test on the prompt, so `KEYS_DIR` is masked too. When masking comes from the predicate and stdin is piped, the answer is read from stdin; an explicit `mask_input=True` always reads the terminal, like `sudo`. ## App folders diff --git a/config2py/tests/test_masking.py b/config2py/tests/test_masking.py index 05e5c35..ce983b9 100644 --- a/config2py/tests/test_masking.py +++ b/config2py/tests/test_masking.py @@ -127,11 +127,11 @@ def respond(prompt=""): assert "Input masking is ENABLED" in prompts[0] -@pytest.mark.parametrize("mask_input", [True, looks_like_secret]) -def test_piped_stdin_is_read_even_when_masking(monkeypatch, mask_input): - """Stdlib ``getpass`` reads the terminal, not stdin, so it would ignore piped - input (or block waiting on the terminal). Without a terminal there is nothing to - echo to, so the value must be read from stdin. +@pytest.mark.parametrize("mask_input", [looks_like_secret, lambda prompt: True]) +def test_piped_stdin_is_read_when_masking_is_inferred(monkeypatch, mask_input): + """When masking was *inferred* (the default predicate), piped input keeps being + read from stdin, as it was before masking became the default for secret-looking + prompts: stdlib ``getpass`` would read the terminal instead (or block on it). """ monkeypatch.setattr(sys, "stdin", io.StringIO("piped value\n")) monkeypatch.setattr( @@ -141,6 +141,21 @@ def test_piped_stdin_is_read_even_when_masking(monkeypatch, mask_input): assert ask_user_for_input("Enter API_KEY", mask_input=mask_input) == "piped value" +def test_explicit_mask_input_true_keeps_reading_the_terminal(monkeypatch): + """An explicit ``mask_input=True`` keeps its pre-#13 meaning: stdlib ``getpass``, + which deliberately reads the terminal even when stdin is piped (like ``sudo``), so + piped data is never mistaken for the secret. + """ + monkeypatch.setattr(sys, "stdin", io.StringIO("piped data, not the secret\n")) + fake_getpass = _fake_getpass("typed at the terminal") + monkeypatch.setattr(getpass, "getpass", fake_getpass) + monkeypatch.setattr(builtins, "input", _fake_input(AssertionError("read stdin"))) + assert ask_user_for_input("Enter API_KEY", mask_input=True) == ( + "typed at the terminal" + ) + assert len(fake_getpass.calls) == 1 + + def test_frontend_getpass_is_used_without_a_terminal(monkeypatch): """Frontends such as Jupyter replace ``getpass.getpass`` with their own masked widget while ``sys.stdin`` is not a terminal. That replacement must be used. diff --git a/config2py/util.py b/config2py/util.py index 23e9826..30695b7 100644 --- a/config2py/util.py +++ b/config2py/util.py @@ -33,7 +33,10 @@ def looks_like_secret(text: str) -> bool: """True if ``text`` (typically a prompt naming a config key) looks secret. It errs on the side of masking: a false positive only means the user doesn't see - what they type, while a false negative echoes a secret to the terminal. + what they type, while a false negative echoes a secret to the terminal. It is a + plain substring match on the whole prompt, so ``KEYS_DIR`` or ``AUTHOR`` also + match, as would a custom prompt template mentioning "key". Pass an explicit + ``mask_input`` (or your own predicate) when that matters. >>> looks_like_secret("Enter a value for OPENAI_API_KEY: ") True @@ -152,14 +155,17 @@ def _stdin_is_a_terminal() -> bool: return False -def _masked_prompt_func() -> Callable[[str], str]: - """The function to read a masked response with. +def _inferred_masked_prompt_func() -> Callable[[str], str]: + """The function to read a response with when masking was *inferred* by a predicate. The stdlib's ``getpass.getpass`` reads the controlling terminal, not ``sys.stdin``: with piped input it would ignore the pipe (or block waiting on the terminal). - When stdin isn't a terminal there's nothing to echo to, so we read stdin with - ``input`` instead. A ``getpass.getpass`` replaced by a frontend (e.g. Jupyter's - masked widget, where stdin is never a terminal) is always used. + Before masking became the default for secret-looking prompts, such prompts read + stdin with ``input``, so when stdin isn't a terminal we keep doing that (there's + no terminal echo of piped data anyway). A ``getpass.getpass`` replaced by a + frontend (e.g. Jupyter's masked widget, where stdin is never a terminal) is always + used. An explicit ``mask_input=True`` doesn't come here: it always uses + ``getpass.getpass``, which deliberately reads the terminal (like ``sudo``). """ getpass_func = getpass.getpass is_stdlib_getpass = getattr(getpass_func, "__module__", None) == "getpass" @@ -185,9 +191,10 @@ def ask_user_for_input( :param mask_input: Whether to mask the user's input: a bool, or a ``prompt -> bool`` predicate. The default, ``looks_like_secret``, masks prompts that mention something secret-looking (``API_KEY``, ``TOKEN``, - ``PASSWORD``, ...) and echoes the others. Masking needs a terminal (or a - frontend such as Jupyter): when stdin is piped, the response is read from - stdin, where nothing is echoed anyway. + ``PASSWORD``, ...) and echoes the others. When masking is decided by a + predicate and stdin is piped (not a terminal), the response is read from + stdin, as it was before this default existed. An explicit ``True`` always + uses ``getpass.getpass``, which reads the terminal even when stdin is piped. :param masking_toggle_str: String to toggle input masking. If ``None``, no toggle is available. If not ``None`` (a common choice is the empty string) the user can enter this string to toggle input masking. @@ -195,7 +202,8 @@ def ask_user_for_input( This can be used to validate the response, for example. :return: The user's response (or the default value if the user entered nothing) """ - if callable(mask_input): + masking_is_inferred = callable(mask_input) + if masking_is_inferred: mask_input = bool(mask_input(prompt)) _original_prompt = prompt if prompt[-1] != " ": # pragma: no cover @@ -209,7 +217,10 @@ def ask_user_for_input( if default not in {""}: prompt = prompt + f" [{default}]: " if mask_input: - _prompt_func = _masked_prompt_func() + if masking_is_inferred: + _prompt_func = _inferred_masked_prompt_func() + else: + _prompt_func = getpass.getpass else: _prompt_func = input diff --git a/skills/config2py-dev/SKILL.md b/skills/config2py-dev/SKILL.md index 202f6c9..1e9b7e8 100644 --- a/skills/config2py-dev/SKILL.md +++ b/skills/config2py-dev/SKILL.md @@ -54,7 +54,7 @@ About 30 packages depend on config2py (for example `py2store`, `xdol`, `oa`, `ai 1. Grep the dependents for the symbol you are changing. 2. Run their test suites against your working tree (`uv pip install -e && uv pip install -e ` in a scratch venv), before and after your change, and compare pass counts. Known pre-existing failures: `xdol` has 2 doctest-format failures and `oa` has 3 tests that need a real OpenAI key. -3. Prefer additive, keyword-only parameters. For example, `mask_input` accepts a bool or a `prompt -> bool` predicate, and explicit booleans keep their old meaning. +3. Prefer additive, keyword-only parameters. For example, `mask_input` accepts a bool or a `prompt -> bool` predicate, and explicit booleans keep their old meaning, including `True` reading the terminal when stdin is piped. Read these open issues before touching the matching area: #25 (the broad `(Exception,)` fallback), #26 (import-time folder creation), #27 (typo'd paths give silent empty configs), #28 (`os.path.sep` path sniffing), #29 (the pickle decoder), #30 (the masking toggle drops `egress`), #33 (file modes of the configs store), #12 and #17 (folder layout and platformdirs).