diff --git a/.env.example b/.env.example index bb90df2..a7b6ab8 100644 --- a/.env.example +++ b/.env.example @@ -2,48 +2,33 @@ # # .env is GITIGNORED and must never be committed (it can hold a real # credential). Everything here is OPTIONAL: it is read only by the -# integration tests (tests/conftest.py loads .env via python-dotenv) and by -# the logging runtime. The library itself needs none of it. +# integration tests and by the logging runtime. The library itself needs none of it. # -# Optional env-prefix cascade (scalo convention): set LOGREDUCER_ENV_PREFIX to -# read _ before the bare name - e.g. LOGREDUCER_ENV_PREFIX="DFE" -# makes DFE_CLICKHOUSE_HOST win over CLICKHOUSE_HOST, so a .env written for -# another project drops in without renaming. -# LOGREDUCER_ENV_PREFIX="DFE" -# -# Test service selection is env-first, then docker: -# 1. if the vars below point at a reachable service, tests use it; -# 2. otherwise a throwaway docker container is started (ClickHouse, Redpanda -# for Kafka, PostgreSQL, MySQL) and cleaned up at the end; -# 3. if neither is available the dependent test skips. -# CI sets none of these -> it always uses docker. Tests create uniquely-named -# tables/topics and drop them afterwards, so pointing at a shared/persistent -# instance is safe. +# ClickHouse and Kafka for the integration tests come from scalo's shared test +# services: one container each per run, or an always-present server named by the +# HYPERI_TEST_* settings below when it answers (never in CI or under the full +# tier). Without Docker the test skips locally and fails in CI. Each test makes +# its own database or topics and drops them afterwards. -# --- ClickHouse (integration tests) ----------------------------------------- -# CLICKHOUSE_HOST="localhost" -# CLICKHOUSE_PORT="8123" -# CLICKHOUSE_USER="default" -# CLICKHOUSE_PASSWORD="" -# CLICKHOUSE_SECURE="false" # true = TLS -# CLICKHOUSE_VERIFY="true" # false = skip TLS certificate verification +# --- ClickHouse pet (integration tests) ------------------------------------- +# HYPERI_TEST_CLICKHOUSE_HOST="localhost" +# HYPERI_TEST_CLICKHOUSE_HTTP_PORT="8123" +# HYPERI_TEST_CLICKHOUSE_USER="default" +# HYPERI_TEST_CLICKHOUSE_PASSWORD="" +# HYPERI_TEST_CLICKHOUSE_CLUSTER="default" -# --- Kafka (integration tests) ---------------------------------------------- -# A bare bootstrap is a plain (no-auth) broker. Add KAFKA_SECURITY_PROTOCOL to -# reach an authenticated broker (SASL_SSL/SASL_PLAINTEXT); the SASL_* + SSL vars -# are then merged into the librdkafka client config. -# KAFKA_BOOTSTRAP_SERVERS="localhost:9092" -# KAFKA_SECURITY_PROTOCOL="SASL_SSL" # e.g. SASL_SSL | SASL_PLAINTEXT -# KAFKA_SASL_MECHANISM="SCRAM-SHA-512" -# KAFKA_SASL_USERNAME="user" -# KAFKA_SASL_PASSWORD="" -# KAFKA_SSL_CA_LOCATION="/path/to/ca.pem" # CA bundle for TLS verification -# KAFKA_SSL_VERIFY="true" # false = skip TLS cert verification +# --- Kafka pet (integration tests) ------------------------------------------ +# HYPERI_TEST_KAFKA_BOOTSTRAP="localhost:9092" +# HYPERI_TEST_KAFKA_SECURITY_PROTOCOL="SASL_SSL" +# HYPERI_TEST_KAFKA_SASL_MECHANISM="SCRAM-SHA-512" +# HYPERI_TEST_KAFKA_USER="user" +# HYPERI_TEST_KAFKA_PASSWORD="" +# HYPERI_TEST_KAFKA_CA_FILE="/path/to/ca.pem" +# HYPERI_TEST_KAFKA_TOPIC_PREFIX="test-" -# --- Docker test containers (optional) -------------------------------------- -# Keep the throwaway docker services RUNNING after the run (default: stop + -# remove). Handy for fast re-runs - export the printed endpoint as the vars -# above so the next run takes the env-first path. +# --- PostgreSQL and MySQL test containers (optional) ------------------------ +# Keep the throwaway PostgreSQL and MySQL containers RUNNING after the run +# (default: stop + remove). # LOGREDUCER_KEEP_CONTAINERS="1" # --- Logging (runtime, optional) --------------------------------------------- diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f9d177e..f5b8397 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -37,23 +37,8 @@ uv build # build wheel + sdist internal code). `tests/integration/` - marked `integration`, run against real services. `tests/testdata/` holds gzipped, PII-cleansed slices of REAL public log datasets (see its README for provenance and the rebuild tool). -- Integration tests find their services env-first, then docker, then skip: - -```mermaid -flowchart TD - ENV{"env vars set?
(CLICKHOUSE_* /
KAFKA_BOOTSTRAP_SERVERS)"} -->|yes| PROBE{"endpoint
reachable?"} - PROBE -->|yes| USE(["use the configured service
(unique table/topic names,
dropped afterwards)"]) - PROBE -->|"no (warns why)"| DOCKER - ENV -->|no| DOCKER{"Docker
available?"} - DOCKER -->|yes| TC(["throwaway testcontainer:
ClickHouse / Redpanda /
PostgreSQL / MySQL"]) - TC --> STOP(["stopped at session end
(LOGREDUCER_KEEP_CONTAINERS=1
keeps it + prints the endpoint)"]) - DOCKER -->|no| SKIP([pytest.skip]) -``` - -- Copy `.env.example` to `.env` to point tests at existing services. CI sets - no env vars, so it always takes the docker path. -- Every test that touches a shared service creates uniquely-named tables and - topics and cleans them up, so runs never collide. +- ClickHouse and Kafka come from scalo's shared test services (`scalo.deployment.test_support.pytest_plugin`): one container each per run, shared by every xdist worker, or the always-present server the `HYPERI_TEST_*` settings in `.env.example` name. Each test gets its own database or topics, dropped afterwards. Without Docker those tests skip locally and fail in CI. +- PostgreSQL and MySQL are throwaway testcontainers, stopped at the end of the session unless `LOGREDUCER_KEEP_CONTAINERS=1`, and skip without Docker. ## Commit messages diff --git a/README.md b/README.md index 721799a..e979128 100644 --- a/README.md +++ b/README.md @@ -11,9 +11,9 @@ It ships as a CLI and as a Python library with an IO-agnostic core. ## What it gives you - A `logreducer` command that reduces a file, a SQL or ClickHouse query, or a Kafka topic, straight from the shell. -- Pattern mode streams through a bounded dedup cache and template store, so memory grows with the number of distinct templates, not with the line count. +- Pattern mode streams through a bounded dedup cache, fuzzy index and template store, so memory does not grow with the line count or the number of distinct lines. - Four modes: pattern (Drain3 template mining), anomaly (Isolation Forest), temporal (one example per template per time window) and hybrid. -- ERROR, FATAL and CRITICAL lines always come through, one per distinct shape and ahead of the reduced output, in every mode and however a file is read. A SQL or ClickHouse source samples in the database first, so only the lines it returns are seen. +- Lines naming ERROR, FATAL or CRITICAL as a whole word, in any case, always come through, one per distinct shape and ahead of the reduced output, in every mode and however a file is read. A SQL or ClickHouse source samples in the database first, so only the lines it returns are seen. - A core with no IO dependencies: anything re-iterable that yields `str` lines is a source, and anything with `write(lines)` is a sink. - Seeded SQL sampling and ClickHouse `SAMPLE` for tables too big to scan in full. - Adapters for SQL, ClickHouse and Kafka as extras, so you install only the one you use. @@ -88,7 +88,7 @@ reduced = reducer.reduce(lines) ## How it works -Pattern mode streams: lines flow through dedup and template mining as one generator pipeline, so the unique-line set is never materialised. Memory grows with the number of distinct templates, since a result is kept for every template seen, not with the line count. +Pattern mode streams: lines flow through dedup and template mining as one generator pipeline, so the unique-line set is never materialised. A template the store evicts keeps its place in the ranking only while it can still make the `max_patterns` cut, so memory is set by `max_clusters`, `max_patterns` and `dedup_cache_size`, not by the input. ```mermaid flowchart LR @@ -223,10 +223,11 @@ logreducer --dsn postgresql://user@host/db --query "SELECT msg FROM logs" \ ## Bounding memory -Pattern mode streams, and its memory grows with distinct templates, not lines. Temporal mode holds the whole source in memory, and anomaly mode holds its unique lines. The knobs: +Pattern mode streams, and its memory is fixed by the knobs below, not by lines or distinct templates. Temporal mode holds the whole source in memory, and anomaly mode holds its unique lines. The knobs: - `max_memory_gb` - the overall cap (also sizes file read strategy and the watchdog). Presets: 0.5 / 1 / 2 GB by level; values above 70% of available RAM are clamped with a warning. -- `max_clusters` - LRU-bound the Drain3 template store (10,000 by default, `None` for unbounded). Mining time grows with the square of the number of templates a line is compared with, so a lower value also speeds up high-cardinality sources. +- `dedup_cache_size` - the lines exact dedup remembers, and at enhanced and maximum the lines the fuzzy index holds. Both drop the oldest first, so a repeat or near-repeat older than the window comes through again. +- `max_clusters` - LRU-bound the Drain3 template store (10,000 by default, `None` for unbounded, which also keeps every evicted template's result). Mining time grows with the square of the number of templates a line is compared with, so a lower value also speeds up high-cardinality sources. - `anomaly_max_rows` - reservoir-cap the rows fed to anomaly detection (off by default; trades anomaly recall for a bounded matrix). ```python @@ -307,7 +308,7 @@ reducer = LogReducer( level="enhanced", mode="hybrid", max_memory_gb=1.0, # memory ceiling (clamped to 70% of available RAM) - dedup_cache_size=100000, # bounded dedup cache + dedup_cache_size=100000, # bounded dedup cache and fuzzy index drain_similarity=0.4, # pattern similarity threshold fuzzy_threshold=0.8, # fuzzy-dedup threshold (enhanced/maximum) anomaly_contamination=0.1, # expected anomaly fraction diff --git a/docs/architecture.md b/docs/architecture.md index e5078e6..4a9461b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -66,8 +66,11 @@ than pass one, and the reducer would be comparing two populations. Pattern mode streams end to end. Exact dedup, optional fuzzy dedup and Drain3 mining run as one generator pipeline, so the unique-line set is never collected -into a list. Peak memory is the bounded dedup cache plus the Drain3 template -store, and neither grows with the number of unique lines the source has. +into a list. Peak memory is the bounded dedup cache, the fuzzy index (the same `dedup_cache_size` window), the Drain3 template store and the results of evicted templates, and none of them grows with the number of unique lines the source has. + +An evicted template's result is kept only while it can still make the `max_patterns` cut. Its count stops changing once the dedup cache has dropped the lines that fed it, so from then on it competes for one of `max_patterns` places, and a result that loses one can never win it back. That keeps the output identical to ranking every template ever seen. + +The fuzzy index measured about 1.8 KB per entry, so the preset windows hold about 180 MB at enhanced (100K lines) and 360 MB at maximum (200K), inside their 1 GB and 2 GB budgets. Drain3 compares each line with every template stored at the leaf it reaches, so time grows with the square of the template count when lines share no tokens. @@ -128,24 +131,14 @@ Three seams, none of which needs the library to know anything about the host. ## Invariants -1. **`__iter__` returns a fresh iterator.** Checked at the entry point, not - documented and hoped for. -2. **The core imports no IO library.** An adapter's dependency is an extra, and - the core never imports the adapter modules. -3. **Analysis state is per run.** `_reset_components()` rebuilds the dedup - seen-set, both Drain3 miners and the fuzzy LSH, so one `LogReducer` is - reusable across calls and hybrid's two passes cannot poison each other. - Without it every line reads as already-seen and the result is empty. -4. **An unknown config keyword raises.** A silently dropped override is tuning - that quietly does nothing, which is worse than a failure. -5. **The row-to-line convention is shared.** The database adapters take the FIRST - column, skip NULLs and skip blank lines, matching `FileSource`, so the same - data reduces identically whichever source carried it. -6. **Every sample is seeded.** See above. -7. **Config enums serialise as their `.value`**, so metadata survives - `json.dumps` and no `OutputFormat.LINE`-style repr leaks into output. -8. **A memory ceiling above 70% of available RAM is clamped with a warning** - rather than accepted. +1. `__iter__` returns a fresh iterator, checked at the entry point rather than documented and hoped for. +2. The core imports no IO library. An adapter's dependency is an extra, and the core never imports the adapter modules. +3. Analysis state is per run. `_reset_components()` rebuilds the dedup seen-set, both Drain3 miners and the fuzzy LSH, so one `LogReducer` is reusable across calls and hybrid's two passes cannot poison each other. Without it every line reads as already-seen and the result is empty. +4. An unknown config keyword raises. A silently dropped override is tuning that quietly does nothing, which is worse than a failure. +5. The row-to-line convention is shared. The database adapters take the FIRST column, skip NULLs and skip blank lines, matching `FileSource`, so the same data reduces identically whichever source carried it. +6. Every sample is seeded. See above. +7. Config enums serialise as their `.value`, so metadata survives `json.dumps` and no `OutputFormat.LINE`-style repr leaks into output. +8. A memory ceiling above 70% of available RAM is clamped with a warning rather than accepted, whether it comes from the config or a `LogReducer` argument. ### Where the numbers are approximate diff --git a/docs/pypi.md b/docs/pypi.md index 5eec1f4..b3ba13a 100644 --- a/docs/pypi.md +++ b/docs/pypi.md @@ -7,10 +7,10 @@ It ships as a CLI and as a Python library with an IO-agnostic core. ## What it gives you - A `logreducer` command that reduces a file, a SQL or ClickHouse query, or a Kafka topic, straight from the shell. -- Pattern mode streams through a bounded dedup cache and template store, so memory grows with the number of distinct templates, not with the line count. +- Pattern mode streams through a bounded dedup cache, fuzzy index and template store, so memory does not grow with the line count or the number of distinct lines. - A memory budget that decides how a file is read: whole, in chunks, or reservoir-sampled once it passes five times the budget. - Four modes: pattern (Drain3 template mining), anomaly (Isolation Forest), temporal (one example per template per time window) and hybrid. -- ERROR, FATAL and CRITICAL lines always come through, one per distinct shape and ahead of the reduced output, in every mode and however a file is read. A SQL or ClickHouse source samples in the database first, so only the lines it returns are seen. +- Lines naming ERROR, FATAL or CRITICAL as a whole word, in any case, always come through, one per distinct shape and ahead of the reduced output, in every mode and however a file is read. A SQL or ClickHouse source samples in the database first, so only the lines it returns are seen. - A core with no IO dependencies: anything re-iterable that yields `str` lines is a source, and anything with `write(lines)` is a sink. - Seeded SQL sampling and ClickHouse `SAMPLE` for tables too big to scan in full. - `reduce_to_target` to collect about N representative lines from a large source, holding roughly one batch in memory. @@ -48,7 +48,7 @@ From the shell it is `logreducer app.log -o reduced.log`. `level` is `standard`, | Extra | Adds | |---|---| -| `enhanced` | Fuzzy dedup with MinHash LSH, whose index grows with near-unique lines, and faster hashing | +| `enhanced` | Fuzzy dedup with MinHash LSH, whose index holds the last `dedup_cache_size` near-unique lines, and faster hashing | | `sql` | `SQLSource` over SQLAlchemy, with your own DBAPI driver | | `clickhouse` | `ClickHouseSource` over clickhouse-connect | | `kafka` | `KafkaSource` and `KafkaSink` over confluent-kafka | diff --git a/examples/README.md b/examples/README.md index b7d06bd..2d8198b 100644 --- a/examples/README.md +++ b/examples/README.md @@ -18,7 +18,7 @@ sample log. ### Docker container ```dockerfile -FROM python:3.12-slim +FROM python:3.14-slim RUN pip install logreducer @@ -38,7 +38,7 @@ spec: spec: containers: - name: logreducer - image: python:3.12-slim + image: python:3.14-slim command: ["sh", "-c"] args: - | diff --git a/pyproject.toml b/pyproject.toml index 7e799d1..ab11286 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,7 +74,7 @@ module-root = "src" # Ruff: lint + format (replaces black + flake8 + isort). Line length 120. [tool.ruff] -target-version = "py312" +target-version = "py314" line-length = 120 indent-width = 4 src = ["src"] @@ -144,7 +144,7 @@ line-ending = "auto" # (datasketch, ...); our own code stays fully typed via # disallow_untyped_defs below. [tool.mypy] -python_version = "3.12" # Held below requires-python with ruff; see target-version +python_version = "3.14" ignore_missing_imports = true warn_return_any = true warn_unused_configs = true @@ -171,6 +171,7 @@ markers = [ "slow: marks tests as slow (deselect with '-m \"not slow\"')", "integration: marks tests as integration tests", ] +shared_service_repo = "logreducer" # Coverage configuration [tool.coverage.run] @@ -218,9 +219,9 @@ index-strategy = "unsafe-best-match" # version published in the last 7 days. A CVE fix bypasses it through a # temporary per-package exclude-newer-package entry, not by lowering this. exclude-newer = "7 days" -# certifi PYSEC-2026-4320 is fixed in 2026.10.19. A per-package entry replaces the -# global span rather than widening it, so remove this one on 2026-10-28. -exclude-newer-package = { certifi = "2026-10-21T00:00:00Z" } +# scalo 2.32.4 carries the shared test services the suite needs, released 2026-11-01. A +# per-package entry replaces the global span rather than widening it, so remove this one on 2026-11-08. +exclude-newer-package = { scalo = "2026-11-01T06:00:00Z" } override-dependencies = [] [[tool.uv.index]] @@ -230,10 +231,11 @@ default = true # ty: Astral type checker, runs alongside mypy. [tool.ty.environment] -python-version = "3.12" +python-version = "3.14" [tool.ty.src] -exclude = ["examples/", "docs/", "scripts/"] +# build_quiet_golden.py runs under uvx with drain3, which is never a dependency here. +exclude = ["examples/", "docs/", "scripts/", "tests/testdata/build_quiet_golden.py"] # Below 80 vulture reports the library's public API and attributes set on # third-party config objects, which buries a real leftover. @@ -242,7 +244,7 @@ min_confidence = 80 # vermin: verify no syntax/features newer than the minimum Python. [tool.vermin] -targets = ["3.12"] +targets = ["3.14"] no_tips = true [tool.bandit.assert_used] @@ -273,7 +275,8 @@ dev = [ "sqlalchemy>=2.1.3", # SQLSource against SQLite (real, no server) "clickhouse-connect>=1.9.0", # ClickHouseSource (local cluster or docker) "confluent-kafka>=2.15.1", # KafkaSource/KafkaSink - "testcontainers[clickhouse,kafka,postgres,mysql]>=4.15.0", # docker fallback; modules under testcontainers.community + "scalo>=2.32.4", # the run's shared ClickHouse and Kafka (scalo.deployment.test_support) + "testcontainers[postgres,mysql]>=4.15.0", # PostgreSQL and MySQL, which scalo does not provide "psycopg[binary]>=3.2", # PostgreSQL driver (real TABLESAMPLE/setseed sampling tests) "pymysql>=1.1.0", # MySQL driver (real RAND(seed) sampling tests) "python-dotenv>=1.0.0", # load the local .env (CLICKHOUSE_*) in tests diff --git a/src/logreducer/anomaly.py b/src/logreducer/anomaly.py index 2ec83b2..6a1a470 100644 --- a/src/logreducer/anomaly.py +++ b/src/logreducer/anomaly.py @@ -4,6 +4,9 @@ logger = logging.getLogger(__name__) +# Fewer lines than this are returned unscored, all of them normal. +MIN_SCORED_LINES = 10 + # Try optional imports try: from sklearn.ensemble import IsolationForest @@ -30,7 +33,7 @@ def detect_anomalies(self, lines: list[str]) -> tuple[list[str], list[str]]: Returns: (anomalous_lines, normal_lines) """ - if not self.enabled or len(lines) < 10: + if not self.enabled or len(lines) < MIN_SCORED_LINES: return [], lines # Vectorize diff --git a/src/logreducer/cli.py b/src/logreducer/cli.py index ce195c6..bd753d5 100644 --- a/src/logreducer/cli.py +++ b/src/logreducer/cli.py @@ -189,7 +189,7 @@ def _reduce( else: _err("Error: provide a log file, or --dsn with --query (SQL/ClickHouse) or --topic/--group (Kafka)") raise typer.Exit(1) - except (typer.Exit, typer.BadParameter): + except typer.Exit, typer.BadParameter: raise # our control-flow exit / typer usage errors - let typer format them except KeyboardInterrupt: _err("Processing interrupted by user") @@ -276,7 +276,7 @@ def _run_target( close = getattr(source, "close", None) if callable(close): close() - except (typer.Exit, typer.BadParameter): + except typer.Exit, typer.BadParameter: raise except KeyboardInterrupt: _err("Processing interrupted by user") diff --git a/src/logreducer/config.py b/src/logreducer/config.py index 983db43..f5ee244 100644 --- a/src/logreducer/config.py +++ b/src/logreducer/config.py @@ -47,6 +47,7 @@ class BigDialConfig: # measured reductions use tens of MB, so the default is deliberately low # and container-friendly. max_memory_gb: float = 1.0 + # Lines exact dedup remembers, and lines the fuzzy index holds. Both drop the oldest first. dedup_cache_size: int = 100000 # Speed Control @@ -101,7 +102,7 @@ def __post_init__(self) -> None: self.max_memory_gb = clamped @classmethod - def from_env(cls, *prefixes: str) -> "BigDialConfig": + def from_env(cls, *prefixes: str) -> BigDialConfig: """Build a config from environment variables, cascade-aware. ``from_env()`` reads ``LOGREDUCER_`` (upper-cased field names, diff --git a/src/logreducer/core.py b/src/logreducer/core.py index 3254f9f..2d59d80 100644 --- a/src/logreducer/core.py +++ b/src/logreducer/core.py @@ -11,8 +11,8 @@ from pathlib import Path from typing import Any, Literal, overload -from .anomaly import AnomalyDetector -from .config import BigDialConfig, OutputFormat, ProcessingLevel, ProcessingMode, get_preset_config, validate_config +from .anomaly import MIN_SCORED_LINES, AnomalyDetector +from .config import BigDialConfig, OutputFormat, ProcessingLevel, ProcessingMode, get_preset_config from .logging_config import get_logger, setup_logging from .memory import BoundedDeduplicator, MemoryMonitor from .patterns import FuzzyDeduplicator, PatternExtractor, SevereShapes, is_severe, pattern_priority @@ -64,24 +64,23 @@ def __init__( self.mode = mode # Base config: an injected one wins over the level preset. - self.config = replace(config) if config is not None else get_preset_config(level) + base = config if config is not None else get_preset_config(level) - # Apply overrides BEFORE setting up logging + overrides: dict[str, Any] = {} if max_memory_gb is not None: - self.config.max_memory_gb = max_memory_gb + overrides["max_memory_gb"] = max_memory_gb if max_patterns is not None: - self.config.max_patterns = max_patterns - + overrides["max_patterns"] = max_patterns for key, value in kwargs.items(): # Fail fast on typos: silently dropping an unknown override means a # user's tuning quietly does nothing. - if not hasattr(self.config, key): + if not hasattr(base, key): raise ValueError(f"Unknown config option {key!r} (see BigDialConfig for valid fields)") - # Special handling for output_format to convert string to enum if key == "output_format" and isinstance(value, str): value = OutputFormat(value.lower()) - setattr(self.config, key, value) - validate_config(self.config) + overrides[key] = value + # replace() builds a new config, so overrides are validated and memory-clamped like any other. + self.config = replace(base, **overrides) # Setup logging based on config (after overrides applied) setup_logging( @@ -124,7 +123,7 @@ def _reset_components(self) -> None: self.deduplicator = BoundedDeduplicator(self.config.dedup_cache_size, self.config.hash_algorithm) self.pattern_extractor = PatternExtractor(self.config) if self.config.fuzzy_threshold and self.level != ProcessingLevel.STANDARD: - self.fuzzy_dedup = FuzzyDeduplicator(self.config.fuzzy_threshold) + self.fuzzy_dedup = FuzzyDeduplicator(self.config.fuzzy_threshold, self.config.dedup_cache_size) else: self.fuzzy_dedup = None if self.mode in [ProcessingMode.TEMPORAL, ProcessingMode.HYBRID]: @@ -349,7 +348,7 @@ def _process_pattern_mode(self, source: Source) -> list[str]: # Collect the examples of quiet patterns (the extractor has already applied max_patterns to # them). Severe lines come from the severe shapes that reduce() adds, one per shape. - result = [] + result: list[str] = [] for pattern in patterns: result.extend(line for line in pattern.examples if not is_severe(line)) @@ -391,13 +390,18 @@ def _process_anomaly_mode(self, source: Source) -> list[str]: self.logger.info("Reading, deduplicating and anomaly detection") anomalous, normal = self._detect_quiet_anomalies(self.anomaly_detector, source) - # Keep all anomalies (capped) + sample of normal - result = anomalous[: self.config.max_patterns // 2] + cap = self.config.max_patterns + if not anomalous and len(normal) < MIN_SCORED_LINES: + # Too few lines to score, so they fill the cap. + result = [] + normal_budget = cap + else: + # Half the cap (rounded up) for anomalies, then a quarter, at least one, for normal context. + result = anomalous[: (cap + 1) // 2] + normal_budget = min(cap - len(result), max(cap // 4, 1)) - # Add some normal lines for context. Seeded RNG so a re-run over the - # same input yields the same sample (consistent with the reproducibility - # guarantees elsewhere); there is no security requirement here. - normal_sample_size = min(len(normal), self.config.max_patterns // 4) + # Seeded so a re-run over the same input yields the same sample. + normal_sample_size = min(len(normal), normal_budget) if normal_sample_size > 0: result.extend(random.Random(0).sample(normal, normal_sample_size)) @@ -419,14 +423,10 @@ def _process_temporal_mode(self, source: Source) -> list[str]: self.logger.info("Phase 2/2: Temporal processing") temporal_results = self.temporal_processor.process_temporal(lines) - # One example per (window, template), the most important first. - def rank(patterns: list[dict]) -> list[dict]: - return sorted(patterns, key=lambda p: pattern_priority(p["template"], p["count"]), reverse=True) - - result = [p["example"] for p in rank(temporal_results["temporal_patterns"])[: self.config.max_patterns]] - result.extend(p["example"] for p in rank(temporal_results["timeless_patterns"])[:100]) - - return result + # One example per (window, template) or timeless template, the most important first, under one cap. + candidates = temporal_results["temporal_patterns"] + temporal_results["timeless_patterns"] + ranked = sorted(candidates, key=lambda p: pattern_priority(p["template"], p["count"]), reverse=True) + return [p["example"] for p in ranked[: self.config.max_patterns]] def _process_hybrid_mode(self, source: Source) -> list[str]: """Process using combined approach.""" @@ -444,9 +444,10 @@ def _process_hybrid_mode(self, source: Source) -> list[str]: self.deduplicator = BoundedDeduplicator(self.config.dedup_cache_size, self.config.hash_algorithm) anomalous, _ = self._detect_quiet_anomalies(self.anomaly_detector, source) - # Combine, preferring anomalies - result = anomalous[: self.config.max_patterns // 2] - result.extend(pattern_lines[: self.config.max_patterns // 2]) + # Anomalies take half the cap (rounded up), patterns half, at least one, of what is left. + cap = self.config.max_patterns + result = anomalous[: (cap + 1) // 2] + result.extend(pattern_lines[: min(cap - len(result), max(cap // 2, 1))]) else: result = pattern_lines diff --git a/src/logreducer/kafka.py b/src/logreducer/kafka.py index 74eaac6..e01a1c1 100644 --- a/src/logreducer/kafka.py +++ b/src/logreducer/kafka.py @@ -13,24 +13,24 @@ be finite and re-iterable (the reducer counts lines by exhausting the source, and re-reads it for multi-pass modes): -* **Bounded.** The read stops when every assigned partition has reached its end - (``enable.partition.eof`` -> ``_PARTITION_EOF``), which is the deterministic - "caught up with the topic" signal. A consecutive-idle-poll count is only a - backstop for the never-assigned case (e.g. a missing topic). -* **No commit.** Offsets are never committed, so each pass re-reads from the - earliest offset - that is what makes the source re-iterable. An application +* **Bounded.** The read records each partition's end offset when it is assigned + and stops once every partition has reached it, by offset or by + ``_PARTITION_EOF``. Messages written after that point wait for the next read. + A read that goes ``idle_timeout`` seconds without a message short of those + offsets raises, so a partial read never looks like a complete one. +* **No commit.** Offsets are never committed, and every assigned partition is + rewound to its earliest offset whatever the group has committed, so each pass + re-reads the whole topic - that is what makes the source re-iterable. An application that needs offset tracking should consume itself and hand the reducer its own iterable of ``str``. Install: ``pip install 'logreducer[kafka]'``. """ -from __future__ import annotations - import importlib.util import time from collections.abc import Iterable, Iterator -from typing import Any +from typing import Any, Self # Production-lean librdkafka defaults: durable produces, bounded timeouts, # cheap batching. Any key can be overridden by the user config. @@ -53,8 +53,7 @@ "max.poll.interval.ms": 300000, # 5 minutes max poll interval "fetch.min.bytes": 1, # Return immediately with any data "fetch.wait.max.ms": 500, # Max wait for fetch.min.bytes - # logreducer-specific: needed so a drained partition reports EOF, which is - # how the bounded read knows it has caught up with the topic. + # Ends a partition whose last offsets hold no message (transaction markers, compaction). "enable.partition.eof": True, } @@ -119,10 +118,10 @@ def _normalise_config(config: str | dict[str, Any]) -> dict[str, Any]: class KafkaSource: """A bounded, re-iterable stream of log lines from a Kafka topic. - Each iteration creates a fresh consumer, subscribes, and reads until every - assigned partition reports end-of-partition (i.e. the topic is drained), - decoding each message value to a ``str`` line. Offsets are never committed, - so every pass re-reads from the earliest offset. + Each iteration creates a fresh consumer, subscribes, and reads every partition + up to the end offset it had when assigned, decoding each message value to a + ``str`` line. Offsets are never committed, so every pass re-reads from the + earliest offset. """ def __init__( @@ -134,7 +133,7 @@ def __init__( verify_ssl: bool = True, max_messages: int | None = None, poll_timeout: float = 1.0, - idle_polls: int = 3, + idle_timeout: float = 30.0, assignment_timeout: float = 30.0, encoding: str = "utf-8", ) -> None: @@ -147,8 +146,9 @@ def __init__( verify_ssl: If False, disable TLS certificate verification. max_messages: Optional hard cap on lines read per pass. poll_timeout: Seconds to block per ``poll`` call. - idle_polls: Backstop for a never-EOF broker - once partitions are - assigned, stop after this many consecutive empty polls. + idle_timeout: Backstop for a stalled broker. Once partitions are + assigned, iteration raises if this many seconds pass with no + message or EOF while a partition is short of its end offset. assignment_timeout: Backstop for the never-assigned case (unreachable broker, missing topic, auth failure). If no partition is assigned within this many seconds, iteration raises instead of hanging. @@ -163,25 +163,31 @@ def __init__( self.topics = [topics] if isinstance(topics, str) else list(topics) self.max_messages = max_messages self.poll_timeout = poll_timeout - self.idle_polls = idle_polls + self.idle_timeout = idle_timeout self.assignment_timeout = assignment_timeout self.encoding = encoding def __iter__(self) -> Iterator[str]: - from confluent_kafka import Consumer, KafkaError + from confluent_kafka import OFFSET_BEGINNING, Consumer, KafkaError + + def from_the_start(consumer: Any, partitions: list[Any]) -> None: + # A committed group offset would skip the history before it. + for tp in partitions: + tp.offset = OFFSET_BEGINNING + consumer.assign(partitions) consumer = Consumer(self._config) - consumer.subscribe(self.topics) - eof_partitions: set[tuple[str, int]] = set() - empty_polls = 0 + consumer.subscribe(self.topics, on_assign=from_the_start) + # End offset of each partition when first assigned: the read stops there, so a write + # made during it is left for the next pass. + end_offsets: dict[tuple[str, int], int] = {} + drained: set[tuple[str, int]] = set() emitted = 0 - # Never-assigned backstop: an unreachable broker, a missing topic, or an - # auth failure leaves assignment() empty forever, so empty_polls (gated - # on assignment) never fires and the loop would spin indefinitely. Bail - # if no partition is ever assigned within the timeout - a stuck - # dependency surfaced as a real error, not a silent hang. + # An unreachable broker, a missing topic or an auth failure leaves assignment() empty + # forever, so a read that is never assigned raises after assignment_timeout. ever_assigned = False assignment_deadline = time.monotonic() + self.assignment_timeout + last_event = time.monotonic() try: while True: if self.max_messages is not None and emitted >= self.max_messages: @@ -189,8 +195,14 @@ def __iter__(self) -> Iterator[str]: msg = consumer.poll(self.poll_timeout) - if consumer.assignment(): - ever_assigned = True + assignment = consumer.assignment() + if assignment: + if not ever_assigned: + ever_assigned = True + last_event = time.monotonic() + self._record_end_offsets(consumer, assignment, end_offsets, drained) + if drained >= end_offsets.keys(): + break elif not ever_assigned and time.monotonic() >= assignment_deadline: raise KafkaConsumerError( f"no partition assignment within {self.assignment_timeout:.0f}s for " @@ -198,28 +210,32 @@ def __iter__(self) -> Iterator[str]: ) if msg is None: - # Before partitions are assigned an empty poll just means the - # group is still joining - don't count it. Once assigned, - # repeated empties are the backstop for a never-EOF broker. - if consumer.assignment(): - empty_polls += 1 - if empty_polls >= self.idle_polls: - break + if ever_assigned and time.monotonic() - last_event >= self.idle_timeout: + short = sorted(end_offsets.keys() - drained) + raise KafkaConsumerError( + f"read stopped short of the end: no message for {self.idle_timeout:.0f}s " + f"with partitions {short!r} still below their end offsets" + ) continue - empty_polls = 0 + last_event = time.monotonic() + topic, partition = msg.topic(), msg.partition() error = msg.error() + if error and error.code() != KafkaError._PARTITION_EOF: + raise KafkaConsumerError(f"Kafka error: {error.str()}", error_code=error.code()) + if topic is None or partition is None: + continue + partition_key = (topic, partition) if error: - if error.code() == KafkaError._PARTITION_EOF: - topic, partition = msg.topic(), msg.partition() - if topic is not None and partition is not None: - eof_partitions.add((topic, partition)) - assigned = {(tp.topic, tp.partition) for tp in consumer.assignment()} - # Drained: every partition we hold has reported its end. - if assigned and eof_partitions >= assigned: - break + drained.add(partition_key) + continue + + end = end_offsets.get(partition_key) + offset = msg.offset() + if end is not None and offset is not None and offset >= end - 1: + drained.add(partition_key) + if offset >= end: continue - raise KafkaConsumerError(f"Kafka error: {error.str()}", error_code=error.code()) value = msg.value() if value is None: @@ -231,6 +247,28 @@ def __iter__(self) -> Iterator[str]: finally: consumer.close() + def _record_end_offsets( + self, + consumer: Any, + assignment: list[Any], + end_offsets: dict[tuple[str, int], int], + drained: set[tuple[str, int]], + ) -> None: + """Record the high watermark of each newly assigned partition, marking empty ones drained.""" + from confluent_kafka import KafkaException + + for tp in assignment: + key = (tp.topic, tp.partition) + if key in end_offsets: + continue + try: + low, high = consumer.get_watermark_offsets(tp, timeout=self.assignment_timeout, cached=False) + except KafkaException as exc: + raise KafkaConsumerError(f"cannot read the end offset of {key!r}: {exc}") from exc + end_offsets[key] = high + if high <= low: + drained.add(key) + def __repr__(self) -> str: return f"KafkaSource(topics={self.topics!r}, config={mask_credentials(self._config)!r})" @@ -251,6 +289,7 @@ def __init__( key: str | None = None, verify_ssl: bool = True, flush_timeout: float | None = None, + queue_full_timeout: float = 30.0, encoding: str = "utf-8", ) -> None: """Build a Kafka sink. @@ -261,6 +300,8 @@ def __init__( key: Optional fixed message key applied to every line. verify_ssl: If False, disable TLS certificate verification. flush_timeout: Seconds to wait on the final flush (None = infinite). + queue_full_timeout: Seconds one line waits for space in a full local + producer queue before ``write`` gives up. encoding: Text encoding for the produced message values. """ try: @@ -272,6 +313,7 @@ def __init__( self.topic = topic self.key = key self.flush_timeout = flush_timeout + self.queue_full_timeout = queue_full_timeout self.encoding = encoding self._producer = Producer(self._config) @@ -279,12 +321,13 @@ def write(self, lines: Iterable[str]) -> int: """Produce each line to the topic and flush; return the number delivered. With the default ``flush_timeout=None`` the final flush blocks until the - queue drains. + queue drains. The flush runs even when a produce raises, so lines queued + before the failure are delivered or counted. Raises: KafkaDeliveryError: A message was rejected by the broker, expired - before delivery, or was still queued when a finite - ``flush_timeout`` ran out. + before delivery, was still queued when a finite ``flush_timeout`` + ran out, or found the local queue full for ``queue_full_timeout``. """ key_bytes = self.key.encode(self.encoding) if self.key is not None else None produced = 0 @@ -294,11 +337,12 @@ def on_delivery(err: Any, _msg: Any) -> None: if err is not None: errors.append(str(err)) - for line in lines: - self._producer.produce(self.topic, value=line.encode(self.encoding), key=key_bytes, on_delivery=on_delivery) - self._producer.poll(0) # Trigger delivery callbacks (non-blocking) - produced += 1 - undelivered = self.flush() + try: + for line in lines: + self._produce(line.encode(self.encoding), key_bytes, on_delivery) + produced += 1 + finally: + undelivered = self.flush() if errors or undelivered: raise KafkaDeliveryError( f"{len(errors)} of {produced} messages failed and {undelivered} were still queued " @@ -308,6 +352,24 @@ def on_delivery(err: Any, _msg: Any) -> None: ) return produced + def _produce(self, value: bytes, key: bytes | None, on_delivery: Any) -> None: + """Queue one message, serving delivery reports to free space while the local queue is full.""" + deadline = time.monotonic() + self.queue_full_timeout + while True: + try: + self._producer.produce(self.topic, value=value, key=key, on_delivery=on_delivery) + break + except BufferError: + if time.monotonic() >= deadline: + queued = len(self._producer) + raise KafkaDeliveryError( + f"local producer queue stayed full for {self.queue_full_timeout:.0f}s for topic {self.topic!r}", + failed=0, + undelivered=queued, + ) from None + self._producer.poll(0.1) + self._producer.poll(0) + def flush(self) -> int: """Wait for outstanding messages; return the count still in queue.""" if self.flush_timeout is not None: @@ -316,7 +378,7 @@ def flush(self) -> int: result = self._producer.flush() return result - def __enter__(self) -> KafkaSink: + def __enter__(self) -> Self: return self def __exit__(self, *exc: object) -> None: diff --git a/src/logreducer/patterns.py b/src/logreducer/patterns.py index 4df114b..09c3958 100644 --- a/src/logreducer/patterns.py +++ b/src/logreducer/patterns.py @@ -1,9 +1,12 @@ """Pattern extraction (Drain3 template mining) and fuzzy deduplication.""" +import heapq import logging +import re +from collections import deque from collections.abc import Callable, Iterable, Iterator, Sequence from dataclasses import dataclass, field -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, cast from .memory import Occurrence from .template_mining import MaskingInstruction, TemplateMiner, TemplateMinerConfig @@ -94,13 +97,13 @@ def _typed_masking_instructions(timestamp: bool = False) -> tuple[MaskingInstruc return tuple(MaskingInstruction(f"{start}(?:{pattern}){end}", name) for pattern, name in shapes) -_ALWAYS_KEPT_LEVELS = ("CRITICAL", "FATAL", "ERROR") +# A level is a whole word in any case, so a field name like error_count does not count. +_SEVERE_LEVEL = re.compile(r"\b(?:CRITICAL|FATAL|ERROR)\b", re.IGNORECASE) def is_severe(text: str) -> bool: """True for a line or template that names a CRITICAL, FATAL or ERROR level.""" - upper = text.upper() - return any(level in upper for level in _ALWAYS_KEPT_LEVELS) + return _SEVERE_LEVEL.search(text) is not None class SevereShapes: @@ -172,10 +175,63 @@ def pattern_priority(template: str, count: int, severe: bool = False) -> float: return priority +class _RetiredPatterns: + """Patterns whose cluster was LRU-evicted, holding only those the capped ranking can still return. + + A retired pattern's template no longer changes, and its count grows only while the dedup + cache still holds an ``Occurrence`` pointing at it. That cache is FIFO, so ``window`` lines + after the last one that carried an ``Occurrence``, the count is final. Final patterns compete + for the ``cap`` places the output has, and one that loses can never win one back. Severe + patterns are all kept, as the cap does not apply to them. + """ + + def __init__(self, cap: int, min_count: int, window: int) -> None: + self._cap = cap + self._min_count = min_count + self._window = window + self._severe: list[tuple[int, LogPattern]] = [] + # Min-heap on (priority, -cluster_id): the root is the one the final sort ranks last. + self._best: list[tuple[float, int, LogPattern]] = [] + # Min-heap on the line number of the pattern's last Occurrence. + self._counting: list[tuple[int, int, LogPattern]] = [] + + def add(self, cluster_id: int, pattern: LogPattern, last_occurrence: int | None) -> None: + """Retire the pattern of an evicted cluster, given the line number of its last ``Occurrence``.""" + if pattern.severe: + self._severe.append((cluster_id, pattern)) + elif last_occurrence is None: + self._offer(cluster_id, pattern) + else: + heapq.heappush(self._counting, (last_occurrence, cluster_id, pattern)) + + def advance(self, line_number: int) -> None: + """Rank every retired pattern whose count can no longer change by ``line_number``.""" + while self._counting and line_number - self._counting[0][0] >= self._window: + _, cluster_id, pattern = heapq.heappop(self._counting) + self._offer(cluster_id, pattern) + + def _offer(self, cluster_id: int, pattern: LogPattern) -> None: + if pattern.count < self._min_count: + return + heapq.heappush(self._best, (_priority(pattern), -cluster_id, pattern)) + if len(self._best) > self._cap: + heapq.heappop(self._best) + + def items(self) -> list[tuple[int, LogPattern]]: + """Return every retired pattern that could still reach the output, keyed by cluster id.""" + best = [(-negative_id, pattern) for _, negative_id, pattern in self._best] + counting = [(cluster_id, pattern) for _, cluster_id, pattern in self._counting] + return self._severe + best + counting + + +def _priority(pattern: LogPattern) -> float: + return pattern_priority(pattern.template, pattern.count, pattern.severe) + + class PatternExtractor: """Extract patterns using the online template miner (see template_mining).""" - def __init__(self, config: "BigDialConfig") -> None: + def __init__(self, config: BigDialConfig) -> None: self.config = config self.setup_drain3() @@ -199,8 +255,9 @@ def extract_patterns(self, lines: Iterable[str]) -> list[LogPattern]: """Extract patterns from lines (accepts a stream; only clusters are held). Feeds each line into the online miner, so the caller can pass a - generator - no need to materialise the whole line set. Memory is bounded - by the number of clusters (see ``max_clusters``), not the line count. + generator - no need to materialise the whole line set. With ``max_clusters`` + set, memory is bounded by it and by ``max_patterns``, not by the line count + or the number of distinct templates. """ return self.extract_counted_patterns((line, None) for line in lines) @@ -209,11 +266,23 @@ def extract_counted_patterns(self, pairs: Iterable[tuple[str, Occurrence | None] Repeats that exact dedup drops after a line was kept still count towards the pattern that line joined, so 100 identical errors make one pattern - with ``count == 100``. + with ``count == 100``. The ``Occurrence`` objects must come from a dedup + cache, and any fuzzy index between, of at most ``dedup_cache_size`` lines: + an evicted pattern's count is taken as final twice that many lines after + its last ``Occurrence``, once a near-duplicate matched to it has also left + the dedup cache. """ pattern_map: dict[int, LogPattern] = {} + last_occurrence: dict[int, int] = {} + retired = _RetiredPatterns( + self.config.max_patterns, self.config.min_pattern_occurrences, 2 * self.config.dedup_cache_size + ) + max_clusters = self.config.max_clusters + # Sweeping at twice the store's bound costs O(1) per line, amortised. + sweep_at = None if max_clusters is None else 2 * max_clusters - for line, occurrence in pairs: + for line_number, (line, occurrence) in enumerate(pairs): + retired.advance(line_number) cluster = self.miner.add_log_message(line) pattern = pattern_map.get(cluster.cluster_id) if pattern is None: @@ -223,10 +292,14 @@ def extract_counted_patterns(self, pairs: Iterable[tuple[str, Occurrence | None] pattern.count += 1 if occurrence is not None: occurrence.target = pattern + last_occurrence[cluster.cluster_id] = line_number if len(pattern.examples) < self.config.examples_per_pattern: pattern.examples.append(line) + if sweep_at is not None and len(pattern_map) >= sweep_at: + self._retire_evicted(pattern_map, last_occurrence, retired) + # Later lines widen a cluster's slots to <*>, so the stored template is # re-read once the feed ends; an LRU-evicted cluster keeps its snapshot. for cluster_id, pattern in pattern_map.items(): @@ -234,7 +307,9 @@ def extract_counted_patterns(self, pairs: Iterable[tuple[str, Occurrence | None] if live_cluster is not None: pattern.template = live_cluster.get_template() - patterns = list(pattern_map.values()) + # Cluster ids rise in creation order, which the stable sort below breaks ties on. + candidates = sorted([*pattern_map.items(), *retired.items()], key=lambda item: item[0]) + patterns = [pattern for _, pattern in candidates] # Apply filters patterns = self._filter_by_occurrence(patterns) @@ -244,6 +319,14 @@ def extract_counted_patterns(self, pairs: Iterable[tuple[str, Occurrence | None] patterns.sort(key=lambda p: p.priority, reverse=True) return cap_outside_severe(patterns, self.config.max_patterns, lambda p: p.severe) + def _retire_evicted( + self, pattern_map: dict[int, LogPattern], last_occurrence: dict[int, int], retired: _RetiredPatterns + ) -> None: + """Move the patterns of clusters the miner has evicted out of ``pattern_map``.""" + evicted = [cluster_id for cluster_id in pattern_map if self.miner.get_cluster(cluster_id) is None] + for cluster_id in evicted: + retired.add(cluster_id, pattern_map.pop(cluster_id), last_occurrence.pop(cluster_id, None)) + def _filter_by_occurrence(self, patterns: list[LogPattern]) -> list[LogPattern]: """Filter patterns by minimum occurrence, keeping every CRITICAL, FATAL or ERROR one.""" return [p for p in patterns if p.count >= self.config.min_pattern_occurrences or p.severe] @@ -257,10 +340,30 @@ def _calculate_priority(self, patterns: list[LogPattern]) -> list[LogPattern]: class FuzzyDeduplicator: - """Fuzzy (near-duplicate) deduplication using MinHash LSH.""" + """Fuzzy (near-duplicate) deduplication using MinHash LSH. + + With ``max_entries`` set, the index holds the most recently admitted lines and drops the + oldest first, the same sliding window ``BoundedDeduplicator`` keeps for exact repeats. A + near-duplicate of a line that has left the window is admitted again. + """ - def __init__(self, threshold: float = 0.8): + def __init__(self, threshold: float = 0.8, max_entries: int | None = None): + """Create a deduplicator. + + Args: + threshold: Jaccard similarity at or above which a line is a near-duplicate. + max_entries: Admitted lines the index holds. ``None`` keeps every one. + + Raises: + ValueError: If ``max_entries`` is below 1. + """ + if max_entries is not None and max_entries < 1: + raise ValueError(f"max_entries must be at least 1 or None, got {max_entries!r}") self.threshold = threshold + self.max_entries = max_entries + self._keys: deque[int] = deque() + self._occurrences: dict[int, Occurrence] = {} + self._next_key = 0 self.enabled = MINHASH_AVAILABLE if self.enabled: @@ -272,36 +375,52 @@ def deduplicate_stream(self, lines: Iterable[str]) -> Iterator[str]: """Yield near-unique lines as they arrive (streaming near-dup filter). Queries the LSH per line and inserts only new ones, so the reducer never - materialises the full unique-line list. Note the LSH itself grows with - the number of near-unique lines - that is inherent to fuzzy dedup. + materialises the full unique-line list. The index grows with near-unique + lines up to ``max_entries``. """ if not self.enabled: yield from lines return - for i, line in enumerate(lines): - if self._admit(i, line): + for line in lines: + if self._admit(line): yield line def deduplicate_pairs(self, pairs: Iterable[tuple[str, Occurrence]]) -> Iterator[tuple[str, Occurrence]]: - """``deduplicate_stream`` over (line, Occurrence) pairs, keeping each pair intact.""" + """``deduplicate_stream`` over (line, Occurrence) pairs, keeping each pair intact. + + A dropped near-duplicate, and every later exact repeat of it, counts towards the + pattern of the admitted line it matched. + """ if not self.enabled: yield from pairs return - for i, pair in enumerate(pairs): - if self._admit(i, pair[0]): + for pair in pairs: + if self._admit(*pair): yield pair - def _admit(self, index: int, line: str) -> bool: - """True (and remembered) when no near-duplicate of ``line`` was admitted before.""" + def _admit(self, line: str, occurrence: Occurrence | None = None) -> bool: + """True (and remembered) when no near-duplicate of ``line`` is in the index.""" m = MinHash(num_perm=64) for word in line.split(): m.update(word.encode("utf-8", "surrogatepass")) - if self.lsh.query(m): + if matches := cast("list[int]", self.lsh.query(m)): + if occurrence is not None and (admitted := self._occurrences.get(min(matches))) is not None: + occurrence.target = admitted.target + admitted.repeat() return False - self.lsh.insert(f"line_{index}", m) + if self.max_entries is not None and len(self._keys) >= self.max_entries: + evicted = self._keys.popleft() + self.lsh.remove(evicted) + self._occurrences.pop(evicted, None) + key = self._next_key + self._next_key += 1 + self.lsh.insert(key, m) + self._keys.append(key) + if occurrence is not None: + self._occurrences[key] = occurrence return True def deduplicate(self, lines: Iterable[str]) -> list[str]: diff --git a/src/logreducer/sampling.py b/src/logreducer/sampling.py index 0c6e8aa..f4b13d9 100644 --- a/src/logreducer/sampling.py +++ b/src/logreducer/sampling.py @@ -2,8 +2,8 @@ Three concerns, all pure and server-free so they unit-test without a database: -* ``build_sample_sql`` - wrap an arbitrary user query in a deterministic, - per-dialect random predicate so a fraction of rows is returned reproducibly +* ``build_sample_sql`` - wrap an arbitrary user query in a per-dialect predicate + (a seeded row hash, or ``random()`` unseeded) so a fraction of rows is returned (needed because ``TABLESAMPLE`` only applies to a base table, not the arbitrary ``SELECT`` a Source is given). Mirrors ibis: raise when a seed cannot be honoured deterministically rather than fake reproducibility. @@ -17,7 +17,7 @@ import math import random -from collections.abc import Iterable +from collections.abc import Iterable, Sequence from itertools import islice @@ -88,27 +88,40 @@ def wrap_subquery(query: str) -> str: return query.rstrip() + "\n" -def _pg_sample(query: str, fraction: float, seed: int | None) -> tuple[str | None, str]: - # setseed makes random() deterministic for the rest of the connection; it - # takes a value in [-1, 1]. Each pass opens a fresh connection and re-seeds, - # so the sample is identical across passes (re-iterable). Caveat: strict - # identity also depends on PostgreSQL visiting rows in the same order - - # synchronized_seqscans on a busy table can rotate the scan start; ORDER BY - # in the query if bit-exact re-reads matter. - setup = None - if seed is not None: - s = ((seed % 2_000_000) / 1_000_000.0) - 1.0 - setup = f"SELECT setseed({s!r})" - return setup, f"SELECT * FROM ({wrap_subquery(query)}) AS _lr_sample WHERE random() < {float(fraction)!r}" +# A seeded sample keeps a row when the first 32 bits of md5(row text + seed) fall under +# fraction * 2**32, so the same row gets the same verdict however the engine scans. +_HASH_SPACE = 2**32 + + +def _hash_threshold(fraction: float) -> int: + return int(float(fraction) * _HASH_SPACE) + +def _pg_sample( + query: str, fraction: float, seed: int | None, columns: Sequence[str] | None = None +) -> tuple[str | None, str]: + wrapped = f"SELECT * FROM ({wrap_subquery(query)}) AS _lr_sample" + if seed is None: + return None, f"{wrapped} WHERE random() < {float(fraction)!r}" + row_hash = f"('x' || lpad(substr(md5(_lr_sample::text || '{int(seed)}'), 1, 8), 16, '0'))::bit(64)::bigint" + return None, f"{wrapped} WHERE {row_hash} < {_hash_threshold(fraction)}" -def _mysql_sample(query: str, fraction: float, seed: int | None) -> tuple[str | None, str]: - # RAND(seed) with a constant seed is a repeatable per-row sequence. - rand = f"rand({int(seed)})" if seed is not None else "rand()" - return None, f"SELECT * FROM ({wrap_subquery(query)}) AS _lr_sample WHERE {rand} < {float(fraction)!r}" +def _mysql_sample( + query: str, fraction: float, seed: int | None, columns: Sequence[str] | None = None +) -> tuple[str | None, str]: + wrapped = f"SELECT * FROM ({wrap_subquery(query)}) AS _lr_sample" + if seed is None: + return None, f"{wrapped} WHERE rand() < {float(fraction)!r}" + if not columns: + raise ValueError("a seeded MySQL sample needs the query's quoted column names") + row_text = f"CONCAT_WS(CHAR(31), {', '.join(columns)}, '{int(seed)}')" + return None, f"{wrapped} WHERE CONV(SUBSTRING(MD5({row_text}), 1, 8), 16, 10) < {_hash_threshold(fraction)}" -def _sqlite_sample(query: str, fraction: float, seed: int | None) -> tuple[str | None, str]: + +def _sqlite_sample( + query: str, fraction: float, seed: int | None, columns: Sequence[str] | None = None +) -> tuple[str | None, str]: if seed is not None: raise SamplingNotSupported( "sqlite has no seedable RNG, so seeded (reproducible) sampling is unsupported; " @@ -133,20 +146,34 @@ def _check_fraction(fraction: float) -> None: raise ValueError(f"sample fraction must be in (0, 1], got {fraction!r}") -def build_sample_sql(dialect: str, query: str, fraction: float, seed: int | None = None) -> tuple[str | None, str]: - """Wrap ``query`` so it returns ~``fraction`` of its rows, deterministically. +def build_sample_sql( + dialect: str, query: str, fraction: float, seed: int | None = None, columns: Sequence[str] | None = None +) -> tuple[str | None, str]: + """Wrap ``query`` so it returns ~``fraction`` of its rows. - Returns ``(setup_sql, sampled_sql)`` where ``setup_sql`` is an optional - per-connection statement to run first (PostgreSQL ``setseed``), or None. + With a seed, a row is kept by a hash of its content and the seed, so the same + rows come back whatever order the engine reads them in. - Raises ``ValueError`` for a bad fraction and ``SamplingNotSupported`` for a - dialect with no sampler or a seed the dialect cannot honour. + Returns ``(setup_sql, sampled_sql)`` where ``setup_sql`` is an optional + per-connection statement to run first, or None. + + Args: + dialect: SQLAlchemy dialect name. + query: The SELECT to sample. + fraction: Share of rows to keep, in (0, 1]. + seed: Seed for a reproducible sample. + columns: The query's column names, quoted. A seeded MySQL sample needs + them to hash the row. + + Raises: + ValueError: For a bad fraction, or a seeded MySQL sample without ``columns``. + SamplingNotSupported: For a dialect with no sampler or a seed it cannot honour. """ _check_fraction(fraction) builder = DIALECT_SAMPLERS.get(dialect) if builder is None: raise SamplingNotSupported(f"no SQL sampler for dialect {dialect!r} (supported: {sorted(DIALECT_SAMPLERS)})") - return builder(query, fraction, seed) + return builder(query, fraction, seed, columns) def build_sample_batch_sql(dialect: str, query: str, n: int) -> str: @@ -197,7 +224,11 @@ def build_table_sample_query( # No TABLESAMPLE on these - a predicate on the table (full scan, but native). if dialect in ("mysql", "mariadb"): - pred = f"rand({int(seed)}) < {float(fraction)!r}" if seed is not None else f"rand() < {float(fraction)!r}" + if seed is None: + pred = f"rand() < {float(fraction)!r}" + else: + row_hash = f"CONV(SUBSTRING(MD5(CONCAT_WS(CHAR(31), {column}, '{int(seed)}')), 1, 8), 16, 10)" + pred = f"{row_hash} < {_hash_threshold(fraction)}" elif dialect == "sqlite": if seed is not None: raise SamplingNotSupported("sqlite has no seedable RNG; seeded table sampling is unsupported") diff --git a/src/logreducer/sql.py b/src/logreducer/sql.py index 2b8e232..8cd1d1e 100644 --- a/src/logreducer/sql.py +++ b/src/logreducer/sql.py @@ -71,8 +71,9 @@ def __init__( yield_per: Server-side cursor batch size (rows fetched per round trip). Larger trades memory for fewer round trips. sample: Optional fraction in (0, 1] - return only that fraction of - rows via a per-dialect random predicate. Pass ``sample_seed`` - for a deterministic, re-iterable sample (required for the + rows. Pass ``sample_seed`` for a re-iterable sample that keeps a + row by a hash of its content, so every pass returns the same rows + whatever order the engine reads them in (required for the reducer's multi-pass modes; PostgreSQL/MySQL only). Without a seed the sample is best-effort and not reproducible across passes. sample_seed: Seed for a reproducible sample. Raises @@ -105,12 +106,34 @@ def __init__( # on SQLite) fails at construction, not deep inside iteration. self._sample_setup: str | None = None self._sample_sql: str | None = None + # A seeded MySQL sample hashes the query's columns, read on the first pass's connection. + self._needs_columns = False if sample is not None: from .sampling import build_sample_sql - self._sample_setup, self._sample_sql = build_sample_sql( - self._engine.dialect.name, query, sample, sample_seed - ) + dialect = self._engine.dialect.name + self._needs_columns = sample_seed is not None and dialect in ("mysql", "mariadb") + placeholder = ["NULL"] if self._needs_columns else None + setup, sql = build_sample_sql(dialect, query, sample, sample_seed, placeholder) + if not self._needs_columns: + self._sample_setup, self._sample_sql = setup, sql + + def _build_column_sample(self, conn: Any, sample: float) -> None: + """Build the seeded MySQL sample from the query's quoted columns, read with a zero-row run of it.""" + from sqlalchemy import text + + from .sampling import build_sample_sql, wrap_subquery + + # Risk accepted: the query is the caller's own SELECT, which this source exists to run. + # nosemgrep: python.sqlalchemy.security.audit.avoid-sqlalchemy-text.avoid-sqlalchemy-text + probe = text(f"SELECT * FROM ({wrap_subquery(self.query)}) AS _lr_columns LIMIT 0") + names = list(conn.execute(probe, self.params or {}).keys()) + quote = self._engine.dialect.identifier_preparer.quote + columns = [f"_lr_sample.{quote(name)}" for name in names] + self._sample_setup, self._sample_sql = build_sample_sql( + self._engine.dialect.name, self.query, sample, self.sample_seed, columns + ) + self._needs_columns = False @classmethod def from_table( @@ -163,8 +186,10 @@ def __iter__(self) -> Iterator[str]: from .sources import rows_to_lines - stmt = text(self._sample_sql if self._sample_sql is not None else self.query) with self._engine.connect() as conn: + if self._needs_columns and self.sample is not None: + self._build_column_sample(conn, self.sample) + stmt = text(self._sample_sql if self._sample_sql is not None else self.query) # A deterministic sample (PostgreSQL) needs its per-connection seed # set before the query, on the same connection. if self._sample_setup is not None: diff --git a/src/logreducer/target.py b/src/logreducer/target.py index cb9fb7b..6778a33 100644 --- a/src/logreducer/target.py +++ b/src/logreducer/target.py @@ -5,8 +5,9 @@ representative lines until it has the number the caller asked for - or the source runs dry, a fetch cap is hit, or the representatives stop growing. -Peak memory is bounded to roughly one batch plus the accumulator: batches are -sized to a byte budget (from the observed average row size), and a psutil +Peak memory is bounded to roughly one batch plus the accumulator: the first +batch is a 1,000-row probe, later ones are sized to a byte budget from the +observed average row size, and a psutil watchdog shrinks the batch and, as a hard backstop, stops the loop under pressure. This is the memory-safe path for "give me X lines from a huge table" without a full scan. @@ -28,7 +29,7 @@ from .core import LogReducer from .sources import Source -_DEFAULT_BATCH_ROWS = 10_000 +# Also the first batch when sizing adapts, as nothing is known about row sizes yet. _MIN_BATCH_ROWS = 1_000 @@ -105,7 +106,7 @@ def reduce_to_target( rng = random.Random(seed) seen: dict[str, None] = {} - batch = batch_rows if batch_rows is not None else _DEFAULT_BATCH_ROWS + batch = batch_rows if batch_rows is not None else _MIN_BATCH_ROWS fetches = 0 plateau = 0 reason = "max_fetches" diff --git a/src/logreducer/temporal.py b/src/logreducer/temporal.py index fe185f2..6a785a9 100644 --- a/src/logreducer/temporal.py +++ b/src/logreducer/temporal.py @@ -19,11 +19,24 @@ class LogEntry: line_number: int = 0 +def _as_naive_utc(timestamp: datetime) -> datetime: + """Return ``timestamp`` shifted to UTC without its offset. A naive one is already read as UTC.""" + if timestamp.tzinfo is None: + return timestamp + try: + return timestamp.astimezone(UTC).replace(tzinfo=None) + except OverflowError: + # Shifting a stamp at the edge of the calendar leaves datetime's range. + return timestamp.replace(tzinfo=None) + + class TimestampParser: """Parse timestamps from various formats.""" + # The format None marks ISO-8601, which fromisoformat reads with its fraction and offset. PATTERNS = [ - (r"\d{4}-\d{2}-\d{2}[T\s]\d{2}:\d{2}:\d{2}", "%Y-%m-%d %H:%M:%S"), + (r"\d{4}-\d{2}-\d{2}[T\s]\d{2}:\d{2}:\d{2}(?:[.,]\d+)?(?:Z|[+-]\d{2}:?\d{2})?", None), + (r"\d{1,2}/\w{3}/\d{4}:\d{2}:\d{2}:\d{2} [+-]\d{4}", "%d/%b/%Y:%H:%M:%S %z"), (r"\d{1,2}/\w{3}/\d{4}:\d{2}:\d{2}:\d{2}", "%d/%b/%Y:%H:%M:%S"), (r"\w{3}\s+\d{1,2}\s+\d{2}:\d{2}:\d{2}", "%b %d %H:%M:%S"), ] @@ -32,23 +45,25 @@ class TimestampParser: LEVEL_PATTERN = re.compile(r"\b(DEBUG|INFO|WARN|ERROR|CRITICAL)\b", re.I) def parse_line(self, line: str, line_number: int = 0) -> LogEntry: - """Parse log line.""" + """Parse log line, normalising a timestamp with an offset to naive UTC.""" entry = LogEntry(raw_line=line.strip(), line_number=line_number) - # Extract timestamp for pattern, date_format in self.COMPILED_PATTERNS: match = pattern.search(line) - if match: - try: - timestamp_str = match.group() - if "%Y" not in date_format: - timestamp_str = f"{datetime.now().year} {timestamp_str}" - date_format = f"%Y {date_format}" - entry.timestamp = datetime.strptime(timestamp_str.replace("T", " "), date_format) - break - except (ValueError, TypeError): - # Timestamp parsing failed, try next pattern - pass + if not match: + continue + timestamp_str = match.group() + try: + if date_format is None: + timestamp = datetime.fromisoformat(timestamp_str.replace(",", ".")) + elif "%Y" not in date_format: + timestamp = datetime.strptime(f"{datetime.now().year} {timestamp_str}", f"%Y {date_format}") + else: + timestamp = datetime.strptime(timestamp_str, date_format) + except ValueError: + continue + entry.timestamp = _as_naive_utc(timestamp) + break # Extract level level_match = self.LEVEL_PATTERN.search(line) diff --git a/tests/conftest.py b/tests/conftest.py index 18f86d7..eb5bcfc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,10 +9,10 @@ import pytest -# Load the local .env (CLICKHOUSE_*, KAFKA_BOOTSTRAP_SERVERS, ...) so the -# integration tests can reach a real local-network service when one is -# configured. An absent or empty .env is fine - those tests then fall back to a -# throwaway docker container, or skip if docker is unavailable too. +# The run's one ClickHouse and Kafka, shared by every xdist worker. +pytest_plugins = ["scalo.deployment.test_support.pytest_plugin"] + +# Load the local .env so a test can read local settings. An absent .env is fine. try: from dotenv import load_dotenv diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 9b34934..0648824 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,65 +1,22 @@ -"""Integration-test fixtures: real services, env-first, docker fallback. +"""Integration-test fixtures: real services, never mocks. -The ClickHouse and Kafka fixtures prefer a configured endpoint (env vars, -typically loaded from the project ``.env`` - see .env.example) and fall back to -a throwaway docker container; PostgreSQL and MySQL are docker-only (they exist -solely to exercise per-dialect sampling). Containers are stopped the moment the -session ends unless LOGREDUCER_KEEP_CONTAINERS=1. If neither an endpoint nor -docker is available, dependent tests skip rather than fail. +ClickHouse and Kafka are the run's shared services from scalo's test plugin (loaded in the +root ``conftest.py``): one container each, shared by every xdist worker, or the pet the +``HYPERI_TEST_*`` settings name. Each test gets a database or topics of its own. PostgreSQL +and MySQL, which the plugin does not provide, are throwaway testcontainers stopped when the +session ends unless LOGREDUCER_KEEP_CONTAINERS=1, and skip without Docker. """ -from __future__ import annotations - import contextlib import os -import time from collections.abc import Iterator from typing import Any import pytest -def _prefixed(names: tuple[str, ...]) -> tuple[str, ...]: - """Expand names with the optional env-prefix cascade. - - LOGREDUCER_ENV_PREFIX="DFE" makes DFE_CLICKHOUSE_HOST win over - CLICKHOUSE_HOST (prefixed first, bare fallback) - the scalo config-cascade - convention, so a .env written for another project (e.g. dfe-engine's - DFE_* names) drops in without renaming. - """ - prefix = os.environ.get("LOGREDUCER_ENV_PREFIX", "").strip().rstrip("_") - if not prefix: - return names - expanded: list[str] = [] - for name in names: - expanded.append(f"{prefix}_{name}") - expanded.append(name) - return tuple(expanded) - - -def _env_bool(name: str, default: bool = False) -> bool: - for candidate in _prefixed((name,)): - value = os.environ.get(candidate) - if value is not None: - return value.strip().lower() in ("1", "true", "yes", "on") - return default - - -def _env_first(*names: str, default: str = "") -> str: - for name in _prefixed(names): - value = os.environ.get(name) - if value: - return value - return default - - def _keep_containers() -> bool: - """LOGREDUCER_KEEP_CONTAINERS=1 leaves docker services RUNNING after the run. - - Default is to stop/remove them. Keeping them lets you re-run fast: grab the - printed endpoint and export it (KAFKA_BOOTSTRAP_SERVERS / CLICKHOUSE_* / a - DB URL) so the next run takes the env-first path and skips container startup. - """ + """LOGREDUCER_KEEP_CONTAINERS=1 leaves the PostgreSQL and MySQL containers RUNNING after the run.""" return os.environ.get("LOGREDUCER_KEEP_CONTAINERS", "").strip().lower() in ("1", "true", "yes", "on") @@ -73,205 +30,39 @@ def _stop(container: Any, label: str, endpoint: str) -> None: # --------------------------------------------------------------------------- -# ClickHouse +# ClickHouse and Kafka - the run's shared services # --------------------------------------------------------------------------- -def _clickhouse_from_env() -> Any | None: - """Return a connected clickhouse-connect client from env config, or None. - - Reads the CLICKHOUSE_* names (see .env.example). Returns None if unset - or unreachable. - """ - host = _env_first("CLICKHOUSE_HOST") - if not host: - return None - try: - import clickhouse_connect - - client = clickhouse_connect.get_client( - host=host, - port=int(_env_first("CLICKHOUSE_PORT", default="8123")), - username=_env_first("CLICKHOUSE_USER", default="default"), - password=_env_first("CLICKHOUSE_PASSWORD"), - secure=_env_bool("CLICKHOUSE_SECURE"), - verify=_env_bool("CLICKHOUSE_VERIFY", default=True), - connect_timeout=5, - send_receive_timeout=10, - ) - client.command("SELECT 1") - return client - except Exception as exc: - # Say WHY the configured endpoint was skipped - a typo'd password - # silently falling back to docker is painful to debug. - print(f"\n[env] CLICKHOUSE_HOST={host} configured but unusable ({exc}); falling back to docker") - return None - - -def _await_clickhouse_http(host: str, port: int, timeout: float = 45.0) -> Any: - """Poll the HTTP interface until a query succeeds (real readiness signal).""" +@pytest.fixture +def clickhouse_client(shared_clickhouse: Any, clickhouse_database: str) -> Iterator[Any]: + """A clickhouse-connect client on the run's ClickHouse, in a database of this test's own.""" import clickhouse_connect - deadline = time.monotonic() + timeout - last_exc: Exception | None = None - while time.monotonic() < deadline: - try: - client = clickhouse_connect.get_client(host=host, port=port, username="default", password="") - client.command("SELECT 1") - return client - except Exception as exc: # container HTTP not up yet - back off and retry - last_exc = exc - time.sleep(1.0) - raise TimeoutError(f"ClickHouse HTTP not ready on {host}:{port}") from last_exc - - -def _clickhouse_from_docker() -> tuple[Any, Any] | None: - """Start a throwaway ClickHouse container; return (client, container) or None. - - Uses a raw DockerContainer over the HTTP port so only clickhouse-connect is - needed (no native clickhouse-driver). Readiness gates on the server's own - "Ready for connections" log line plus a successful query. - """ - try: - from testcontainers.core.container import DockerContainer - from testcontainers.core.wait_strategies import LogMessageWaitStrategy - except ImportError: - return None - try: - container = ( - DockerContainer("clickhouse/clickhouse-server:24.8.14") - .with_exposed_ports(8123) - .waiting_for(LogMessageWaitStrategy("Ready for connections").with_startup_timeout(60)) - ) - container.start() - # Inside the try: if the HTTP probe times out, the container must - # still be stopped rather than leaked as a fixture ERROR. - host = container.get_container_host_ip() - port = int(container.get_exposed_port(8123)) - client = _await_clickhouse_http(host, port) - except Exception: - with contextlib.suppress(Exception): - container.stop() - return None - return client, container - - -@pytest.fixture(scope="session") -def clickhouse_client() -> Iterator[Any]: - """A ClickHouse client - local-network if configured, else docker, else skip.""" - client = _clickhouse_from_env() - if client is not None: - try: - yield client - finally: - with contextlib.suppress(Exception): - client.close() - return - - docker = _clickhouse_from_docker() - if docker is None: - pytest.skip("no ClickHouse: set CLICKHOUSE_* in the local .env, or start Docker") - client, container = docker + options: dict[str, Any] = {"secure": shared_clickhouse.secure} + if shared_clickhouse.ca_file: + options["ca_cert"] = shared_clickhouse.ca_file + client = clickhouse_connect.get_client( + host=shared_clickhouse.host, + port=shared_clickhouse.http_port, + username=shared_clickhouse.user, + password=shared_clickhouse.password, + database=clickhouse_database, + **options, + ) try: yield client finally: - with contextlib.suppress(Exception): - client.close() - endpoint = f"{container.get_container_host_ip()}:{container.get_exposed_port(8123)}" - _stop(container, "ClickHouse", endpoint) - - -# --------------------------------------------------------------------------- -# Kafka -# --------------------------------------------------------------------------- - - -def _kafka_config_from_env() -> str | dict[str, Any] | None: - """Return a reachable Kafka config from env, or None. - - A bare ``KAFKA_BOOTSTRAP_SERVERS`` yields a plain bootstrap string. Adding - ``KAFKA_SECURITY_PROTOCOL`` (e.g. SASL_SSL) upgrades it to a full librdkafka - config dict (SASL mechanism/username/password + TLS CA) - used to reach the - authenticated PET broker. Probed with an AdminClient before use. - """ - servers = _env_first("KAFKA_BOOTSTRAP_SERVERS") - if not servers: - return None - protocol = _env_first("KAFKA_SECURITY_PROTOCOL") - config: str | dict[str, Any] - if protocol: - config = {"bootstrap.servers": servers, "security.protocol": protocol} - for env_key, conf_key in ( - ("KAFKA_SASL_MECHANISM", "sasl.mechanism"), - ("KAFKA_SASL_USERNAME", "sasl.username"), - ("KAFKA_SASL_PASSWORD", "sasl.password"), - ("KAFKA_SSL_CA_LOCATION", "ssl.ca.location"), - ): - value = _env_first(env_key) - if value: - config[conf_key] = value - # Internal PET broker: skip TLS cert verification (same stance as - # CLICKHOUSE_VERIFY=false) - the chain uses an internal, pre-rebrand CA. - if not _env_bool("KAFKA_SSL_VERIFY", default=True): - config["enable.ssl.certificate.verification"] = "false" - else: - config = servers - try: - from confluent_kafka.admin import AdminClient - - probe = dict(config) if isinstance(config, dict) else {"bootstrap.servers": config} - probe["socket.timeout.ms"] = 6000 - AdminClient(probe).list_topics(timeout=8) - return config - except Exception as exc: - # Say WHY the configured broker was skipped rather than silently - # falling back to docker (bad SASL creds look identical otherwise). - print(f"\n[env] KAFKA_BOOTSTRAP_SERVERS={servers} configured but unusable ({exc}); falling back to docker") - return None + client.close() -def _redpanda_from_docker() -> tuple[str, Any] | None: - """Start a throwaway Redpanda container; return (bootstrap, container) or None. +@pytest.fixture +def kafka_bootstrap(shared_kafka: Any) -> dict[str, Any]: + """The librdkafka config for the run's Kafka, security settings included. - Redpanda is the docker broker (Kafka-API compatible, single binary, far - smaller/faster to start than Apache Kafka) - confluent-kafka talks to it - unchanged. + Topic auto-create is off there, so a test makes its topics with ``kafka_topics``. """ - try: - from testcontainers.community.kafka import RedpandaContainer - except ImportError: - return None - try: - container = RedpandaContainer() - container.start() - except Exception: - with contextlib.suppress(Exception): - container.stop() - return None - return container.get_bootstrap_server(), container - - -@pytest.fixture(scope="session") -def kafka_bootstrap() -> Iterator[str | dict[str, Any]]: - """A Kafka config - a configured broker (str, or a SASL dict for PET) if - reachable, else a Redpanda docker broker (str), else skip. - - The value is accepted directly by KafkaSource/KafkaSink and the corpora - loaders (all take a bootstrap string or a full librdkafka config dict). - """ - config = _kafka_config_from_env() - if config is not None: - yield config - return - - docker = _redpanda_from_docker() - if docker is None: - pytest.skip("no Kafka broker: set KAFKA_BOOTSTRAP_SERVERS, or start Docker (Redpanda)") - servers, container = docker - try: - yield servers - finally: - _stop(container, "Redpanda", servers) + return dict(shared_kafka.client_config) # --------------------------------------------------------------------------- diff --git a/tests/integration/corpora.py b/tests/integration/corpora.py index 548f223..4755e7a 100644 --- a/tests/integration/corpora.py +++ b/tests/integration/corpora.py @@ -137,15 +137,3 @@ def load_into_kafka(config: str | dict[str, Any], topic: str, name: str, *, limi if undelivered: raise RuntimeError(f"{undelivered} of {produced} messages not delivered to {topic} within 60s") return produced - - -def delete_kafka_topic(config: str | dict[str, Any], topic: str) -> None: - """Best-effort delete of a temp topic (cleanup on shared/persistent brokers).""" - import contextlib - - from confluent_kafka.admin import AdminClient - - admin = AdminClient(_kafka_conf(config)) - for future in admin.delete_topics([topic]).values(): - with contextlib.suppress(Exception): - future.result(timeout=30) diff --git a/tests/integration/test_clickhouse_source.py b/tests/integration/test_clickhouse_source.py index a5aae17..e3f9a32 100644 --- a/tests/integration/test_clickhouse_source.py +++ b/tests/integration/test_clickhouse_source.py @@ -1,13 +1,7 @@ -"""Integration test: ClickHouseSource against a real ClickHouse. +"""Integration test: ClickHouseSource against the run's shared ClickHouse. -Runs against a local/network ClickHouse when CLICKHOUSE_* is set (see -.env.example), otherwise a throwaway docker ClickHouse that is stopped as -soon as the session ends. Marked `integration`; deselect with -`-m "not integration"`. - -The query uses ClickHouse's built-in numbers() table function, so the test is -READ-ONLY: no table is created, nothing is written, nothing to clean up - safe -even against a shared cluster. +The query uses ClickHouse's built-in numbers() table function, so the test +writes nothing. Marked `integration`; deselect with `-m "not integration"`. """ import pytest diff --git a/tests/integration/test_kafka_adapters.py b/tests/integration/test_kafka_adapters.py index 96f35ac..12046da 100644 --- a/tests/integration/test_kafka_adapters.py +++ b/tests/integration/test_kafka_adapters.py @@ -1,15 +1,10 @@ -"""Integration test: KafkaSource + KafkaSink round-trip against a real broker. +"""Integration test: KafkaSource + KafkaSink against the run's shared Kafka. -Runs against a local broker when KAFKA_BOOTSTRAP_SERVERS is set, otherwise a -throwaway docker Kafka that is stopped as soon as the session ends. Marked -`integration` so the default suite skips it. - -Each run uses a unique topic and consumer group, so it is safe against a shared -broker and leaves no durable state behind. +Each test makes its own single-partition topics and names its own consumer groups through +scalo's ``kafka_topics`` and ``kafka_group`` fixtures, so it is safe on a shared broker and +its topics are deleted afterwards. """ -import uuid - import pytest from logreducer import LogReducer @@ -17,23 +12,24 @@ pytestmark = pytest.mark.integration -def test_kafka_sink_then_source_roundtrip(kafka_bootstrap): +def _config(bootstrap: dict, **overrides) -> dict: + return {**bootstrap, **overrides} + + +# The consumer holds about 1 KB locally, so a paused read is far from the end of the topic. +THROTTLED = {"queued.max.messages.kbytes": 1, "max.partition.fetch.bytes": 1024} + + +def test_kafka_sink_then_source_roundtrip(kafka_bootstrap, kafka_topics, kafka_group): from logreducer.kafka import KafkaSink, KafkaSource - topic = f"logreducer-it-{uuid.uuid4().hex[:8]}" + topic = kafka_topics("roundtrip", partitions=1) lines = [f"ERROR request failed shard={i % 3}" for i in range(30)] - # Produce with the sink. produced = KafkaSink(kafka_bootstrap, topic).write(lines) assert produced == 30 - # Consume with the source (bounded read; stops at partition EOF). - source = KafkaSource( - kafka_bootstrap, - group_id=f"logreducer-it-{uuid.uuid4().hex[:8]}", - topics=topic, - idle_polls=10, - ) + source = KafkaSource(kafka_bootstrap, group_id=kafka_group("roundtrip"), topics=topic) consumed = list(source) assert len(consumed) == 30 assert set(consumed) == set(lines) @@ -44,3 +40,90 @@ def test_kafka_sink_then_source_roundtrip(kafka_bootstrap): result = reducer.reduce(source) assert 0 < len(result) < 30 assert reducer.stats["input_lines"] == 30 + + +def test_source_stops_at_the_end_offsets_it_started_with(kafka_bootstrap, kafka_topics, kafka_group): + from logreducer.kafka import KafkaSink, KafkaSource + + topic = kafka_topics("end-offsets", partitions=1) + lines = [f"INFO first batch {i}" for i in range(2_000)] + KafkaSink(kafka_bootstrap, topic).write(lines) + source = KafkaSource(_config(kafka_bootstrap, **THROTTLED), group_id=kafka_group("end-offsets"), topics=topic) + + consumed = [] + for line in source: + if not consumed: + KafkaSink(kafka_bootstrap, topic).write([f"INFO written during the read {i}" for i in range(50)]) + consumed.append(line) + + assert consumed == lines + + +def test_source_reads_from_the_start_whatever_its_group_committed(kafka_bootstrap, kafka_topics, kafka_group): + from confluent_kafka import Consumer, TopicPartition + + from logreducer.kafka import KafkaSink, KafkaSource + + topic, group = kafka_topics("committed", partitions=1), kafka_group("committed") + lines = [f"INFO line {i}" for i in range(100)] + KafkaSink(kafka_bootstrap, topic).write(lines) + # Another member of the group commits half way through the topic. + other = Consumer(_config(kafka_bootstrap, **{"group.id": group})) + try: + other.commit(offsets=[TopicPartition(topic, 0, 50)], asynchronous=False) + finally: + other.close() + + assert list(KafkaSource(kafka_bootstrap, group_id=group, topics=topic)) == lines + + +def test_empty_polls_before_the_end_do_not_end_the_read(kafka_bootstrap, kafka_topics, kafka_group): + from logreducer.kafka import KafkaSink, KafkaSource + + topic = kafka_topics("empty-polls", partitions=1) + lines = [f"INFO line {i}" for i in range(2_000)] + KafkaSink(kafka_bootstrap, topic).write(lines) + # A 1 ms poll returns empty between most fetches. + throttled = _config(kafka_bootstrap, **THROTTLED) + source = KafkaSource(throttled, group_id=kafka_group("empty-polls"), topics=topic, poll_timeout=0.001) + + assert list(source) == lines + + +def test_a_read_that_stalls_short_of_the_end_raises(kafka_bootstrap, kafka_topics, kafka_group): + from logreducer.kafka import KafkaConsumerError, KafkaSink, KafkaSource + + topic = kafka_topics("stall", partitions=1) + KafkaSink(kafka_bootstrap, topic).write([f"INFO line {i}" for i in range(10)]) + # The broker holds each fetch until 10 MB is ready or 20 s pass, so nothing arrives inside the idle timeout. + stalled = _config(kafka_bootstrap, **{"fetch.min.bytes": 10_000_000, "fetch.wait.max.ms": 20_000}) + source = KafkaSource(stalled, group_id=kafka_group("stall"), topics=topic, idle_timeout=2.0) + + with pytest.raises(KafkaConsumerError, match="stopped short of the end"): + list(source) + + +def test_sink_waits_for_queue_space_rather_than_failing(kafka_bootstrap, kafka_topics, kafka_group): + from logreducer.kafka import KafkaSink, KafkaSource + + topic = kafka_topics("queue-full", partitions=1) + lines = [f"INFO line {i}" for i in range(2_000)] + # A 10-message local queue fills long before the broker acknowledges anything. + sink = KafkaSink(_config(kafka_bootstrap, **{"queue.buffering.max.messages": 10}), topic) + + assert sink.write(lines) == 2_000 + assert sorted(KafkaSource(kafka_bootstrap, group_id=kafka_group("queue-full"), topics=topic)) == sorted(lines) + + +def test_sink_flushes_what_it_queued_when_a_produce_fails(kafka_bootstrap, kafka_topics): + from confluent_kafka import KafkaException + + from logreducer.kafka import KafkaSink + + sink = KafkaSink(kafka_bootstrap, kafka_topics("oversized", partitions=1)) + oversized = "x" * 2_000_000 # over librdkafka's default message.max.bytes + + with pytest.raises(KafkaException): + sink.write([f"INFO line {i}" for i in range(5)] + [oversized]) + + assert len(sink._producer) == 0 diff --git a/tests/integration/test_real_datasets.py b/tests/integration/test_real_datasets.py index 82b7718..6eb4ec2 100644 --- a/tests/integration/test_real_datasets.py +++ b/tests/integration/test_real_datasets.py @@ -2,7 +2,7 @@ Verifies the core reduction capability on varied, messy, real-world logs - and that the SAME data reduces consistently whether it arrives as an in-memory list, -from PostgreSQL (SQLSource) or from a Redpanda/Kafka topic (KafkaSource). +from PostgreSQL (SQLSource) or from a Kafka topic (KafkaSource). Docker-gated where a store is involved; the in-memory sweep needs no services. No mocks - real corpora, real engines/brokers via testcontainers. @@ -81,28 +81,19 @@ def test_reduce_from_clickhouse(clickhouse_client): corpora.drop_clickhouse_table(clickhouse_client, table) -def test_reduce_from_kafka(kafka_bootstrap): - """A dataset produced to a Kafka topic reduces via KafkaSource. - - Runs against whatever broker the fixture resolves: a configured broker - from the env (possibly SASL-authenticated) or the docker Redpanda fallback. - """ +def test_reduce_from_kafka(kafka_bootstrap, kafka_topics, kafka_group): + """A dataset produced to a Kafka topic reduces via KafkaSource.""" pytest.importorskip("confluent_kafka") from logreducer.kafka import KafkaSource - name = "loghub_openssh" - topic = corpora.unique_name("logreducer-real") - group = corpora.unique_name("logreducer-grp") - try: - produced = corpora.load_into_kafka(kafka_bootstrap, topic, name, limit=20000) - assert produced == 20000 + topic = kafka_topics("real-dataset") + produced = corpora.load_into_kafka(kafka_bootstrap, topic, "loghub_openssh", limit=20000) + assert produced == 20000 - source = KafkaSource(kafka_bootstrap, group_id=group, topics=topic, max_messages=20000) - reduced = LogReducer(level="standard", mode="pattern").reduce(source) - assert reduced - assert len(reduced) < produced # real reduction happened - finally: - corpora.delete_kafka_topic(kafka_bootstrap, topic) + source = KafkaSource(kafka_bootstrap, group_id=kafka_group("real-dataset"), topics=topic, max_messages=20000) + reduced = LogReducer(level="standard", mode="pattern").reduce(source) + assert reduced + assert len(reduced) < produced # real reduction happened def test_reduce_to_target_over_real_postgres(pg_engine): diff --git a/tests/integration/test_sql_sampling.py b/tests/integration/test_sql_sampling.py index 0b67367..3e5afbc 100644 --- a/tests/integration/test_sql_sampling.py +++ b/tests/integration/test_sql_sampling.py @@ -16,6 +16,31 @@ pytestmark = pytest.mark.integration +@pytest.fixture(params=["pg_logs_engine", "mysql_logs_engine"]) +def reversed_logs(request): + """The seeded ``logs`` table and a copy whose rows are stored in the reverse order.""" + from sqlalchemy import text + + engine = request.getfixturevalue(request.param) + with engine.begin() as conn: + conn.execute(text("CREATE TABLE logs_reversed AS SELECT * FROM logs ORDER BY id DESC")) + try: + yield engine + finally: + with engine.begin() as conn: + conn.execute(text("DROP TABLE logs_reversed")) + + +def test_seeded_sample_does_not_depend_on_scan_order(reversed_logs): + forward = SQLSource(reversed_logs, "SELECT line FROM logs", sample=0.2, sample_seed=42) + backward = SQLSource(reversed_logs, "SELECT line FROM logs_reversed", sample=0.2, sample_seed=42) + + sampled = set(forward) + + assert sampled == set(backward) + assert 0 < len(sampled) < 5000 + + class TestPostgreSQLSampling: def test_seeded_sample_is_deterministic_across_passes(self, pg_logs_engine): source = SQLSource(pg_logs_engine, "SELECT line FROM logs", sample=0.2, sample_seed=42) diff --git a/tests/testdata/build_quiet_golden.py b/tests/testdata/build_quiet_golden.py index 04e9010..c5d792d 100644 --- a/tests/testdata/build_quiet_golden.py +++ b/tests/testdata/build_quiet_golden.py @@ -17,6 +17,7 @@ import gzip import hashlib import json +import re import sys from pathlib import Path @@ -30,12 +31,11 @@ # logreducer's BigDialConfig defaults for the fields the five configurations override. DEFAULTS = {"max_clusters": 10_000, "drain_similarity": 0.4, "typed_masking": False} -LEVELS = ("CRITICAL", "FATAL", "ERROR") +SEVERE_LEVEL = re.compile(r"\b(?:CRITICAL|FATAL|ERROR)\b", re.IGNORECASE) def is_severe(line: str) -> bool: - upper = line.upper() - return any(level in upper for level in LEVELS) + return SEVERE_LEVEL.search(line) is not None def read_lines(name: str, limit: int) -> list[str]: diff --git a/tests/testdata/template_mining_quiet_golden.json.gz b/tests/testdata/template_mining_quiet_golden.json.gz index d7f9635..61fbdd4 100644 Binary files a/tests/testdata/template_mining_quiet_golden.json.gz and b/tests/testdata/template_mining_quiet_golden.json.gz differ diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index ab54ca7..8ec23a7 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -2,6 +2,7 @@ Unit tests for configuration module """ +import psutil import pytest from logreducer import LogReducer @@ -68,13 +69,23 @@ def test_typed_masking_defaults_off(self): """Typed masking is opt-in; default behaviour stays unmasked.""" assert BigDialConfig().typed_masking is False - def test_post_init_memory_adjustment(self): - """Test that __post_init__ adjusts memory if too high""" - # This will depend on system memory, so we test behavior - config = BigDialConfig(max_memory_gb=999999.0) # Unrealistic high value + @pytest.mark.parametrize( + "build", + [ + pytest.param(lambda: BigDialConfig(max_memory_gb=999999.0), id="config"), + pytest.param(lambda: LogReducer(max_memory_gb=999999.0).config, id="reducer-argument"), + pytest.param( + lambda: LogReducer(config=BigDialConfig(), max_memory_gb=999999.0).config, id="injected-config-argument" + ), + ], + ) + def test_memory_above_70_percent_of_available_is_clamped(self, build): + # Available memory moves between the clamp and this line, and never exceeds the total. + assert build().max_memory_gb <= psutil.virtual_memory().total / 1024**3 * 0.7 - # Should be adjusted down based on available memory - assert config.max_memory_gb < 999999.0 + def test_reducer_override_is_validated(self): + with pytest.raises(ValueError, match="max_patterns"): + LogReducer(max_patterns=0) class TestGetPresetConfig: diff --git a/tests/unit/test_core.py b/tests/unit/test_core.py index 1b111f8..6450286 100644 --- a/tests/unit/test_core.py +++ b/tests/unit/test_core.py @@ -637,6 +637,26 @@ def _anomaly_peak_bytes(count: int) -> int: tracemalloc.stop() +@pytest.mark.parametrize("mode", ["anomaly", "hybrid"]) +@pytest.mark.parametrize("cap", [1, 2, 3, 500]) +def test_a_small_cap_still_returns_quiet_lines(mode, cap): + lines = [f"INFO {_letters(n)} request {n} took {n % 7}ms" for n in range(60)] * 2 + + out = LogReducer(mode=mode, max_patterns=cap).reduce(lines) + + assert 1 <= len(out) <= cap + + +@pytest.mark.parametrize(("cap", "expected"), [(1, 1), (3, 3), (10, 5)]) +def test_too_few_lines_to_score_fill_the_anomaly_cap(cap, expected): + lines = [f"INFO {_letters(n)} started" for n in range(5)] + + out = LogReducer(mode="anomaly", max_patterns=cap).reduce(lines) + + assert len(out) == expected + assert set(out) <= set(lines) + + def test_anomaly_peak_memory_does_not_grow_with_the_input(): """The detector's input is sampled while the source streams, so 10x the lines costs no more memory.""" assert _anomaly_peak_bytes(200_000) < _anomaly_peak_bytes(20_000) * 1.5 @@ -659,12 +679,25 @@ def __iter__(self): ) -def _pattern_flood_run(count: int) -> tuple[list[str], int]: +class _QuietFlood: + """A re-iterable source of ``count`` quiet lines that share no word, so each is a new template and fuzzy-unique.""" + + def __init__(self, count: int) -> None: + self.count = count + + def __iter__(self): + return ( + f"svc{i} req{i * 7919 % 100003} user{i * 31} took{i % 977} path{i * 13} ok{i * 17}" + for i in range(self.count) + ) + + +def _pattern_flood_run(source, level: str = "standard", count: int = 5_000) -> tuple[list[str], int]: tracemalloc.start() try: - out = LogReducer(mode="pattern", max_clusters=5, dedup_cache_size=1000, typed_masking=False).reduce( - _SevereFlood(count) - ) + out = LogReducer( + level=level, mode="pattern", max_clusters=5, dedup_cache_size=1000, typed_masking=False + ).reduce(source(count)) return out, tracemalloc.get_traced_memory()[1] finally: tracemalloc.stop() @@ -672,8 +705,17 @@ def _pattern_flood_run(count: int) -> tuple[list[str], int]: def test_severe_flood_of_one_shape_gives_one_line_in_bounded_memory(): """Severe lines are held by the source tap alone, so the pattern miner's store does not grow with them.""" - _, small_peak = _pattern_flood_run(5_000) - out, large_peak = _pattern_flood_run(50_000) + _, small_peak = _pattern_flood_run(_SevereFlood, count=5_000) + out, large_peak = _pattern_flood_run(_SevereFlood, count=50_000) assert out == ["ERROR 0 0 0 0 0 0"] assert large_peak < small_peak * 1.5 + + +@pytest.mark.parametrize("level", ["standard", "enhanced", "maximum"]) +def test_quiet_flood_of_new_templates_runs_in_bounded_memory(level): + """Evicted clusters and the fuzzy index are bounded, so 10x the distinct lines costs no more memory.""" + _, small_peak = _pattern_flood_run(_QuietFlood, level, count=5_000) + _, large_peak = _pattern_flood_run(_QuietFlood, level, count=50_000) + + assert large_peak < small_peak * 1.5 diff --git a/tests/unit/test_examples.py b/tests/unit/test_examples.py new file mode 100644 index 0000000..0a934ce --- /dev/null +++ b/tests/unit/test_examples.py @@ -0,0 +1,18 @@ +"""The deployment recipes in examples/ install on the Python the package requires.""" + +import re +import tomllib +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] + + +def test_example_images_meet_requires_python(): + requires = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8"))["project"]["requires-python"] + floor_match = re.fullmatch(r">=(\d+)\.(\d+)", requires) + assert floor_match, requires + floor = tuple(int(part) for part in floor_match.groups()) + images = re.findall(r"python:(\d+)\.(\d+)", (ROOT / "examples" / "README.md").read_text(encoding="utf-8")) + + assert images + assert all((int(major), int(minor)) >= floor for major, minor in images), images diff --git a/tests/unit/test_patterns.py b/tests/unit/test_patterns.py index f68eb41..b04b09d 100644 --- a/tests/unit/test_patterns.py +++ b/tests/unit/test_patterns.py @@ -2,6 +2,7 @@ Unit tests for pattern extraction module """ +import tracemalloc from unittest.mock import Mock, patch import pytest @@ -9,7 +10,7 @@ from logreducer import LogReducer from logreducer.config import BigDialConfig from logreducer.memory import BoundedDeduplicator -from logreducer.patterns import FuzzyDeduplicator, LogPattern, PatternExtractor, SevereShapes +from logreducer.patterns import FuzzyDeduplicator, LogPattern, PatternExtractor, SevereShapes, is_severe class TestLogPattern: @@ -243,6 +244,108 @@ def test_max_clusters_bounds_template_store(self): extractor.extract_patterns(lines) assert len(extractor.miner.drain.id_to_cluster) <= 5 + def test_fuzzy_index_holds_at_most_max_entries(self): + dedup = FuzzyDeduplicator(max_entries=100) + if not dedup.enabled: + pytest.skip("datasketch not installed") + + kept = list(dedup.deduplicate_stream(_near_unique_lines(1_000))) + + assert len(kept) == 1_000 + assert len(dedup.lsh.keys) == 100 + + def test_near_duplicate_past_the_window_is_admitted_again(self): + dedup = FuzzyDeduplicator(max_entries=2) + if not dedup.enabled: + pytest.skip("datasketch not installed") + first, second, third = _near_unique_lines(3) + + kept = list(dedup.deduplicate_stream([first, first, second, third, first])) + + assert kept == [first, second, third, first] + + @pytest.mark.parametrize("max_entries", [0, -1]) + def test_fuzzy_window_must_hold_one_entry(self, max_entries): + with pytest.raises(ValueError, match="max_entries"): + FuzzyDeduplicator(max_entries=max_entries) + + +def _near_unique_lines(count: int): + """``count`` lines that share no word, so no two are near-duplicates.""" + return (f"svc{i} req{i * 7} user{i * 13} took{i * 17} path{i * 19} ok{i * 23}" for i in range(count)) + + +def _peak_bytes(run) -> int: + tracemalloc.start() + try: + run() + return tracemalloc.get_traced_memory()[1] + finally: + tracemalloc.stop() + + +def _distinct_templates(count: int): + """``count`` quiet lines whose first token differs, so each founds its own Drain cluster.""" + return (f"svc{i} request{i * 7919 % 100003} took {i % 977}ms path{i}" for i in range(count)) + + +class TestPeakMemoryDoesNotGrowWithDistinctLines: + """Ten times the distinct input costs no more memory once the bounds are full.""" + + def test_pattern_store_is_bounded_across_cluster_evictions(self): + def run(count: int) -> None: + PatternExtractor(BigDialConfig(max_clusters=5, min_pattern_occurrences=1)).extract_patterns( + _distinct_templates(count) + ) + + assert _peak_bytes(lambda: run(50_000)) < _peak_bytes(lambda: run(5_000)) * 1.5 + + def test_fuzzy_index_is_bounded(self): + if not FuzzyDeduplicator().enabled: + pytest.skip("datasketch not installed") + + def run(count: int) -> None: + for _ in FuzzyDeduplicator(max_entries=1_000).deduplicate_stream(_near_unique_lines(count)): + pass + + assert _peak_bytes(lambda: run(20_000)) < _peak_bytes(lambda: run(2_000)) * 1.5 + + +def _evicting_feed(rounds: int) -> list[str]: + """Hot lines repeated between bursts of new templates, so their clusters are evicted while they still count.""" + lines = [] + for burst in range(rounds): + lines.append("INFO heartbeat ok") + lines.append(f"INFO tick {burst % 4} done") + lines.extend(f"svc{burst}x{n} started worker{n}" for n in range(6)) + lines.append(f"svc{burst}x0 started worker0") + return lines + + +class TestBoundedRankingKeepsTheSameAnswer: + """The capped result is the prefix of the uncapped one, so bounding the store changes nothing it returns.""" + + @pytest.mark.parametrize("window", [8, 1_000]) + @pytest.mark.parametrize("cap", [1, 2, 5, 40]) + @pytest.mark.parametrize("max_clusters", [1, 3, 25]) + def test_capped_is_prefix_of_uncapped(self, cap, max_clusters, window): + lines = _evicting_feed(60) + + def extract(max_patterns: int) -> list[tuple[str, int]]: + config = BigDialConfig(max_clusters=max_clusters, max_patterns=max_patterns, dedup_cache_size=window) + counted = BoundedDeduplicator(window).deduplicate_counted(lines) + return [(p.template, p.count) for p in PatternExtractor(config).extract_counted_patterns(counted)] + + assert extract(cap) == extract(10**9)[:cap] + + def test_evicted_pattern_that_keeps_counting_still_ranks_first(self): + config = BigDialConfig(max_clusters=3, max_patterns=1) + counted = BoundedDeduplicator(1_000).deduplicate_counted(_evicting_feed(60)) + + patterns = PatternExtractor(config).extract_counted_patterns(counted) + + assert [(p.template, p.count) for p in patterns] == [("INFO heartbeat ok", 60)] + class TestTypedMasking: """Opt-in typed Drain3 masking (config.typed_masking).""" @@ -374,6 +477,31 @@ def test_pattern_counts_include_dropped_repeats(self, lines, expected): def test_reduce_keeps_repeated_lines_and_drops_singletons(self, level, lines, expected): assert sorted(LogReducer(level=level).reduce(lines)) == sorted(expected) + @pytest.mark.parametrize( + ("lines", "expected"), + [ + pytest.param(["INFO disk check ok", "INFO disk check ok"], 2, id="near-duplicate"), + pytest.param(["INFO disk check ok"] + ["INFO disk check ok"] * 4, 5, id="repeats-of-a-near-duplicate"), + ], + ) + def test_near_duplicates_count_towards_the_pattern_they_matched(self, lines, expected): + fuzzy = FuzzyDeduplicator(max_entries=1_000) + if not fuzzy.enabled: + pytest.skip("datasketch not installed") + counted = fuzzy.deduplicate_pairs(BoundedDeduplicator(1_000).deduplicate_counted(lines)) + + patterns = PatternExtractor(BigDialConfig()).extract_counted_patterns(counted) + + assert [(p.template, p.count) for p in patterns] == [("INFO disk check ok", expected)] + + def test_reduce_keeps_a_pattern_seen_only_as_near_duplicates(self): + if not FuzzyDeduplicator().enabled: + pytest.skip("datasketch not installed") + + assert LogReducer(level="enhanced").reduce(["INFO disk check ok", "INFO disk check ok"]) == [ + "INFO disk check ok" + ] + def test_repeat_past_the_dedup_window_counts_as_a_new_sighting(self): extractor = PatternExtractor(BigDialConfig()) lines = [ERROR_LINE, "filler a", "filler b", ERROR_LINE] @@ -443,6 +571,22 @@ def test_default_config_evicts_beyond_the_bound(self): ] +@pytest.mark.parametrize( + ("line", "severe"), + [ + pytest.param("ERROR disk full", True, id="level-word"), + pytest.param("2026-01-01 [error] disk full", True, id="lower-case-in-brackets"), + pytest.param("level=FATAL out of memory", True, id="level-field"), + pytest.param("CRITICAL: replica lost", True, id="level-with-colon"), + pytest.param("INFO error_count=0 route=A1", False, id="field-name"), + pytest.param("INFO no errors in the last hour", False, id="plural"), + pytest.param("INFO TERRORBIRD sighted", False, id="inside-a-word"), + ], +) +def test_is_severe_matches_a_level_word_not_a_substring(line, severe): + assert is_severe(line) is severe + + @pytest.mark.parametrize(("templates", "shapes", "stamps"), SHAPE_CASES) def test_severe_shapes(templates, shapes, stamps): """Stamps, numbers and addresses do not make a shape, identifiers with digits do.""" diff --git a/tests/unit/test_sampling.py b/tests/unit/test_sampling.py index e8a7e10..b94a8d4 100644 --- a/tests/unit/test_sampling.py +++ b/tests/unit/test_sampling.py @@ -16,23 +16,26 @@ class TestBuildSampleSql: - def test_postgresql_seeded_sets_seed_and_predicate(self): + def test_postgresql_seeded_hashes_the_row(self): setup, sql = build_sample_sql("postgresql", "SELECT msg FROM logs", 0.1, seed=42) - assert setup is not None - assert "setseed" in setup - assert "random() <" in sql + assert setup is None + assert "md5(_lr_sample::text || '42')" in sql + assert f"< {int(0.1 * 2**32)}" in sql assert "SELECT msg FROM logs" in sql # user query wrapped, not rewritten - assert "_lr_sample" in sql def test_postgresql_unseeded_has_no_setup(self): setup, sql = build_sample_sql("postgresql", "SELECT msg FROM logs", 0.25) assert setup is None assert "random() <" in sql - def test_mysql_uses_rand(self): - setup, sql = build_sample_sql("mysql", "SELECT msg FROM logs", 0.1, seed=7) + def test_mysql_seeded_hashes_the_named_columns(self): + setup, sql = build_sample_sql("mysql", "SELECT msg FROM logs", 0.1, seed=7, columns=["_lr_sample.`msg`"]) assert setup is None - assert "rand(7)" in sql + assert "MD5(CONCAT_WS(CHAR(31), _lr_sample.`msg`, '7'))" in sql + + def test_mysql_seeded_without_columns_raises(self): + with pytest.raises(ValueError, match="column names"): + build_sample_sql("mysql", "SELECT msg FROM logs", 0.1, seed=7) def test_fraction_one_is_valid(self): # 1.0 is the inclusive upper bound: sample everything. @@ -111,7 +114,7 @@ def test_postgresql_bad_method_raises(self): def test_mysql_predicate_fallback(self): sql = build_table_sample_query("mysql", "`logs`", "`msg`", fraction=0.1, seed=7) - assert "rand(7)" in sql + assert "MD5(CONCAT_WS(CHAR(31), `msg`, '7'))" in sql assert "TABLESAMPLE" not in sql def test_sqlite_predicate_and_seed_raises(self): diff --git a/tests/unit/test_sql_source.py b/tests/unit/test_sql_source.py index 9f6dd1a..18f6de3 100644 --- a/tests/unit/test_sql_source.py +++ b/tests/unit/test_sql_source.py @@ -104,6 +104,23 @@ def test_sqlsource_from_table_seed_on_sqlite_raises(sqlite_url): SQLSource.from_table(sqlite_url, "logs", "line", sample=0.3, sample_seed=1) +@pytest.mark.parametrize(("sample", "error"), [(0.2, None), (1.5, ValueError)]) +def test_seeded_mysql_source_connects_only_when_read(sample, error): + """Construction validates the request but opens no connection, so an unreachable server fails on the read.""" + pytest.importorskip("pymysql") + from sqlalchemy.exc import OperationalError + + unreachable = "mysql+pymysql://user:pw@127.0.0.1:1/logs" # nothing listens on port 1 + if error is not None: + with pytest.raises(error): + SQLSource(unreachable, "SELECT line FROM logs", sample=sample, sample_seed=42) + return + + source = SQLSource(unreachable, "SELECT line FROM logs", sample=sample, sample_seed=42) + with pytest.raises(OperationalError): + list(source) + + def test_reduce_to_target_over_sqlsource(sqlite_url): # 4 distinct patterns in the table; ask for 3 - reached via sampled batches. reducer = LogReducer(level="standard", mode="pattern") diff --git a/tests/unit/test_target.py b/tests/unit/test_target.py index 456763e..8ba490a 100644 --- a/tests/unit/test_target.py +++ b/tests/unit/test_target.py @@ -118,6 +118,27 @@ def test_invalid_loop_params_raise(kwargs, match): reduce_to_target(EmptySource(), _reducer(), target_rows=5, **kwargs) +class WideRowSource(GrowingSource): + """Rows of about 100 KB, recording every batch size asked for.""" + + def __init__(self) -> None: + super().__init__() + self.requested: list[int] = [] + + def sample_batch(self, n: int) -> list[str]: + self.requested.append(n) + return [f"{line} {'x' * 100_000}" for line in super().sample_batch(n)] + + +@pytest.mark.parametrize(("batch_rows", "first"), [(None, 1_000), (123, 123)]) +def test_first_batch_is_a_small_probe_until_row_sizes_are_known(batch_rows, first): + source = WideRowSource() + + reduce_to_target(source, _reducer(), target_rows=10_000, max_fetches=2, batch_rows=batch_rows, plateau_rounds=5) + + assert source.requested[0] == first + + def test_fixed_batch_rows_disables_adaptive_sizing(): source = ManyTemplateSource(n_templates=40) result = reduce_to_target( diff --git a/tests/unit/test_temporal.py b/tests/unit/test_temporal.py index 3aeaef0..9012900 100644 --- a/tests/unit/test_temporal.py +++ b/tests/unit/test_temporal.py @@ -3,7 +3,7 @@ """ import time -from datetime import datetime +from datetime import UTC, datetime import pytest @@ -230,6 +230,25 @@ def test_window_is_independent_of_local_timezone(self, monkeypatch): assert windows[1] - windows[0] == 2 + @pytest.mark.parametrize( + "stamps", + [ + pytest.param(["2026-01-01T01:00:00+01:00", "2026-01-01T00:00:00Z"], id="iso-offset-and-z"), + pytest.param(["2026-01-01 05:30:00.250+0530", "2026-01-01 00:00:00,750"], id="compact-offset-and-naive"), + pytest.param(["01/Jan/2026:01:00:00 +0100", "01/Jan/2026:00:00:00 +0000"], id="apache-offset"), + ], + ) + def test_one_instant_lands_in_one_window_whatever_its_offset(self, stamps): + result = TemporalProcessor(60).process_temporal([f"{stamp} INFO heartbeat ok" for stamp in stamps]) + + midnight_utc_hour = int(datetime(2026, 1, 1, tzinfo=UTC).timestamp() // 3600) + assert [(p["count"], p["window"]) for p in result["temporal_patterns"]] == [(2, midnight_utc_hour)] + + def test_offset_at_the_end_of_time_does_not_crash(self): + result = TemporalProcessor().process_temporal(["9999-12-31T23:59:59-01:00 INFO edge"]) + + assert len(result["temporal_patterns"]) == 1 + @pytest.mark.parametrize("minutes", [0, -5]) def test_non_positive_window_rejected(self, minutes): with pytest.raises(ValueError, match="window_minutes"): @@ -288,3 +307,12 @@ def test_severe_shape_is_one_line_outside_the_cap(self): # One ERROR line whatever the cap, and the cap fills with the rest. assert sum("ERROR" in line for line in reduced) == 1 assert len(reduced) == 1 + 2 + + @pytest.mark.parametrize(("cap", "expected"), [(1, 1), (2, 2), (5, 4)]) + def test_timed_and_timeless_patterns_share_one_cap(self, cap, expected): + timed = [line for line in _timed_lines(3, 1) if "ERROR" not in line] + timeless = [f"worker {n} heartbeat ok" for n in range(50)] + [f"queue depth {n} items" for n in range(50)] + + reduced = LogReducer(mode="temporal", max_patterns=cap).reduce(timed + timeless) + + assert len(reduced) == expected diff --git a/uv.lock b/uv.lock index fe54c93..a5e8a6b 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 5 requires-python = ">=3.14" resolution-markers = [ "python_full_version >= '3.15'", @@ -11,7 +11,7 @@ exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for exclude-newer-span = "P7D" [options.exclude-newer-package] -certifi = "2026-10-21T00:00:00Z" +scalo = "2026-11-01T06:00:00Z" [[package]] name = "annotated-doc" @@ -22,6 +22,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3e/30/e900b21425a860e195f32e37657aa1f7c7f2b1bfb26f03ca209b90933c06/annotated_doc-0.0.5-py3-none-any.whl", hash = "sha256:117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101", size = 5302, upload-time = "2026-07-28T13:50:57.239Z" }, ] +[[package]] +name = "anyio" +version = "4.15.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "typing-extensions", marker = "python_full_version < '3.15'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a9/d2/f4d173e22df740bc37b1db102b386ba719b66e95b0f0d751f556b387e6d2/anyio-4.15.1.tar.gz", hash = "sha256:9f28306018cbd6d329e64a36d58256edff76dd996fe423bc957326e578b82a94", size = 276966, upload-time = "2026-09-05T10:42:39.44Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/12/b8/4bd346e22b28902df4d651910f5242c28d84e4a5c2435ca5c3f797ed7e2e/anyio-4.15.1-py3-none-any.whl", hash = "sha256:6152fdbbf9a77fdec97731721bebf7c4c44f7c29b424b0065826173efc7ed101", size = 132079, upload-time = "2026-09-05T10:42:37.923Z" }, +] + [[package]] name = "ast-serialize" version = "0.12.1" @@ -86,6 +99,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/f4/b54123680025c0b7253117418f023d1b2487f1102552acbdd9d8ee96b622/ast_serialize-0.12.1-cp39-abi3-win_arm64.whl", hash = "sha256:610a41351de68199de9a1434499083b4256c0df7658ec1cfc0a0a7b20b08d317", size = 1136560, upload-time = "2026-10-03T12:24:58.689Z" }, ] +[[package]] +name = "asyncer" +version = "0.0.18" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "sniffio" }, + { name = "typing-extensions", marker = "python_full_version < '3.15'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/95/e3/a9de31596d386b8e76e17c609e20775c6a414cff95d025551aca3c63f282/asyncer-0.0.18.tar.gz", hash = "sha256:dc233a466f85f1bc17dafcdb861b06291b64777a8d9076f7884467cbb66f95ef", size = 24503, upload-time = "2026-06-25T13:41:08.826Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1d/62/73cea402f7c55b830bc02850f84724d9f6e3538c68610f69016f6f480a94/asyncer-0.0.18-py3-none-any.whl", hash = "sha256:054be3c0c3d7f0f79aac5b7d2cf7ad3a740bc465b62a77b48716a30c22679962", size = 12099, upload-time = "2026-06-25T13:41:07.98Z" }, +] + [[package]] name = "bandit" version = "1.9.4" @@ -322,30 +349,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/86/59/0a47b35a47a723ef45c3c51a6469464e5937420fc601d2a9b014865228c6/clickhouse_connect-1.9.0-cp314-cp314t-win_arm64.whl", hash = "sha256:ab9f291563f7cb10c9d26b4d190c918072e610ed8d228dcf48017f746b9e00dc", size = 437485, upload-time = "2026-09-22T15:09:12.108Z" }, ] -[[package]] -name = "clickhouse-driver" -version = "0.2.11" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pytz" }, - { name = "tzlocal" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/da/75/42c6c0f1e0b84213ff096f913ac0ff82037da6f06754bff28685b2d1e23a/clickhouse_driver-0.2.11.tar.gz", hash = "sha256:1bec70343bde9e9a55c2254c5960d34c682ff7d60256589226d96c67a112f95a", size = 434076, upload-time = "2026-07-17T18:31:46.035Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0d/3b/47c4143d003011aa3977b3ab3eafa2ae338cb483387886700c384488226a/clickhouse_driver-0.2.11-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:3e1c3b08907e836de894054d4c66bcf1415cd0a3fe94e3e2c865abe43a635c82", size = 245301, upload-time = "2026-07-17T18:30:46.573Z" }, - { url = "https://files.pythonhosted.org/packages/2d/10/b1ba901d7c71360cafa594130ac43b65f5574e1b03a6fb0c3000c28b68ae/clickhouse_driver-0.2.11-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:eb88ef5ed671e260a6493d7f16f12e21a4f2d04afcfb58c3200324f9621dc553", size = 248506, upload-time = "2026-07-17T18:30:47.866Z" }, - { url = "https://files.pythonhosted.org/packages/6c/cb/68508736b1a537b48a64b053ffbd5b62bd7384ddee97e77efb392164cac7/clickhouse_driver-0.2.11-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:47b72c26343a2b946e589a4d5de259c3705f6969facfd4b283a6c5015fc67c92", size = 1057958, upload-time = "2026-07-17T18:30:49.364Z" }, - { url = "https://files.pythonhosted.org/packages/9e/a8/8e4d7cb2ae58919313c6f4d8641a49c41dc15540572efbd88a46893c0f36/clickhouse_driver-0.2.11-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:fd3b1af7c7174428007b20bbec4c60699758e20cf6beba45c61039762a06116a", size = 1113297, upload-time = "2026-07-17T18:30:50.821Z" }, - { url = "https://files.pythonhosted.org/packages/58/85/ae1be44941e46660aebd70d2ce05729a7edffa42663e31892c4a98910fb7/clickhouse_driver-0.2.11-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5540cfbae18997e625c4fd8ec9875da46c816143c6456cab9a11adfaed38cc34", size = 1119563, upload-time = "2026-07-17T18:30:52.402Z" }, - { url = "https://files.pythonhosted.org/packages/ae/a9/52d086fa76adee9961434b7672706e0403ee09a0c3c49d5c713391d15e36/clickhouse_driver-0.2.11-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d7c9152bfdfd4ebe0fb3b1c4320f39e3b88c8f4f3c6b3db625a5c0cb5b86f955", size = 1049416, upload-time = "2026-07-17T18:30:53.995Z" }, - { url = "https://files.pythonhosted.org/packages/2f/0e/6aafbe06ac73d061554bdbc7dae28b564c0e17619b266e661e235a71b297/clickhouse_driver-0.2.11-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2fa0a4a72618f06c0ee308117253fe351161a562aea8b31641fcbf7d9ad6080d", size = 1022725, upload-time = "2026-07-17T18:30:55.478Z" }, - { url = "https://files.pythonhosted.org/packages/f9/3a/97593f51c0ead217705c46e90f6f23522fd9f252a3f27c84acc38c04113b/clickhouse_driver-0.2.11-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:94593758fa36195fe56199422fe8188cabd6403c3950140cd8cffbd89af34f24", size = 1073586, upload-time = "2026-07-17T18:30:57.641Z" }, - { url = "https://files.pythonhosted.org/packages/c9/d3/9c4a2b7d589e86ea635d85137ed3c5228069cdb780c1919be0f533728b72/clickhouse_driver-0.2.11-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:aa3961a892b94aaa83571e773347d7f36ab5e1334c6711bfd63f611b88c4c541", size = 1070303, upload-time = "2026-07-17T18:30:59.569Z" }, - { url = "https://files.pythonhosted.org/packages/76/58/005fa6ae82131404a1a0053043f7711332c01cf0c0e6863268f0be104e12/clickhouse_driver-0.2.11-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:0e09e4f2cff823027a222c5bd8d9b3b3ca7d70f26ef0dccd2e496820280333ed", size = 1024768, upload-time = "2026-07-17T18:31:01.282Z" }, - { url = "https://files.pythonhosted.org/packages/0b/0e/334868a895896fc83262200f06b8e25c0573f2e5a7299ebbdb1fdbbb3230/clickhouse_driver-0.2.11-cp314-cp314-win32.whl", hash = "sha256:688a2cd31a7fd87a9f2a1bb669a04874b5ae17bbca22a373610e84aa241b1e9a", size = 220173, upload-time = "2026-07-17T18:31:02.759Z" }, - { url = "https://files.pythonhosted.org/packages/7b/51/900211ab9390f64edef10564a206e2dca89fc49a4191aee870d2c18002f6/clickhouse_driver-0.2.11-cp314-cp314-win_amd64.whl", hash = "sha256:0a67ce59def2e08cbda1cf12d9e8a7a6879cee8048ee46bbd415e9111033f775", size = 234021, upload-time = "2026-07-17T18:31:04.129Z" }, -] - [[package]] name = "cloudpickle" version = "3.1.2" @@ -546,6 +549,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", size = 25604, upload-time = "2021-03-08T10:59:24.45Z" }, ] +[[package]] +name = "detect-secrets" +version = "1.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyyaml" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/69/67/382a863fff94eae5a0cf05542179169a1c49a4c8784a9480621e2066ca7d/detect_secrets-1.5.0.tar.gz", hash = "sha256:6bb46dcc553c10df51475641bb30fd69d25645cc12339e46c824c1e0c388898a", size = 97351, upload-time = "2024-05-06T17:46:19.721Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4e/5e/4f5fe4b89fde1dc3ed0eb51bd4ce4c0bca406246673d370ea2ad0c58d747/detect_secrets-1.5.0-py3-none-any.whl", hash = "sha256:e24e7b9b5a35048c313e983f76c4bd09dad89f045ff059e354f9943bf45aa060", size = 120341, upload-time = "2024-05-06T17:46:16.628Z" }, +] + [[package]] name = "docker" version = "7.2.0" @@ -560,6 +576,55 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/75/23/529140fe1aab80fc6992f93a706deec709140a6397439139a054e1515c45/docker-7.2.0-py3-none-any.whl", hash = "sha256:a3f45fdeb9165e2d25d9a1d02ddf3bc70fb572cf5ebbf9b58558c22caf29b71f", size = 148775, upload-time = "2026-07-09T14:53:45.224Z" }, ] +[[package]] +name = "dulwich" +version = "1.2.19" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/98/cf/0af9df98f5d117cd5cef1abedf8d91c7c0681286ea5984b9e792104631f8/dulwich-1.2.19.tar.gz", hash = "sha256:b3cd6b17bfdf7bbae0d04d0f46fc23ce9e175e8871884a1de344f92e2a623a15", size = 1435621, upload-time = "2026-10-22T10:06:55.148Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/b0/46a1aeccf9165bd7ef30a0b21d4b899f63172ff3a4ce38edaa3384609acc/dulwich-1.2.19-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:245d189b752ae73037d83a91e69715739e4c2e0a5882c38a3c089952a9ff3e13", size = 1588910, upload-time = "2026-10-22T10:06:07.304Z" }, + { url = "https://files.pythonhosted.org/packages/6c/aa/6bb4ecc440e8937fb0183400802d0f774b9ac4a29e2d8e034d5854b9caf0/dulwich-1.2.19-cp314-cp314-android_24_x86_64.whl", hash = "sha256:6f6934299c81464b12f475150e5ecee0674fa89915eb94f1866387bf5810b50e", size = 1644322, upload-time = "2026-10-22T10:06:08.775Z" }, + { url = "https://files.pythonhosted.org/packages/d2/cd/57b7eaccd527f17d217e5c5745f614844d4955f626836a1f779ee0105df4/dulwich-1.2.19-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:e093fe41e1843c2175cc9e5ca7bdb2356e5bd2a819b3b5f5eb3c8ed980617443", size = 1440513, upload-time = "2026-10-22T10:06:10.419Z" }, + { url = "https://files.pythonhosted.org/packages/6a/77/fc117f193e71aa07a2e4c03bc67c57579bcd4ec77325657af55fef6d7a57/dulwich-1.2.19-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:5c7ab399bcf7a76a1989a85872ab719e284d2f22b44952a9489c87e214a6ae85", size = 1421945, upload-time = "2026-10-22T10:06:11.894Z" }, + { url = "https://files.pythonhosted.org/packages/e2/68/6f88ae82d27624673152dac20f57eb6b8155f63e5cdd5c5be47ed4ba947b/dulwich-1.2.19-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:25cb4f98ddbe54c9b8d3757d53a1bbee98cd64f4c4b3d0a4dc9d920a46bdb7e7", size = 1556730, upload-time = "2026-10-22T10:06:13.371Z" }, + { url = "https://files.pythonhosted.org/packages/33/a5/d4187fd0491a2a9e2fa66ad7695991fb2d07f3a7f7afb0ac190f9529122f/dulwich-1.2.19-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:61a079c10d46e7e7a4bdfdef77995ee8521a94964ceaa0aba3a7ad3129802e6f", size = 1532490, upload-time = "2026-10-22T10:06:14.943Z" }, + { url = "https://files.pythonhosted.org/packages/71/77/533a170c78ff33885064a7fc29e7d7ad5330fe70f921f7f41793e703bf00/dulwich-1.2.19-cp314-cp314-win32.whl", hash = "sha256:e7fb76abd2a11c596539f52916c0acdaf3211f5ecd34235e79b58fb44fc8a23c", size = 1170582, upload-time = "2026-10-22T10:06:17.143Z" }, + { url = "https://files.pythonhosted.org/packages/59/b9/e9806747eb4169940e6237ff62f863e57397a1349e95465045cd7e2c06be/dulwich-1.2.19-cp314-cp314-win_amd64.whl", hash = "sha256:1145ee36c4668e47cccd3b26f340a2efee5c20ac274915d8c57313aef41f4fce", size = 1127567, upload-time = "2026-10-22T10:06:18.789Z" }, + { url = "https://files.pythonhosted.org/packages/a1/6b/7fffc5adab5b68e2fc1d3da785ce7b5b77b1dad11b3da52900cce275ef07/dulwich-1.2.19-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:c844db4c27661f242b87ca78252e5764216ef9b09fe820c5b91b3148fc0324c1", size = 1501695, upload-time = "2026-10-22T10:06:20.799Z" }, + { url = "https://files.pythonhosted.org/packages/69/81/6d4e84f3db54f6fc59344261fab46c09461c43e21cbb27b67ead5abd741e/dulwich-1.2.19-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c56985f81c7eb1f4ddc7264406db8cf8b1e07b2d071b553080ac64679d28cba6", size = 1419374, upload-time = "2026-10-22T10:06:22.252Z" }, + { url = "https://files.pythonhosted.org/packages/2c/8f/bc0e533687be0f8542eecf5c488df20844311930f513ec62a3264bce09ff/dulwich-1.2.19-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:2a08915edc83ccf63521cd0dfcd1011f1cb3e56f874fc147432865fea2b0ab84", size = 1553462, upload-time = "2026-10-22T10:06:23.725Z" }, + { url = "https://files.pythonhosted.org/packages/65/07/ff75b9b759795e4970d414ebf82a6cc894544d72db41592597da6ef28317/dulwich-1.2.19-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:e97c739826aaeb2dfa1232cedbe8e83107c4a133c7faa8e3d239cc8bbcb526de", size = 1532484, upload-time = "2026-10-22T10:06:25.819Z" }, + { url = "https://files.pythonhosted.org/packages/05/ed/d54f1dbbbbb985e2a4af2b95cfa0d30b881c1b509f4df26463425283e93e/dulwich-1.2.19-cp314-cp314t-win32.whl", hash = "sha256:33ef5ff30b561fa216e1ce44eb0e9d33c989c0f8d63999c11d9b596ebb472eb3", size = 1110753, upload-time = "2026-10-22T10:06:27.237Z" }, + { url = "https://files.pythonhosted.org/packages/24/95/cf8c98699a3d36021cea36a0ad657f2fbf04081d675e1fa29a268119b54a/dulwich-1.2.19-cp314-cp314t-win_amd64.whl", hash = "sha256:17a24dfdd0544e6e63bb70ea9d9eadcc831b24ae70bc85fa978ed4ad7340d59a", size = 1128110, upload-time = "2026-10-22T10:06:28.804Z" }, + { url = "https://files.pythonhosted.org/packages/9c/13/14ab2211b3524fceb646667bf8524ea36419864f9bedbe74be1381ba6c2e/dulwich-1.2.19-cp315-cp315-android_24_arm64_v8a.whl", hash = "sha256:20d592bc9e6b2f2aa0354b6b92589d11196833abc9fd134ba36fb2e7555a8e99", size = 1590451, upload-time = "2026-10-22T10:06:30.399Z" }, + { url = "https://files.pythonhosted.org/packages/68/c4/6ebb71babe0c5045acd4337d7a0622051843a3b37e1336e569789b9a3b3a/dulwich-1.2.19-cp315-cp315-android_24_x86_64.whl", hash = "sha256:63dee71ef10af8aabc7814283b34758298e0abd96010a78937641a9d4b7dd980", size = 1645626, upload-time = "2026-10-22T10:06:32.386Z" }, + { url = "https://files.pythonhosted.org/packages/c3/2e/82713e6cf226f43b42fb3a0e8c0d246eb81bfd0fbe74a90582f33c2eb883/dulwich-1.2.19-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:504ce2688513110864ef1c38119f08198e0c71a369e6769b12d005af84871ea8", size = 1442029, upload-time = "2026-10-22T10:06:34.095Z" }, + { url = "https://files.pythonhosted.org/packages/89/0c/f1fd06c0bd2c8946514e5e44e18d73d66b263e5c6883ae226e833c12341e/dulwich-1.2.19-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:705e84bc212da709f1af97fe81d170f371ef82209ab6049242cd41a18cac50f6", size = 1422820, upload-time = "2026-10-22T10:06:35.58Z" }, + { url = "https://files.pythonhosted.org/packages/f1/33/15d855be105a44079e817a28b861df36ef022e0b00deda98b758cbfa7d85/dulwich-1.2.19-cp315-cp315-manylinux_2_28_aarch64.whl", hash = "sha256:5c29f2affb4983bec7dce075d79f974e1def37ea2c20ad836ee37a18a699bb5f", size = 1557281, upload-time = "2026-10-22T10:06:37.332Z" }, + { url = "https://files.pythonhosted.org/packages/e8/af/ca55ba2b70efbe26b6ab3ff02570adada5554bde584ac1b005ed299a2939/dulwich-1.2.19-cp315-cp315-manylinux_2_28_x86_64.whl", hash = "sha256:b21034e52c1040ea49565f35c481b00852c3a16ace39e5a6c9c5ce172bfd6652", size = 1533280, upload-time = "2026-10-22T10:06:38.797Z" }, + { url = "https://files.pythonhosted.org/packages/d5/68/b97bb42933d60769e9c4078ea5173b78152280d5a384e6a11a39f2471771/dulwich-1.2.19-cp315-cp315-win32.whl", hash = "sha256:4022fa323b5b83ad14d63fcc7cf5e8c2ec2a6d60971915c4414ce13857b5d396", size = 1171035, upload-time = "2026-10-22T10:06:40.613Z" }, + { url = "https://files.pythonhosted.org/packages/ec/23/c84e0d84acd0e677ae2f9940580a2e4ded810a43558bad971fd3cb326737/dulwich-1.2.19-cp315-cp315-win_amd64.whl", hash = "sha256:b103523bbe0454594174069d4672bff86e91d0976ba690a6457f4677c2cadab6", size = 1127786, upload-time = "2026-10-22T10:06:42.095Z" }, + { url = "https://files.pythonhosted.org/packages/92/46/735e8c1b0c4056b914045c030597689514fa9d61f15866a70cd02096e2b2/dulwich-1.2.19-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:04a22388664813857aa72db2892f315a1d018ab864e72775dfac0a90c026f5f1", size = 1502593, upload-time = "2026-10-22T10:06:43.562Z" }, + { url = "https://files.pythonhosted.org/packages/49/18/cc3bd0d5317a18c4014eeaf7cdad4f596dc2c5a091d5a12c042b96a64f0e/dulwich-1.2.19-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:4ac5de1b2e3ec51c2b97ce5892f414095ae48078f0a4169902f7175556abce80", size = 1478815, upload-time = "2026-10-22T10:06:45.258Z" }, + { url = "https://files.pythonhosted.org/packages/e2/fa/daf17f6eba7f532efbd272597b36d71e2e98b59aafcfe679fff94b3a3422/dulwich-1.2.19-cp315-cp315t-manylinux_2_28_aarch64.whl", hash = "sha256:095f7994c3eb069c6fd6b8c7fa305219d431b2b3ead04bbdc10b032b2b628ade", size = 1554900, upload-time = "2026-10-22T10:06:46.794Z" }, + { url = "https://files.pythonhosted.org/packages/2e/1b/db595e1787ef848add6b491ee61a0688a053e6421ee42a93c3eeb611a95a/dulwich-1.2.19-cp315-cp315t-manylinux_2_28_x86_64.whl", hash = "sha256:39e150ea7f96a5600d94be676d065e796dce90826dd315bf8b83a315a202c38a", size = 1533270, upload-time = "2026-10-22T10:06:48.593Z" }, + { url = "https://files.pythonhosted.org/packages/bb/49/441199f5970812a82783c8841d579b6956dd62279e1117237f055bd23368/dulwich-1.2.19-cp315-cp315t-win32.whl", hash = "sha256:b1e763816c16637cc0ccfdb6d6c927e41e4ff40c7d52bf69017360a33b42cffd", size = 1111024, upload-time = "2026-10-22T10:06:50.19Z" }, + { url = "https://files.pythonhosted.org/packages/39/c6/1ba5ff4a5cba18360bc52ee098d7116e8e4fedb8a153b54923a4ec8efae1/dulwich-1.2.19-cp315-cp315t-win_amd64.whl", hash = "sha256:3983613d7027c60085bf9e3b3ff05591fb48ec34db5498e25de4e708dcb37039", size = 1128824, upload-time = "2026-10-22T10:06:51.871Z" }, + { url = "https://files.pythonhosted.org/packages/f3/f1/f8abd5626e413c356a70aef0804893895dbc75de770cec87dba72302c577/dulwich-1.2.19-py3-none-any.whl", hash = "sha256:efe585c9c2a921a26136dc2fba785faafb16d5eed39ea6d40c5dd1cd8140f0a3", size = 755694, upload-time = "2026-10-22T10:06:53.357Z" }, +] + +[[package]] +name = "dynaconf" +version = "3.3.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/71/e4/723ba469856bb493c948985e5bd562c8a65f2b93e70c896e9764ab76de00/dynaconf-3.3.5.tar.gz", hash = "sha256:a08f6ab44025034ef3c9f86b32548ab01efd4039094a74bd9f028a43c63d016f", size = 333426, upload-time = "2026-08-05T19:50:25.16Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/95/41/b38501c34c40c3261868a2af8ed5fffe3e6d889c680638708de27eb57788/dynaconf-3.3.5-py3-none-any.whl", hash = "sha256:b3b4a753d5c866fc85466b65fc58d4954f6ed7dc8aa67f2b15bcbfd85d184687", size = 271426, upload-time = "2026-08-05T19:50:23.802Z" }, +] + [[package]] name = "execnet" version = "2.1.2" @@ -731,8 +796,9 @@ dev = [ { name = "pytest-xdist" }, { name = "python-dotenv" }, { name = "ruff" }, + { name = "scalo" }, { name = "sqlalchemy" }, - { name = "testcontainers", extra = ["clickhouse", "mysql"] }, + { name = "testcontainers", extra = ["mysql"] }, { name = "ty" }, { name = "vermin" }, { name = "vulture" }, @@ -768,14 +834,28 @@ dev = [ { name = "pytest-xdist", specifier = ">=3.8.0" }, { name = "python-dotenv", specifier = ">=1.0.0" }, { name = "ruff", specifier = ">=0.16.10" }, + { name = "scalo", specifier = ">=2.32.4" }, { name = "sqlalchemy", specifier = ">=2.1.3" }, - { name = "testcontainers", extras = ["clickhouse", "kafka", "postgres", "mysql"], specifier = ">=4.15.0" }, + { name = "testcontainers", extras = ["postgres", "mysql"], specifier = ">=4.15.0" }, { name = "ty", specifier = ">=0.0.34" }, { name = "vermin", specifier = ">=1.6.0" }, { name = "vulture", specifier = ">=2.16" }, { name = "xxhash", specifier = ">=4.0.2" }, ] +[[package]] +name = "loguru" +version = "0.7.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "win32-setctime", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3a/05/a1dae3dffd1116099471c643b8924f5aa6524411dc6c63fdae648c4f1aca/loguru-0.7.3.tar.gz", hash = "sha256:19480589e77d47b8d85b2c827ad95d49bf31b0dcde16593892eb51dd18706eb6", size = 63559, upload-time = "2024-12-06T11:20:56.608Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl", hash = "sha256:31a33c10c8e1e10422bfd431aeb5d351c7cf7fa671e3c4df004162264b28220c", size = 61595, upload-time = "2024-12-06T11:20:54.538Z" }, +] + [[package]] name = "lz4" version = "4.4.5" @@ -813,6 +893,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, ] +[[package]] +name = "mergedeep" +version = "1.3.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3a/41/580bb4006e3ed0361b8151a01d324fb03f420815446c7def45d02f74c270/mergedeep-1.3.4.tar.gz", hash = "sha256:0096d52e9dad9939c3d975a774666af186eda617e6ca84df4c94dec30004f2a8", size = 4661, upload-time = "2021-02-05T18:55:30.623Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/19/04f9b178c2d8a15b076c8b5140708fa6ffc5601fb6f1e975537072df5b2a/mergedeep-1.3.4-py3-none-any.whl", hash = "sha256:70775750742b25c0d8f36c55aed03d24c3384d17c951b3175d898bd778ef0307", size = 6354, upload-time = "2021-02-05T18:55:29.583Z" }, +] + [[package]] name = "msgpack" version = "1.2.3" @@ -1005,6 +1094,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/d9/7fb5aa316bc299258e68c73ba3bddbc499654a07f151cba08f6153988714/pathspec-1.1.1-py3-none-any.whl", hash = "sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189", size = 57328, upload-time = "2026-04-27T01:46:07.06Z" }, ] +[[package]] +name = "phonenumbers" +version = "9.0.41" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2f/df/cc0d70f1c79e436ea00d935b6352053d526252b81ce6c130d39eee846fb2/phonenumbers-9.0.41.tar.gz", hash = "sha256:dfa6f74eeac67c044b75313fe0af10774d7d1e1242241437279d4c2fb8027c01", size = 2311017, upload-time = "2026-10-08T10:51:09.778Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/60/1a/4059026e9c8a4c3faeab4a45f5aa67fb77d1f0d9c767085ded69c5c533e2/phonenumbers-9.0.41-py2.py3-none-any.whl", hash = "sha256:ccf2ea44f8aa35c487f26146a31520ecedf8e1af1f57c803678ecb5ef5c01668", size = 2594658, upload-time = "2026-10-08T10:51:07.317Z" }, +] + [[package]] name = "pip" version = "26.2.1" @@ -1252,12 +1350,12 @@ wheels = [ ] [[package]] -name = "pytz" -version = "2026.5" +name = "python-stdnum" +version = "2.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/14/21/d83d6ef28c4c912c4bb4d1dcf591f7b8c6bde87b9c66f9f454677314e16d/pytz-2026.5.tar.gz", hash = "sha256:fa23724b9c486543b9ff54a327ee7569ac83ade54bb9afd0fc18676620401c86", size = 318572, upload-time = "2026-10-04T02:37:58.719Z" } +sdist = { url = "https://files.pythonhosted.org/packages/15/7f/96c2b9de6024353177dc6139c33730d5ac25877bc33215515d6b95b84555/python_stdnum-2.2.tar.gz", hash = "sha256:e95fcfa858a703d4a40130cb3eaac133c60d8808a7f3c98efeedac968c2479b9", size = 1311813, upload-time = "2026-01-04T19:36:16.753Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/4f/ef/c66110d46fb800dda0bf33164182dfadabe26a90e4476844d502a23dca8e/pytz-2026.5-py2.py3-none-any.whl", hash = "sha256:e658af3757f9e26a9d25dd2aff38335acd92bc9104f890a894b2c1ba28311b03", size = 506342, upload-time = "2026-10-04T02:37:56.814Z" }, + { url = "https://files.pythonhosted.org/packages/2f/61/aa32d9c79f83a2fae033cd6496fb2a24aba918d31c73704271dfcfb48375/python_stdnum-2.2-py3-none-any.whl", hash = "sha256:bdf98fd117a0ca152e4047aa8ad254bae63853d4e915ddd4e0effb33ba0e9260", size = 1193213, upload-time = "2026-01-04T19:36:14.812Z" }, ] [[package]] @@ -1299,6 +1397,78 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, ] +[[package]] +name = "regex" +version = "2026.9.29" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fc/f2/af1da9d3ceed77bfcdce40427d49ba0be94e4fe84245e3bfef68c10e75b6/regex-2026.9.29.tar.gz", hash = "sha256:8b5fcc4771732191b2b7d1dd68d8f0353f47f8d90b6150f6dce58bf1112442cb", size = 419199, upload-time = "2026-09-29T00:49:58.298Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/93/1f/d9dc6f02f569625faf67a4daec926cd5023472dcd69bb44286dccd5a5ab3/regex-2026.9.29-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:957bb708e8057ab1649ba566456429d691ec9b90d1c9ad1af1ba7ffbbeaf05f2", size = 497598, upload-time = "2026-09-29T00:47:36.541Z" }, + { url = "https://files.pythonhosted.org/packages/9c/83/9b693a3fd1451381e812031a8961ec5b3b8f0c8cc6871f14c5223642804d/regex-2026.9.29-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:c9b602fae1e00b7c035d661ce85575365719192a7b46784bd71cf64c68053aa0", size = 296250, upload-time = "2026-09-29T00:47:38.233Z" }, + { url = "https://files.pythonhosted.org/packages/dd/5f/52bc2abc3fef040cd9de76ab29c918d6a717a454ae2b9dd7938b0c95656d/regex-2026.9.29-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0166844493626c5015c6088ee15c9ca2fd060ca15b7641d1657da6a58432ae33", size = 293518, upload-time = "2026-09-29T00:47:39.957Z" }, + { url = "https://files.pythonhosted.org/packages/dc/fc/cf50671215ee0057046980b4571ef8646a005819bb67f0957e779ed107a5/regex-2026.9.29-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b97a38fb4c732b6832db6bf108963adbcd82ef1268ba2025dce390f45af75efa", size = 806037, upload-time = "2026-09-29T00:47:41.676Z" }, + { url = "https://files.pythonhosted.org/packages/14/4b/dddef8fc15c63e4347cc9efb138d0cd306f30e6c98acbcc81a8f780083b9/regex-2026.9.29-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a540abfab208e1b7ef2df231c40ef3b6cbb30a0aad6204e9b6a81c10a6794628", size = 878881, upload-time = "2026-09-29T00:47:43.755Z" }, + { url = "https://files.pythonhosted.org/packages/9f/cb/38daabed32d28f7e58a06e9344ce00dc67952e9996bc578ed6a29fe1240e/regex-2026.9.29-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ddfa987262763c3c22a8367d2a49c244b018a74c3a8e3ab1a864119ad45c5633", size = 918684, upload-time = "2026-09-29T00:47:45.594Z" }, + { url = "https://files.pythonhosted.org/packages/a9/4d/041d9458a645fee4fce4d642a89d27271a3cfcd91095104f6dde44da70bf/regex-2026.9.29-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2f7f7aa47b229f2b39a2ae2596d2ad5625d77b5eb9856fac2dab3eb506cdd0a0", size = 807176, upload-time = "2026-09-29T00:47:47.372Z" }, + { url = "https://files.pythonhosted.org/packages/bf/c4/4383eed7aa5aef67616cb1b3f3ad06b7c624c4e6cced48630cd5ce133d85/regex-2026.9.29-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d9b77b25b4f395f92de6099ab08e8ae2bc7e51dfe157f22900902243a5cc90c7", size = 784315, upload-time = "2026-09-29T00:47:49.518Z" }, + { url = "https://files.pythonhosted.org/packages/5c/a6/0086ad31cebb183c637d3198547075aa493afde308e1ff61fccccb29ba6e/regex-2026.9.29-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:34b6925af9853bf461950e6508910f179fd6e9b1a7ec8548e069606b7e51a26b", size = 793748, upload-time = "2026-09-29T00:47:51.279Z" }, + { url = "https://files.pythonhosted.org/packages/d5/a0/f9005cba3f629a859573fc5d1224ea4e1f97919ec8581d018e03a351a604/regex-2026.9.29-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:addd736a0547d553283adaf4e05d7104e7f2c7b0b092e9b4d28756825f14531f", size = 870302, upload-time = "2026-09-29T00:47:53.368Z" }, + { url = "https://files.pythonhosted.org/packages/01/4f/e1a3e46bb5315a4e18b01a990e7a28e2a16595609d50c442baf2815a3c65/regex-2026.9.29-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fe3fa1dd453ed5c7f5ea23a26218329790ed7197a99b90e94330e313959a7f52", size = 770299, upload-time = "2026-09-29T00:47:55.606Z" }, + { url = "https://files.pythonhosted.org/packages/2c/fe/f303b4acfda44e1ff1379368748c1ef2dad04a6a8e9c0ecbc970b19d97ca/regex-2026.9.29-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0cc63b5e47c12a48d90c7e9d7de6a035dd14f62868aaedbb4e0ff8ba2b8bfe7b", size = 861570, upload-time = "2026-09-29T00:47:57.617Z" }, + { url = "https://files.pythonhosted.org/packages/60/b6/b4f7e99249f596017c60ccad5faf9310fc8e3e59bb2244940a90a1b0bdff/regex-2026.9.29-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:724184b4aafed865e4f13ca313fdcb43024300c028ec67319cfa16847d84685e", size = 795967, upload-time = "2026-09-29T00:47:59.922Z" }, + { url = "https://files.pythonhosted.org/packages/fb/d3/fc865a4638d9f6762192b6bab5b7aa1f33a90e9e99578c2e111e2a63c8c3/regex-2026.9.29-cp314-cp314-win32.whl", hash = "sha256:c6c8fabf1dafc1f1ddcbb67896d3f93efb092e8c4b6322d7389b944e76a484e5", size = 274758, upload-time = "2026-09-29T00:48:01.8Z" }, + { url = "https://files.pythonhosted.org/packages/31/e2/c2b466924ccbeb874862968ca638051b15a8fd29d994a0e99004a5cbf78e/regex-2026.9.29-cp314-cp314-win_amd64.whl", hash = "sha256:1c2a0026062abcc321a53db4a185ceba0b59a66b5d37b0808917a88b55a5257f", size = 283817, upload-time = "2026-09-29T00:48:03.614Z" }, + { url = "https://files.pythonhosted.org/packages/c6/42/ea0f8dbaa924fa75c6338935eaee2f44dab369b27f02db1e03d74344b049/regex-2026.9.29-cp314-cp314-win_arm64.whl", hash = "sha256:121a76a0985db80ceae9e171c337f8c927868e37d01b54e3ce87bc87f9c6a208", size = 283663, upload-time = "2026-09-29T00:48:05.624Z" }, + { url = "https://files.pythonhosted.org/packages/44/48/d58e5081119f5c223bbb37d2340acde3d069e1df8e8cd166c37502eee4da/regex-2026.9.29-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:e31f72490b7c12f7790e1e25c3afffd20503ee1bfb43461d7838b871ff244b19", size = 501393, upload-time = "2026-09-29T00:48:07.833Z" }, + { url = "https://files.pythonhosted.org/packages/72/3c/c49945287d4f9efee7d41f98072f8ad880efb8f430595a612fbdea996a4e/regex-2026.9.29-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:80ea96f5c1a30bf09007d48466521d9c294bebe197c708c3359096e3e3691632", size = 298237, upload-time = "2026-09-29T00:48:09.684Z" }, + { url = "https://files.pythonhosted.org/packages/f9/1f/688cb61c3d4cf7bcc1ed444b5cc49399eba3e51c469ae285cf87fea3022e/regex-2026.9.29-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:554bffadcbcb6d5f4e5fb10a61cc52084b9a63d1dab5f10bcd2c4343972e8e2c", size = 295936, upload-time = "2026-09-29T00:48:11.454Z" }, + { url = "https://files.pythonhosted.org/packages/26/a3/de43ac6b877b7d09c19a3a426b1bd5acdd209eaaf68f406466f80439ccf6/regex-2026.9.29-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:864e9b87ac33c3fb9fb4ad48166d4fdb579c351d5c77deb0d34bccb36a775cd9", size = 816905, upload-time = "2026-09-29T00:48:13.321Z" }, + { url = "https://files.pythonhosted.org/packages/62/14/9940763201c51d537786304984c67d0fc3d2ed18837ffb6f09a869f6b6c9/regex-2026.9.29-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:044265d77d94f5e3cb2fd72c76723807c429cb8c533e9d4672d0334a6f14f588", size = 881527, upload-time = "2026-09-29T00:48:15.313Z" }, + { url = "https://files.pythonhosted.org/packages/d3/e1/c842d8df0b23245ebf202f8ab9c39fd48e2db39959454ec39a41c8c72082/regex-2026.9.29-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2089fe39c406784d90101c726755ffa1497bb74638fd434300d2b88006186de8", size = 923115, upload-time = "2026-09-29T00:48:17.328Z" }, + { url = "https://files.pythonhosted.org/packages/d8/c1/98622479e3c354a446a75232e522d747d2b3df23092dcd8a5309380a2020/regex-2026.9.29-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0def9fb6abac55492d6d51cddb7225d07d6f279e774e0adc08569a54a5fc8d46", size = 820674, upload-time = "2026-09-29T00:48:19.32Z" }, + { url = "https://files.pythonhosted.org/packages/6c/d0/5808c95f9c79ed27b5eedaafc3df6239ec56a49f2e23ea8f831b18427c82/regex-2026.9.29-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:888d60953908dcf761aa320c3e390ab8556efbdb551ace63921de90f6ae0848d", size = 793306, upload-time = "2026-09-29T00:48:21.615Z" }, + { url = "https://files.pythonhosted.org/packages/bf/d3/021ca2638671ad20603bcd9b4d5bfa35d2610cd216a043ea7f0b44ea39f6/regex-2026.9.29-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ed511a0708e2297e1d6431e7fb217e3402791e491e02da800658ace4973df1bb", size = 803103, upload-time = "2026-09-29T00:48:23.871Z" }, + { url = "https://files.pythonhosted.org/packages/6b/2d/755c6d13ef9c657378013676c391c7a402166b3f419a464a3e058dcbe533/regex-2026.9.29-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e1172147d28d8fbcf8cb8d26c41506169f5ad8fe9ec969cb116835a19d4d8eca", size = 872175, upload-time = "2026-09-29T00:48:26.255Z" }, + { url = "https://files.pythonhosted.org/packages/6c/fc/e1cab183b9dafe8597f58c1c766da9bf96204d3b2f232bcf3eeb75ff7b6c/regex-2026.9.29-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:92f05c9c42bde5785dc48770bc2194d9f7442544156f951e19cd31b096cec562", size = 777362, upload-time = "2026-09-29T00:48:28.389Z" }, + { url = "https://files.pythonhosted.org/packages/06/7c/e10ea17fba31fb4a1f9d13ed53a2d2a9066a2aea58d7557e263f6d99e7b0/regex-2026.9.29-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f37964e4a5e993d2fd45147741e9dff7f34a2d8c00ab94c4ea0514a4677f959e", size = 865606, upload-time = "2026-09-29T00:48:30.4Z" }, + { url = "https://files.pythonhosted.org/packages/8e/6e/69824d9aee1fd41c54ea7264654a47c8d9d84d8a228e11c2bcf4c201ed81/regex-2026.9.29-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:951733b1bbdb71e377cec567b409f1a7881b47cfcad84121aa74cb575fa425ea", size = 807945, upload-time = "2026-09-29T00:48:32.375Z" }, + { url = "https://files.pythonhosted.org/packages/89/22/857050a86e21ce60193e02a8ef662521f2e263a645c8b1b905fc136b61a7/regex-2026.9.29-cp314-cp314t-win32.whl", hash = "sha256:65b408d8fcb273e3499e7ef2ce796810da1becd208c7fb4373692a242d79d461", size = 276758, upload-time = "2026-09-29T00:48:34.72Z" }, + { url = "https://files.pythonhosted.org/packages/4d/96/56808fe029553d7d4c703414f2a527faad2ea2bfa9ca094a2e7f8762b530/regex-2026.9.29-cp314-cp314t-win_amd64.whl", hash = "sha256:bf48516e35cf848390ea68850aba53e7c333720d2945b4d2c25b69fc5171723f", size = 286527, upload-time = "2026-09-29T00:48:36.864Z" }, + { url = "https://files.pythonhosted.org/packages/01/aa/074e2cfb3d8101a6a764aba5f7c5d1e21de087483e35bdc0c4ce2eb60364/regex-2026.9.29-cp314-cp314t-win_arm64.whl", hash = "sha256:9173db3be74a35cb6731701094b98120f7ee4876a287882a59cdea1fa7da342f", size = 285954, upload-time = "2026-09-29T00:48:38.901Z" }, + { url = "https://files.pythonhosted.org/packages/a7/dc/d84990386c9dfdf8c377f00f371b241fdc9a2c8aea0e3d66941b2e51be0b/regex-2026.9.29-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:c3589f40749acce747510bf5d589d54e376cb0930ea58b35effac97e5312b0c1", size = 497836, upload-time = "2026-09-29T00:48:40.858Z" }, + { url = "https://files.pythonhosted.org/packages/c2/ab/a569ebde875fa12ff8c6c9a30e07503620f195e4be4d54c3d3ee8eecc283/regex-2026.9.29-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:32ab11df9677ca80bcbb5fe4eb1da9109a5019239a054836efc6fa1c64e683cf", size = 296244, upload-time = "2026-09-29T00:48:42.952Z" }, + { url = "https://files.pythonhosted.org/packages/f3/3e/7d548e82a108e7c8b2d5246650e397a2f8db599f9b2e975466939c5b4e70/regex-2026.9.29-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:7c03031610e3e6ed1768a2b7a8fc84637c1257b50c5eacaf094c6e17a84fc563", size = 293748, upload-time = "2026-09-29T00:48:44.985Z" }, + { url = "https://files.pythonhosted.org/packages/40/34/a8e19a52f452bbb07b32a2bef70dcdf90c2737049749f74cc12d7486fb4f/regex-2026.9.29-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:42e82e578c904445d4c8a35b8f28052cf567593215fa5db06266fbc6f77aaa2e", size = 807840, upload-time = "2026-09-29T00:48:46.948Z" }, + { url = "https://files.pythonhosted.org/packages/88/7b/11fbd4640b3bb82b72822a63c20ade4013d562d291703a9debeedc24e682/regex-2026.9.29-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0b65c72739f981377c9c22e0c5c3cd7f42da7bd8a3c9209330fac772c7d893ed", size = 879330, upload-time = "2026-09-29T00:48:49.168Z" }, + { url = "https://files.pythonhosted.org/packages/f3/55/de58c74f1f4e31586d83eb39c56872d686c4e0d0966d151884c833b94ced/regex-2026.9.29-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4408b2b27a95ca8cc48b7411945753773353b5c93b307754781086c99d3a576f", size = 919251, upload-time = "2026-09-29T00:48:51.322Z" }, + { url = "https://files.pythonhosted.org/packages/81/42/a8c480f6dd5ac59fa28ddae79afd9d7ac7e596fdb61813adc65bb6e674b8/regex-2026.9.29-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a714befaacbd10092ffe4cea0d3c5f008fb9efe9bc322c715bcdfdee414b9a3d", size = 808808, upload-time = "2026-09-29T00:48:53.529Z" }, + { url = "https://files.pythonhosted.org/packages/68/60/0bc0d1ec8b37ad64be6fa30e035251f11de9667a0fac9e82ee74517d81be/regex-2026.9.29-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33026515aebc0e70d1c89978e53e8d695d35d9e472f8d5b34465ba3c74028650", size = 789907, upload-time = "2026-09-29T00:48:56.036Z" }, + { url = "https://files.pythonhosted.org/packages/da/84/116a3ef19b3acfe81077f0bf2cbc7714a5e94bc8935b7243ab61cb0f1c3c/regex-2026.9.29-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:31b003f9a070335e2a8233ee9b14a3ca8e6d792012ae011f741bf0aaf11744c5", size = 795770, upload-time = "2026-09-29T00:48:58.284Z" }, + { url = "https://files.pythonhosted.org/packages/96/ba/e38c3f203e7e7e18c957d48e6cb6dbf96c11e95a44efa4a480522afc5d6d/regex-2026.9.29-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:c03c6eb6ece86dfdcbb34799efaa339b093132e1aceed491ba5e08fe06cdf699", size = 870671, upload-time = "2026-09-29T00:49:00.506Z" }, + { url = "https://files.pythonhosted.org/packages/2f/0f/9ee0b0cb76c55f63684bd7fff554978e8773b4fc86e2bcb2d50772dc1086/regex-2026.9.29-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:a5300757f8a68f5b6cc33f57338d72a0e3589c5cc9ad5f8504ea06f028be582a", size = 778631, upload-time = "2026-09-29T00:49:02.984Z" }, + { url = "https://files.pythonhosted.org/packages/b6/19/e6e3eeb226af5872c4958002f6edef4e4f40ea4cc5f5665023f2019eb045/regex-2026.9.29-cp315-cp315-musllinux_1_2_s390x.whl", hash = "sha256:80c7cadd3fd2bfde5df8aa0787e315812cad0c313a753095d02f4c2b6c01677b", size = 862187, upload-time = "2026-09-29T00:49:05.264Z" }, + { url = "https://files.pythonhosted.org/packages/5b/62/823c102e106bb2711d6b7dfe5981552fe4467b2969c46a20c5c383cf498c/regex-2026.9.29-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:3f1e6cb402a89457582cd696f982559217d13484a193202c394015297968c86d", size = 798423, upload-time = "2026-09-29T00:49:07.644Z" }, + { url = "https://files.pythonhosted.org/packages/37/e0/e927776258fa70b2f6feffc3be584ffc85ba4c1e20a320f0aee9a632fc7d/regex-2026.9.29-cp315-cp315-win32.whl", hash = "sha256:a64b85a4760337cfefdb27d42da6ed8b58e8cde3f2d57b6ef43e76ef6ea9ef47", size = 274757, upload-time = "2026-09-29T00:49:10.513Z" }, + { url = "https://files.pythonhosted.org/packages/77/04/358de85d1860238e1b4fa98fc2c80c990124a25d2e14739e28cc02c25562/regex-2026.9.29-cp315-cp315-win_amd64.whl", hash = "sha256:b3e445b66c80b4eb4234e855ce94d9adc183eedbd632816228d89930b91b2c5b", size = 283824, upload-time = "2026-09-29T00:49:12.849Z" }, + { url = "https://files.pythonhosted.org/packages/92/d3/d5c5b264784a5ab2b0f8cf620c1eeb4dbf3440d306761905e7d99345bef5/regex-2026.9.29-cp315-cp315-win_arm64.whl", hash = "sha256:8f39588af4731c8923c26810eb3b33f76f17633985e40f59c3cd45a33805a895", size = 283664, upload-time = "2026-09-29T00:49:15.331Z" }, + { url = "https://files.pythonhosted.org/packages/02/dc/f63ec2c201445ce1150fe780f5c56f16a10124d9a9da3a93161dbb0d8892/regex-2026.9.29-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:fb99cc9d45f48895d9d67f6a0b8a57f08d39c174d9f25ad97a313e0470267b1c", size = 501549, upload-time = "2026-09-29T00:49:17.705Z" }, + { url = "https://files.pythonhosted.org/packages/9a/5c/d2a698dc6bfc11fbce03f1cb0249c13284e93b79ed11f893edf6fac431c9/regex-2026.9.29-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:720537c7ea6f80dc61913184edb0ce2497a306b39ef19f28505b322553d52bdb", size = 298187, upload-time = "2026-09-29T00:49:20.171Z" }, + { url = "https://files.pythonhosted.org/packages/85/b7/88dcdb38cd3935d4ee9e9ce9b8e56cb3b3518d1f020acfa7dd62ad289bf8/regex-2026.9.29-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:0fd2c901cc307a745ad4bc87f20060d7a0825a3371d1e93488af22e7a387f78f", size = 296157, upload-time = "2026-09-29T00:49:22.342Z" }, + { url = "https://files.pythonhosted.org/packages/d3/8e/ba6c01dde33a69fc294b38b43f6677baaa5735a6248f39708031a738158a/regex-2026.9.29-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b11b589e00095ec69cf79841a76360f9b079e95b0368a25b5ebb951ab0c157ff", size = 818468, upload-time = "2026-09-29T00:49:24.612Z" }, + { url = "https://files.pythonhosted.org/packages/2a/f1/2586693e3a2d6b1247852593d37a6c17b42a92ee44f7cdcb9a0c1494e64a/regex-2026.9.29-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d7cab119d0df0b9413f106b4d7fc34f2872d3574ed3806fb48959c830b1537da", size = 882825, upload-time = "2026-09-29T00:49:26.996Z" }, + { url = "https://files.pythonhosted.org/packages/30/51/084f3e7bdcd0e9c33665c938cf5d134dc3548cbb4a75f0197ec7bfd754b1/regex-2026.9.29-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b89efc38431793d28b7cd91227e2f952ad7c48df19132b17f43a5fec3c14143b", size = 923314, upload-time = "2026-09-29T00:49:29.822Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f1/066c6fc23b7dc229789c21c880b5ba5ad689fb95fed12e078266f55a1f9b/regex-2026.9.29-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:80a5ea3b4fd9d6a5b9a44f7976a9acaaab35aa3c1f6b29e5bd857dfabaded223", size = 822222, upload-time = "2026-09-29T00:49:32.404Z" }, + { url = "https://files.pythonhosted.org/packages/0a/56/592cd46fdb8f2f8682a1d7fd1310e4d0bcb93fbd0e6bbe4141ac28240227/regex-2026.9.29-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:19959129885356df0e97556856f77eb2888380dac18bed075a7c05c5128c618d", size = 794882, upload-time = "2026-09-29T00:49:35.076Z" }, + { url = "https://files.pythonhosted.org/packages/ee/4d/d65384bb071c864b01aa8314e3a6a687845ebd57588390976edc960c218b/regex-2026.9.29-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:6a1a824fbed817e0a891103886b68f063b1e83cc51bc97192a90a60195a9291f", size = 805887, upload-time = "2026-09-29T00:49:37.395Z" }, + { url = "https://files.pythonhosted.org/packages/65/b6/358de0d8f40d5178e4f7e7e121cfd5b961c812b77a055d11f5079e3f8fd7/regex-2026.9.29-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:1ba8c6a416569ce0d37e83e28a254a61dc99a419084dfb6476cea02d997f74fa", size = 872901, upload-time = "2026-09-29T00:49:39.927Z" }, + { url = "https://files.pythonhosted.org/packages/00/06/6bfded72d043240c6b52bbb5e16f639d81affbf7484b4fe2ec45f3d4afc9/regex-2026.9.29-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:446654b29bfaa30500d80947eda42cef1449dc8a87f4e3cf061cc8485d3a1f0b", size = 782970, upload-time = "2026-09-29T00:49:42.581Z" }, + { url = "https://files.pythonhosted.org/packages/5a/20/9f418a50baa78b3ed8308fcb0cc49e472dd000b7ef935a7295af202ea744/regex-2026.9.29-cp315-cp315t-musllinux_1_2_s390x.whl", hash = "sha256:bf3c49863c23a1ad6da9c30351aed6cff8d5ddbeb63c5c8420ae54e98c7d0138", size = 865441, upload-time = "2026-09-29T00:49:45.238Z" }, + { url = "https://files.pythonhosted.org/packages/2c/29/817c7eacdeaf8463123e949bd394c39ad024eea1ec38ddf5ad141da2f3bd/regex-2026.9.29-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:01000ddf0e3ffef97f2413ceb514f6313040106b6d18a03ee00a4fe35c1eb1db", size = 809431, upload-time = "2026-09-29T00:49:47.878Z" }, + { url = "https://files.pythonhosted.org/packages/63/0b/83aab3b5b739947f744135a7a3a446e25433ebc92b05e01aae197ccbfdda/regex-2026.9.29-cp315-cp315t-win32.whl", hash = "sha256:c4e38dd8f39c43a91d2410ad2b85610701b0979342c3df1d69eaf8e838c757d8", size = 276964, upload-time = "2026-09-29T00:49:50.524Z" }, + { url = "https://files.pythonhosted.org/packages/72/f2/6314b5fc68789b5dcc38885bc6e3d6986b34fb3372b7231088ee5cecaa05/regex-2026.9.29-cp315-cp315t-win_amd64.whl", hash = "sha256:e2c89e9b762c57f59d5e99ee8b20202adb892e35f8d3485741340999ca55058e", size = 286487, upload-time = "2026-09-29T00:49:53.224Z" }, + { url = "https://files.pythonhosted.org/packages/56/bc/97b2245c8c7b2dd01f2db74f2bea003cd33c15009b4996a2447f46b5325c/regex-2026.9.29-cp315-cp315t-win_arm64.whl", hash = "sha256:e8c65ef3862a8ad6e86492b6ed9327805dd66904c012bd3649dc67d822ed6c34", size = 285955, upload-time = "2026-09-29T00:49:55.655Z" }, +] + [[package]] name = "requests" version = "2.34.2" @@ -1352,6 +1522,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fc/05/c16957eb287c3fc062e032619a25868d93d408a844b725bcf514f7a378ff/ruff-0.16.10-py3-none-win_arm64.whl", hash = "sha256:25a65fe998c4e6861ec079ada5826a2fc605e6cbccbe9dcd7fac1f54e791621b", size = 10366440, upload-time = "2026-10-01T18:03:19.04Z" }, ] +[[package]] +name = "scalo" +version = "2.32.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "asyncer" }, + { name = "detect-secrets" }, + { name = "dulwich" }, + { name = "dynaconf" }, + { name = "loguru" }, + { name = "mergedeep" }, + { name = "phonenumbers" }, + { name = "python-dotenv" }, + { name = "python-stdnum" }, + { name = "pyyaml" }, + { name = "regex" }, + { name = "tomli-w" }, + { name = "typer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/40/c3/0fd22195a16851d11368a090fe5a110c7e81fef2bc608b14df0eb2b9ba26/scalo-2.32.4.tar.gz", hash = "sha256:d5cc1c9dd5cd14c19dca996de6d564223669b067b80f2ecefb0de7f3e810c830", size = 548113, upload-time = "2026-11-01T05:14:03.674Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8f/bd/311ad62baf5b09b616500434d89bbe7fc35a33eb0017a3ecd4f99a09d2cd/scalo-2.32.4-py3-none-any.whl", hash = "sha256:bc58c89f6814c629717f2fb9e22be650c38c94e3da656ae3522938e628e6019f", size = 640615, upload-time = "2026-11-01T05:14:00.012Z" }, +] + [[package]] name = "scikit-learn" version = "1.9.1" @@ -1451,6 +1646,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, ] +[[package]] +name = "sniffio" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372, upload-time = "2024-02-25T23:20:04.057Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, +] + [[package]] name = "sortedcontainers" version = "2.4.0" @@ -1538,9 +1742,6 @@ wheels = [ ] [package.optional-dependencies] -clickhouse = [ - { name = "clickhouse-driver" }, -] mysql = [ { name = "pymysql", extra = ["rsa"] }, { name = "sqlalchemy" }, @@ -1649,18 +1850,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/94/21/1e5995a1c920cce14e4bffae20c665ec10e7ed03ab25e006cd741092b718/tzdata-2026.5-py2.py3-none-any.whl", hash = "sha256:b683bd1b6659ddcd810ff02ad09ba821d4bf1065072805063eb35c49617905ac", size = 347996, upload-time = "2026-10-03T09:23:12.535Z" }, ] -[[package]] -name = "tzlocal" -version = "5.4.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "tzdata", marker = "sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/81/5b/879b2f932adfa7a053c360d50bc896c977fa6426109185f7c12ebdd0cb9d/tzlocal-5.4.4.tar.gz", hash = "sha256:8dbb8660838688a7b6ba4fed31d18dedf842afb4d47ca050d6d891c2c15f3be4", size = 31170, upload-time = "2026-06-29T08:03:40.026Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9e/a4/017a7a6cbe387d961a688ec31364ae60a5c4e22c96ae9921b79a947c855d/tzlocal-5.4.4-py3-none-any.whl", hash = "sha256:aae09f0126a8a86fa736be266eb4a471380d26a0de3bc14844e7821fee3e2a15", size = 18115, upload-time = "2026-06-29T08:03:38.666Z" }, -] - [[package]] name = "urllib3" version = "2.8.0" @@ -1688,6 +1877,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f5/be/f935130312330614811dae2ea9df3f395f6d63889eb6c2e68c14507152ee/vulture-2.16-py3-none-any.whl", hash = "sha256:6e0f1c312cef1c87856957e5c2ca9608834a7c794c2180477f30bf0e4cc58eee", size = 26993, upload-time = "2026-03-25T14:41:26.21Z" }, ] +[[package]] +name = "win32-setctime" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b3/8f/705086c9d734d3b663af0e9bb3d4de6578d08f46b1b101c2442fd9aecaa2/win32_setctime-1.2.0.tar.gz", hash = "sha256:ae1fdf948f5640aae05c511ade119313fb6a30d7eabe25fef9764dca5873c4c0", size = 4867, upload-time = "2024-12-07T15:28:28.314Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/07/c6fe3ad3e685340704d314d765b7912993bcb8dc198f0e7a89382d37974b/win32_setctime-1.2.0-py3-none-any.whl", hash = "sha256:95d644c4e708aba81dc3704a116d8cbc974d70b3bdb8be1d150e36be6e9d1390", size = 4083, upload-time = "2024-12-07T15:28:26.465Z" }, +] + [[package]] name = "wrapt" version = "2.5.0"