From 617b8973f5f66eb65d1a2e07471cb730ae410286 Mon Sep 17 00:00:00 2001 From: Boris De Vloed Date: Sun, 6 Sep 2026 18:57:11 +0200 Subject: [PATCH] feat(fr): eAmbrosia register as a second cahier source + keyed CARTO basemap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two independent pieces of work that were sitting together in the working tree. Both are described in full in CLAUDE.md. 1. eAmbrosia register as a fallback cahier source (FR) BO Agri is the FR pipeline's primary cahier source and its long tail is expensive: hand-curated URLs, a Légifrance cookie fetcher, OCR over professional-org mirrors. The eAmbrosia EU GI register serves the *same* INAO cahier PDF per GI as an attachment, with none of that — so it is wired as a self-service **last tier**, not a replacement. BO Agri and the curator overrides still resolve first, so every currently-canonical resolution stays canonical (today the tier fires for 0 of 466). - 01d_resolve_register.py supplies the join key FR lacks (INAO-sourced records carry no id_eambrosia): SIQO name → register protectedName. A wrong bind attaches another appellation's cahier, which is worse than a gap, so _lib/fr/register_match.py refuses rather than guesses — product-type partition (Calvados is both a Wine PGI and a Spirit-drink PGI), no fuzzy matching (exact normalised alias parts only), and a one-to-one check. 466/466 parents resolved, 0 unresolved: 444 on the full name, 20 on an alias, 2 pinned in register_overrides.json. - RegisterTier in 01_scrape_cahiers.py runs only after BO Agri AND the curator overrides miss. Attachments cache by sha256 OUTSIDE raw/inao/cahiers/ so stage 02's global segment index cannot silently rewrite unrelated records; only a winning cahier is promoted. Provenance is written only for entries the register won, so an appellation still served by BO Agri keeps a byte-identical manifest. - audit_fr_register_shadow.py compares register vs in-build cahier without writing into the build. v1 sweep over all 466 parents: the register is systematically the OLDER vintage (newer text on the BO Agri side in 199 of 215 substantive differences), which settles the open question — it must stay a last tier and must not be promoted. The determinism control came back 356/356 byte-identical, so the differences are real, not extractor noise. Three records where the register is materially richer are queued in CURATOR_TODO. - Endpoint gotchas live in _lib/eambrosia_register.py: the internal id is not derivable from giIdentifier, the detail path has no /v1/, the attachment path does, and an explicit Accept: application/pdf trips the anti-bot gate. 2. Keyed CARTO basemap + staged OpenFreeMap successor CARTO retired key-less raster access in 2026-08: an unkeyed tile still returns HTTP 200 but with "API KEY REQUIRED" stamped into the PNG — a failure invisible to any status-code check. Tile URLs now carry ?key=, injected at build time from CARTO_BASEMAP_KEY in the repo-root .env via the new _lib/env.py (which also takes over _load_dotenv from _lib/batch). Unset ⇒ no key and a watermarked basemap, so stage 04 warns loudly. CARTO + OpenStreetMap attribution stays visible, as the free tier requires. The key-free vector successor is vendored at _lib/vendor/openfreemap-{positron,dark}.json for a later switch. Tests: tests/test_fr_register_{match,tier}.py (30 tests) cover the matcher's three guards and the tier's ordering + cache isolation. Co-Authored-By: Claude Opus 5 (1M context) --- CLAUDE.md | 195 +- CURATOR_TODO.md | 144 +- docs/fr-eambrosia-pipeline-prompt.md | 155 + docs/reconciliation-log.md | 61 + scripts/01_scrape_cahiers.py | 266 +- scripts/01d_resolve_register.py | 125 + scripts/02_extract_cahiers.py | 48 +- scripts/03_generate_wiki.py | 8 +- scripts/04_build_maps.py | 12 +- scripts/_lib/assets/app.js | 26 +- scripts/_lib/batch.py | 14 +- scripts/_lib/eambrosia_register.py | 69 + scripts/_lib/env.py | 56 + scripts/_lib/fr/__init__.py | 0 scripts/_lib/fr/naming.py | 47 + scripts/_lib/fr/register_cahier.py | 207 + scripts/_lib/fr/register_match.py | 334 ++ scripts/_lib/fr/register_overrides.json | 15 + scripts/_lib/map_template.py | 3 + scripts/_lib/vendor/README.md | 37 + scripts/_lib/vendor/openfreemap-dark.json | 2692 +++++++++++++ scripts/_lib/vendor/openfreemap-positron.json | 3344 +++++++++++++++++ scripts/audit_fr_register_shadow.py | 489 +++ scripts/deploy.sh | 5 + tests/test_fr_register_match.py | 226 ++ tests/test_fr_register_tier.py | 233 ++ 26 files changed, 8729 insertions(+), 82 deletions(-) create mode 100644 docs/fr-eambrosia-pipeline-prompt.md create mode 100644 scripts/01d_resolve_register.py create mode 100644 scripts/_lib/env.py create mode 100644 scripts/_lib/fr/__init__.py create mode 100644 scripts/_lib/fr/naming.py create mode 100644 scripts/_lib/fr/register_cahier.py create mode 100644 scripts/_lib/fr/register_match.py create mode 100644 scripts/_lib/fr/register_overrides.json create mode 100644 scripts/_lib/vendor/openfreemap-dark.json create mode 100644 scripts/_lib/vendor/openfreemap-positron.json create mode 100644 scripts/audit_fr_register_shadow.py create mode 100644 tests/test_fr_register_match.py create mode 100644 tests/test_fr_register_tier.py diff --git a/CLAUDE.md b/CLAUDE.md index 5577ea6..65f8729 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -220,6 +220,171 @@ override URL. Stage 02's cross-bundle rescue then matches the cahier header by name across the corpus, so overrides automatically promote matching stubs to full extracts. Re-run stages 01 → 04 after edits. +## eAmbrosia register — second source for the FR cahier + +BO Agri is the FR pipeline's primary cahier source, and its long tail is +expensive: hand-curated URLs in `manual_overrides.json`, a Légifrance +cookie-injection fetcher ([scripts/01b_solve_legifrance.py](scripts/01b_solve_legifrance.py)), +and OCR over professional-org mirrors for the 2011 Burgundy cohort. The +**eAmbrosia EU GI register** serves the *same INAO cahier PDF* per GI as an +attachment — `productSpecifications[0]`, whose filenames are literally +`CDC_Batard-Montrachet.pdf` / `CDC IGP Haute-Marne.pdf` — with none of that. +It is wired as a **self-service fallback tier**, not a replacement: BO Agri and +the curator overrides still resolve first, so every currently-canonical +resolution stays canonical. + +The register attachment is the same document already in the build, not a +different vintage: for Bâtard-Montrachet (`PDO-FR-A0571`) and IGP Puy-de-Dôme +(`PGI-FR-A1211`) the extracted `lien_au_terroir` is byte-identical to the +in-build record. See the shadow report below for the corpus-wide picture. + +### The join key — stage 01d + +France is the one country sourced from INAO rather than eAmbrosia, so its +records carry **no `id_eambrosia`**. [scripts/01d_resolve_register.py](scripts/01d_resolve_register.py) +supplies the missing link by matching the SIQO appellation name against the +register's `protectedName`, and writes `raw/inao/register/resolved.json` +(id_appellation → GI binding) plus `raw/inao/register/unresolved.json` (the +curator queue, one `reason` per entry). + +A wrong bind attaches another appellation's cahier, which is worse than a gap, +so [scripts/_lib/fr/register_match.py](scripts/_lib/fr/register_match.py) +refuses rather than guesses. Three guards: + +- **Product-type partition.** The candidate pool is split by the register's + `qualityProductType` (Wine / Spirit drink / Food), keyed off the SIQO + `categorie` via `CATEGORIE_PRODUCT_TYPE`. *Calvados* is both a Wine PGI (the + Normandy IGP wine) and a Spirit-drink PGI (the eau-de-vie); without the + partition either could claim the other's cahier. A SIQO row carrying no + `categorie` searches every partition instead and must come back with a + single GI. +- **No fuzzy matching.** Keys are the exact normalised alias parts of both + names — the shared `candidate_keys()` in + [scripts/_lib/fr/naming.py](scripts/_lib/fr/naming.py) (splitting SIQO's + `" ou "` / `" et "` / commas) plus a `/` split on the register side, which + is how the register writes synonym lists (`Bourg / Côtes de Bourg / + Bourgeais`). Exact matching alone would miss ~20 appellations; fuzzy + matching would risk a wrong bind, so the alias fold does the work. A key + hitting more than one GI resolves to nothing. +- **One-to-one.** Two appellations claiming the same file number means at + least one bind is wrong, so both go back to the queue. + +The residue is pinned in +[scripts/_lib/fr/register_overrides.json](scripts/_lib/fr/register_overrides.json) +(checked in — curator input a fresh checkout needs), keyed by +`id_appellation` → `{file_number, register_name, note}`. An empty +`file_number` records a verified absence and drops the appellation from the +queue. v1 needs two pins: *Calvados Domfontais* (SIQO spelling; the register +and the cahier both say *Domfrontais*) and *Marc d'Alsace Gewurztraminer* +(SIQO leaves `categorie` empty on the manifest row). + +Result: **466 / 466 parents resolved, 0 unresolved** — 444 on the full name, +20 on an alias, 2 pinned. Sub-denominations are not resolved: they have no +register entry of their own and inherit the parent cahier exactly as they do +today. + +### The fallback tier — stage 01 + +`RegisterTier` in [scripts/01_scrape_cahiers.py](scripts/01_scrape_cahiers.py) +runs only after BO Agri **and** the curator overrides have failed to produce a +PDF — all three miss paths in `_process_app` funnel through `_register_tier`. +Today that is 0 of 466 appellations, so a re-run is a no-op. Disable with +`--no-register`; `--register-delay` sets the inter-request pause. + +Register attachments are cached by sha256 under `raw/inao/register/cahiers/`, +**deliberately outside `raw/inao/cahiers/`**: stage 02's +`build_global_segment_index` indexes every PDF in that directory and feeds each +record's `source.latest_known_pdf`, so dumping a few hundred register cahiers +there would silently rewrite unrelated records. Only a cahier that actually +wins a resolution is promoted into the cahiers directory. + +Provenance is written **only for entries the register won**, so an appellation +still served by BO Agri keeps a byte-identical manifest entry: `source_kind: +"eambrosia-register"`, `register_file_number`, `register_attachment_uri`, +`register_attachment_name`, `register_protected_name`, `register_matched_via`. +Stage 02 passes the same keys into `record["source"]` under the same guard. +`fetched_at` comes from the register cache manifest, not from the clock, so a +cache hit does not churn the manifest. + +Endpoint mechanics live in +[scripts/_lib/eambrosia_register.py](scripts/_lib/eambrosia_register.py) (see +the cross-country endpoint section for the gotchas: the internal `id` is not +derivable from `giIdentifier`, the detail path has no `/v1/`, the attachment +path does, and an explicit `Accept: application/pdf` trips the anti-bot gate). +`load_gi_rows()` caches the register's GI listing — `protectedName`, country, +product type, status — to `raw/eambrosia-register/gi-index.json` from the same +single ~4 MB POST that backs the id map. + +### Shadow report + +[scripts/audit_fr_register_shadow.py](scripts/audit_fr_register_shadow.py) +compares the register cahier against the in-build record **without writing +into the build**: it extracts both with the unchanged stage-02 parser in +memory and reports three columns per appellation — `build` (the record on +disk), `self` (re-extracting the build's own PDF now, as a determinism +control) and `register`. `lien_au_terroir` is the headline comparison, with +section / commune counts and the homologation date alongside so a *vintage* +difference is distinguishable from a *parse* difference. Output: +`raw/inao/register/shadow-report.{json,md}`. + + .venv/bin/python scripts/01d_resolve_register.py + .venv/bin/python scripts/audit_fr_register_shadow.py + +v1 sweep (2026-08-29, all 466 parents): + +| step | count | share | +|---|---:|---:| +| resolved to a register file number | 464 | 99.6 % | +| register serves a cahier attachment | 361 | 77.5 % | +| register cahier extracts | 352 | 75.5 % | +| `lien_au_terroir` byte-identical to the build | 92 | 26 % of extracted | +| differs cosmetically (word-overlap ≥ 0.99) | 45 | | +| differs substantively (< 0.99) | 215 | | + +The determinism control came back **356 / 356 byte-identical**, so every +difference below is a real difference between the two documents, not +extractor noise. + +**The register is systematically the older vintage.** Of the 215 +substantively-different records, the newer text is on the **BO Agri** side in +**199**, the register side in 4, the same year in 7, undated in 5 (max year +mentioned in each `lien`). The differences are real regulatory ones, not +typography: BO Agri's Anjou Villages cahier reads "43 communes du +Maine-et-Loire et 3 communes des Deux-Sèvres" where the register's reads "24" +and "2"; Anjou and Arbois carry 2022–2023 republications the register has not +picked up. That settles the open question in the brief — **the register must +stay a last tier and must not be promoted to a primary source.** + +Three records where the register is nonetheless materially *richer* than the +build are queued in [CURATOR_TODO.md](CURATOR_TODO.md) as targeted re-source +candidates: Collioure (315 → 14,412 chars), Pouilly-Loché (255 → 8,261), +Franche-Comté (4,062 → 7,583). The first two are among the only six parents +whose build `lien` is under 1,000 chars — the register closes the tail of the +PNOCDC short-lien backlog. + +Two things the shadow report exists to surface, both live findings: + +- **Vintage.** The register attachment is not always the newest publication. + Bâtard-Montrachet and IGP Puy-de-Dôme are byte-identical to the build, but + Chablis's BO Agri PDF is a 2025 republication (with typo corrections — + *au-dessus*, *Ces vallons*) that the register has not picked up. That is the + argument for keeping the register a *last* tier rather than promoting it. +- **A pre-existing extractor defect.** For some GIs the register serves the + whole *Journal officiel* issue (`joe_20110630_0060.pdf` for Pouilly-Fumé) + instead of a standalone `CDC_*.pdf`. `extract_aire`'s department-header + regex backtracks pathologically on that layout — minutes on a 2 KB section. + The audit bounds it (`--extract-timeout`, default 20 s → verdict + `register-extract-timeout`); the regex itself is tracked in + [CURATOR_TODO.md](CURATOR_TODO.md) and must be fixed, with its own + corpus-wide diff, before the register can be promoted. + +Licence / attribution: the register is the European Commission's own +publication of the member state's product specification; the served PDF is the +INAO cahier des charges. Public-source rule satisfied. The map panel and the +wiki label a register-sourced cahier as such — `boagri_url` stays empty and +the attachment URL rides its own `register_attachment_url` field, so nothing +attributes an EU-register document to BO Agri. + ## Cadastre lieux-dits (sub-commune climat geometry) Cadastre Etalab (`cadastre.data.gouv.fr`, Licence Ouverte 2.0) publishes @@ -393,8 +558,10 @@ twice with no changes upstream must be a no-op (cache hits). | Script | Reads | Writes | |---|---|---| | 00_fetch_data.py | (network) | raw/inao/siqo-referentiel.csv, raw/ign/communes.geojson, raw/inao/parcellaire/*.shp, raw/cadastre/lieux-dits/*.json.gz | -| 01_scrape_cahiers.py | raw/inao/siqo-referentiel.csv | raw/inao/cahiers/*.pdf, raw/inao/cahiers/manifest.json | +| 01_scrape_cahiers.py | raw/inao/siqo-referentiel.csv + raw/inao/cahiers/manual_overrides.json + raw/inao/register/resolved.json | raw/inao/cahiers/*.pdf, raw/inao/cahiers/manifest.json, raw/inao/register/{cahiers/*.pdf,manifest.json} | +| 01d_resolve_register.py | raw/inao/cahiers/manifest.json (or raw/inao/siqo-referentiel.csv) + raw/eambrosia-register/gi-index.json + scripts/_lib/fr/register_overrides.json | raw/inao/register/{resolved,unresolved}.json | | 02_extract_cahiers.py | raw/inao/cahiers/*.pdf | raw/inao/cahier-extracted/*.json + _index.json | +| audit_fr_register_shadow.py | raw/inao/cahiers/manifest.json + raw/inao/cahier-extracted/ + raw/inao/register/resolved.json | raw/inao/register/{shadow-report.json,shadow-report.md,cahiers/*.pdf,manifest.json} | | 02b_fetch_grape_lexicon.py | raw/inao/cahier-extracted/*.json + raw/vivc/by-slug/ | raw/wikipedia/grapes//*.json + manifest.json | | 02b_fetch_aoc_lexicon.py | raw/inao/cahier-extracted/*.json | raw/wikipedia/aocs/fr/*.json + manifest.json | | 02b_fetch_style_lexicon.py | raw/wikipedia/style_overrides.json | raw/wikipedia/styles//*.json + manifest.json | @@ -4472,6 +4639,32 @@ runs it. Edit app.js directly; do not move the JS back into the template. The CSS still ships inline in `_TEMPLATE` and is lifted to the shared `style..css` by `_split_template`. +### Basemap (CARTO raster, keyed) and its successor + +The basemap under the polygons is CARTO raster — Voyager (light) + dark_all +(dark), both added up front and switched by layer *visibility*, never by +`setStyle` (which would reorder layers above the appellation polygons and drop +their selection feature-state). + +In **2026-08 CARTO retired key-less access**: an unkeyed tile still returns +HTTP 200, but with "API KEY REQUIRED" stamped into the PNG — a failure that is +invisible to any status-code check and only shows up by looking at the image. +The tile URLs now carry `?key=`, injected at build time from +**`CARTO_BASEMAP_KEY`** in the repo-root `.env` (the same file +[scripts/deploy.sh](scripts/deploy.sh) sources) via +[scripts/_lib/env.py](scripts/_lib/env.py) → the `__OWM_carto_key_json__` +token. The key is public once the map ships (the browser makes the tile +request); `.env` keeps it rotatable and out of git history. Unset ⇒ no `?key=` +and a watermarked basemap, so stage 04 warns loudly on stderr. Free tier is 5M +tile requests/month, **conditional on keeping the CARTO + OpenStreetMap +attribution visible** — it is set on both raster sources in `app.js`; do not +drop it. + +CARTO is retiring raster altogether in favour of vector. The staged successor +is vendored at `scripts/_lib/vendor/openfreemap-{positron,dark}.json` — key-free +OpenStreetMap vector tiles that merge into one style and keep the same +visibility toggle. See [scripts/_lib/vendor/README.md](scripts/_lib/vendor/README.md). + ## Structured data (JSON-LD) on entity pages Each **indexable** per-appellation entity page (`//`) carries a diff --git a/CURATOR_TODO.md b/CURATOR_TODO.md index 45521c2..b4b3ee0 100644 --- a/CURATOR_TODO.md +++ b/CURATOR_TODO.md @@ -4,7 +4,7 @@ Actionable manual lookups across the corpus. One section per country. Reconcile Legend: ✅ done · 🟡 URL queued, awaiting pipeline rerun · 🟢 in progress · ⏳ blocked on code · ❌ open -Last reconciled: 2026-08-26 — full pass history in [docs/reconciliation-log.md](docs/reconciliation-log.md). +Last reconciled: 2026-08-29 — full pass history in [docs/reconciliation-log.md](docs/reconciliation-log.md). --- @@ -221,6 +221,41 @@ Reconciled 2026-08-26 against `raw/terroir-facts/`: --- +### `extract_aire` regex backtracks pathologically on JORF-issue layouts — ❌ open + +Found while running the register shadow sweep (2026-08-29). Not a register +bug and not new — a **pre-existing latent defect in the FR extractor** that no +document in the current build happens to trigger. + +`_DEPT_HEADER_PATTERN` in [scripts/02_extract_cahiers.py](scripts/02_extract_cahiers.py) +contains `(?P(?:[^:\n]*\n?){0,4}?):` — a nested quantifier whose inner +branch can match the empty string. On an area section that carries a +`Département du Cher` heading followed by commune lines with **no colon**, the +engine explores exponentially many ways to split those lines before failing. + +Reproducer (register attachment `joe_20110630_0060.pdf` for `PDO-FR-A0824` +Pouilly-Fumé — the register serves the whole *Journal officiel* issue for this +GI rather than a standalone `CDC_*.pdf`): + +``` +.venv/bin/python scripts/audit_fr_register_shadow.py --only "Pouilly-Fumé" --extract-timeout 0 +# hangs; `extract_aire` on a 2,044-char section runs for minutes +``` + +Impact today: none — the affected document only reaches the extractor through +the shadow audit, which now bounds it (`--extract-timeout`, default 20 s, +verdict `register-extract-timeout`). Impact if the register tier ever wins for +such an appellation: **stage 02 hangs the whole build**, and the same shape +could arrive from BO Agri. + +Fix deliberately NOT bundled with the register change: `extract_aire` feeds +the commune list of all 1,540 FR records, so rewriting the pattern needs its +own verification pass (full stage-02 re-run + `fr-cahier-extracted` diff +against the baseline), not a drive-by edit. Candidate rewrite: +`(?P[^:\n]*(?:\n[^:\n]*){0,4}?)` — same language, but every +repetition consumes at least the newline, so the empty-loop ambiguity goes +away. Confirm match-by-match equivalence over the corpus before shipping. + ### Interprofession / syndicat URLs — 🟡 1537 / 1540 (2026-08-26) Was 1244/1540. The 296 gaps were two distinct populations: @@ -271,6 +306,103 @@ Side-findings for later (not URL work): `cote-roannaise` and and therefore land as `is_wine=false` on the map, despite both being wine AOCs. +### eAmbrosia register as a second cahier source — ✅ shipped (2026-08-29) + +The FR pipeline now has a self-service fallback behind BO Agri: the eAmbrosia +EU GI register serves the same INAO cahier PDF per GI +(`productSpecifications[0]`). When BO Agri surfaces **no PDF at all** for an +appellation, that gap can now be closed without a hand-curated BO Agri URL, a +Légifrance cookie, or OCR over a professional-org mirror. (An appellation +whose INAO product page links the *wrong* arrêté still needs curation — that +PDF downloads fine, so stage 01 never reaches the register tier. Re-sourcing +on a stage-02 stub is a separate change.) This is the **first landing of the +cross-country Phase-2 register retrofit** (see that section below) — +plumbing, not a data fix: FR was already at 1,540 records / 0 stubs, and the +tier runs strictly *after* BO Agri, so it wins nothing today. + +The tier is additionally gated on the appellation having **no cahier PDF on +disk**: `resolve_cahier` and `_download_first_pdf` both return None for a 5xx, +a timeout or an empty Drupal body exactly as they do for a genuinely absent +cahier, so without that gate one INAO outage would re-source working +appellations and churn every downstream surface with no upstream change behind +it. + +- `scripts/01d_resolve_register.py` writes the name → `fileNumber` map + (`raw/inao/register/resolved.json`) + curator queue + (`raw/inao/register/unresolved.json`). **466 / 466 parents resolved, queue + empty** (444 full-name, 20 alias, 2 pinned). +- `scripts/audit_fr_register_shadow.py` is the read-only comparison against the + in-build record (`raw/inao/register/shadow-report.md`). +- Pins live in `scripts/_lib/fr/register_overrides.json` (checked in). + +Curator actions when the queue is non-empty: look the name up at +`https://ec.europa.eu/geographical-indications-register/`, add +`{"": {"file_number": "PDO-FR-…", "register_name": "…", +"note": "…"}}`, re-run `01d`. An empty `file_number` records a verified +absence. The resolver deliberately refuses to fuzzy-match, so every residue is +a curator decision. + +Shadow-report findings (`raw/inao/register/shadow-report.md`, 466 parents): + +- **464 / 466 resolve**, 361 (77.5 %) get a cahier attachment, 352 extract. +- **92 byte-identical** to the build, 45 cosmetically different, 215 + substantively different — and in **199 of those 215 the newer text is BO + Agri's**. The register is systematically behind (Anjou Villages: BO Agri + "43 communes … 3 communes" vs register "24 … 2"; Anjou and Arbois carry + 2022–2023 republications). **Do not promote the register to a primary + source.** Determinism control: 356/356 re-extractions byte-identical, so + these are real document differences. +- 🟡 **3 records where the register is materially richer than the build** — a + targeted re-source is a genuine quality win. Collioure and Pouilly-Loché are + two of only six parents whose build `lien` is under 1 000 chars (the other + four are eaux-de-vie, which carry no lien section by design), so this closes + the tail of the PNOCDC short-lien backlog above. Needs a stub-driven + re-source path — stage 01 only reaches the register when there is *no* PDF, + and these have one: + + | id | appellation | build lien | register lien | + |---:|---|---:|---:| + | 254 | Collioure | 315 | 14 412 | + | 217 | Pouilly-Loché | 255 | 8 261 | + | 959 | Franche-Comté | 4 062 | 7 583 | + +- ❌ **103 appellations (22 %) have no register cahier attachment** — a bare + `Ares(…)` reference in `productSpecifications`. The brief's 15 % estimate + came from an n=80 sample; the true figure is higher. Not actionable (BO + Agri covers them all today). +- ❌ **6 register documents time out in `extract_aire`** and 3 fail to + extract — see the regex defect logged above. + +Open queue — ❌ **2 appellations INAO still publishes but the EU register has +struck off.** The resolver refuses to bind a `Cancelled` GI on its own (a +withdrawn registration cannot be an appellation's current specification), so +both sit in `raw/inao/register/unresolved.json` awaiting a curator call. Both +already have a working BO Agri cahier, so nothing is missing from the corpus — +what needs deciding is whether SIQO is stale (the mirror image of the +2026-08-26 "eAmbrosia has them, INAO doesn't" retirement above): + +| id | appellation | file number | register state | +|---:|---|---|---| +| 877 | Cité de Carcassonne (IGP) | `PGI-FR-A1203` | `Cancelled`; registered 17/03/1978; no successor row in the register | +| 881 | Coteaux de Narbonne (IGP) | `PGI-FR-A1202` | `Cancelled`; registered 09/12/1983; no successor row in the register | + +Verify against the OJ-L cancellation regulation, then either pin the file +number in `register_overrides.json` (if the attachment is still the right +cahier) or retire the appellation the way the two SIQO ghosts were. + +Known register-side negatives (not actionable — BO Agri already covers them): + +| appellation | file number | gap | +|---|---|---| +| Musigny | `PDO-FR-A0582` | single-document only; `productSpecifications` is a bare `Ares(2013)3755890` reference | + +Two pins were needed: + +| id | appellation | file number | why | +|---:|---|---|---| +| 335 | Calvados Domfontais | `PGI-FR-01837` | SIQO spelling; register + cahier both say *Domfrontais* | +| 1091 | Marc d'Alsace Gewurztraminer | `PGI-FR-01836` | SIQO carries no `categorie`, so the product-type partition cannot be picked (same root cause as the `cote-roannaise` / `muscat-du-cap-corse` `is_wine` side-finding above; those two resolve on the all-partition fallback) | + ## Spain ### Pliego URLs — ✅ complete (2026-05-10) @@ -2556,6 +2688,16 @@ Applied **where there was an actual terroir gap** — not as a rip-and-replace: ### Phase 2 — unify source-fetch on the register API (elegance retrofit, NON-URGENT) +🟢 **First landing: France (2026-08-29).** FR now carries the register as a +last-resort cahier tier behind BO Agri, with a name → `fileNumber` resolver +(FR is the one INAO-sourced country, so it has no `id_eambrosia` join key) and +a read-only shadow report. See the France section above. What it proves for +the rest of the retrofit: the register's `productSpecifications` attachment is +the **full national spec**, not the thinner single document, and it parses with +the country's existing extractor unchanged. What it does NOT yet do: retire BO +Agri, `01b_solve_legifrance.py` or the OCR mirror path — those come out only +once the shadow report proves the register covers them, as a separate change. + Forward-looking cleanup, not user-visible: make the register API the **canonical first-fetch** for the source document, so the codebase is more uniform and sheds fragile dependencies. The register single-document is diff --git a/docs/fr-eambrosia-pipeline-prompt.md b/docs/fr-eambrosia-pipeline-prompt.md new file mode 100644 index 0000000..cbe5271 --- /dev/null +++ b/docs/fr-eambrosia-pipeline-prompt.md @@ -0,0 +1,155 @@ +# Task brief — source the French cahiers des charges from the eAmbrosia EU GI register + +## Objective + +Give the French pipeline a second, self-service source for the *cahier des +charges*: the eAmbrosia EU GI register attachment endpoint. Today FR fetches +exclusively from BO Agri, with a long tail that needs hand-curated URLs, a +Légifrance cookie-injection fetcher, and OCR over professional-org mirrors. +The register serves the **same INAO cahier PDF** with none of that. + +This is **plumbing, not a data fix**. FR coverage is already complete +(1,540 records, 0 stubs). The win is retiring fragile fetch paths and +making future FR gaps self-service. Treat regression risk on the 1,540 +working records as the primary cost. + +## Established facts — verified 2026-08-29, do not re-litigate + +Spike run against the live register. Reproduce only if something looks wrong. + +| finding | value | +|---|---| +| FR wine GIs in the register (`countryId=fr`, `qualityProductType=Wine`) | **458** | +| carry the full national cahier (`productSpecifications[0].uri`) | **85 %** (n=80 random sample: 68 cahier / 9 single-doc-only / 3 neither) | +| attachment filenames | literally `CDC_Batard-Montrachet.pdf`, `CDC IGP Haute-Marne.pdf` — the INAO cahier, not the thinner EU single document | +| parses with the existing FR extractor | **yes**, unchanged `scripts/02_extract_cahiers.py` | +| AOC round-trip (Bâtard-Montrachet, `PDO-FR-A0571`) | 12 roman sections; `lien` text **byte-identical** to the in-build record (8,424 chars, sha `9bdd4182…`) | +| IGP round-trip (Puy-de-Dôme, `PGI-FR-A1211`) | 8 sections; `lien` **byte-identical** to in-build (5,714 chars, sha `7a3e6a6f…`) | +| the 9 AOCs that today need CAVB/FGVB mirrors + Type-1C OCR | **7 / 9** have a register cahier | + +The byte-identical result is the load-bearing fact: for these records the +register is the *same document already in the build*, so a source swap is +content-neutral, not a re-extraction from a different vintage. + +Known negatives from the same spike: +- **Musigny** (`PDO-FR-A0582`) — single-doc only; `productSpecifications` is + a bare `Ares(2013)3755890` reference and `fetch_attachment` returns False. +- **IGP Lorraine** (`PGI-FR-02815`) — neither attachment. +- **Côte de Nuits-Villages** and **Muscat de Frontignan** — no register entry + under their exact SIQO name. Name matching, not absence: see below. + +## What already exists — reuse, do not rebuild + +- `scripts/_lib/eambrosia_register.py` — the whole resolver. + `load_id_map()` (cached at `raw/eambrosia-register/filenumber-id-map.json`, + 3,964 entries), `gi_detail()`, `attachment_refs()` → returns `cahier_uri` + **and** `single_doc_uri`, `fetch_attachment()` (handles the browser-UA gate + and HTTP 202-with-PDF-body). +- `scripts/_lib/fiche_technique.py` — parser for the single document, if you + ever need the 11 % fallback tier. +- `scripts/02_extract_cahiers.py` — `extract_sections` (AOC roman-numeral), + `extract_igp_sections`, `route_sections`, `candidate_keys()`. +- `scripts/01_scrape_cahiers.py` — `_process_app` walks `pdf_urls` in order + via `_download_first_pdf`; `_prepend_override_urls` puts curator URLs first. + +Endpoint gotchas already encoded in the lib (do not rediscover): the internal +`id` is **not** derivable from `giIdentifier`; the detail path has **no +`/v1/`**; the attachment path **does**; an explicit `Accept: application/pdf` +trips the anti-bot gate. + +## The one genuinely new component + +**A name → `fileNumber` resolver for FR.** France is INAO-sourced and carries +no `id_eambrosia`, so unlike every other country there is no join key. + +- Match SIQO `name` against register `protectedName`, casefolded and + accent-stripped. +- Exact match alone is **not sufficient** — it missed 2 of 9 spot-checks. + Route through the existing `candidate_keys()`, which already splits on + `" ou "`, `" et "`, and commas (`Muscat de Frontignan` is a + `X ou Y ou Z` case). +- The mapping must be **auditable and pinnable**: emit a resolved map plus an + unresolved queue, and honour a curator override file for the residue. Do + not silently fuzzy-match — a wrong bind attaches the wrong cahier to an + appellation, which is worse than a gap. +- Sub-denominations (1,074 of the 1,540) have no register entry of their own; + they inherit the parent cahier, as they do today. Resolve at parent level + only (466 parents vs 458 register GIs). + +## Constraints + +1. **Additive first.** Append the register URL as a final tier in + `pdf_urls`, after BO Agri and the curator overrides. Do not reorder or + remove existing sources in this change. The 466 current canonical + resolutions must stay canonical. +2. **Shadow before swap.** Land a mode that fetches and extracts from the + register *without* writing into `raw/inao/cahier-extracted/`, and report + per-appellation whether the register text matches the in-build text. Ship + the comparison before shipping the behaviour change. +3. **Public sources only** — satisfied; the register is the EU's own. + Attribute it as such. +4. **Reproducibility + no-op-on-rerun.** Cache attachments by sha256, record + full provenance (`register_file_number`, attachment uri, sha256, + `fetched_at`) in the manifest. A second run with no upstream change must + be a cache hit. +5. **Polite client.** The 202 + UA gate is deliberate anti-bot. Rate-limit, + reuse the cached id map (one ~4 MB POST), never parallel-hammer. +6. **No comments unless the *why* is non-obvious**; ruff line length 100; + logs to stderr with per-appellation progress. + +## Verification protocol — mandatory + +A pre-change baseline exists at +`/Users/boris/playground/owm-fr-baseline-2026-08-29/` (git HEAD `7e814cd`, +clean tree): full APFS clones of `raw/` and `wiki/`, plus SHA-256 manifests +of 39,334 files across 10 surfaces. See its `BASELINE.md`. + + /Users/boris/playground/owm-fr-baseline-2026-08-29/compare.sh + +Exit 0 means every surface is byte-identical. It was verified green at +capture time, so any divergence it reports is caused by your change. + +- While the change is additive/shadow-only, `compare.sh` **must stay exit 0**. +- Once the register tier can actually win a resolution, every diff must be + explained per record — "the extractor ran again" is not an explanation. + Expect `fr-cahier-extracted` churn to be **zero** for records whose BO Agri + URL still resolves, since the register is a later tier. +- Stage 04 failures are silent (a country just vanishes from the map). + `wiki-data`, `wiki-entity-pages` and `wiki-map-data` diffs are the tripwire + — do not dismiss them as noise. + +## Definition of done + +- A resolved FR name → `fileNumber` map, an unresolved queue, and a curator + override file for pinning the residue. +- Register tier wired into stage 01 as the last fallback, with provenance in + the manifest. +- Shadow report over all 466 parents: how many resolve, how many fetch a + cahier, and for how many the extracted `lien_au_terroir` is byte-identical + to the current build. +- `compare.sh` either exit 0, or every diff explained record by record. +- `CLAUDE.md` (FR pipeline section + scripts contract table) and + `CURATOR_TODO.md` updated. The cross-country Phase-2 retrofit item in + `CURATOR_TODO.md` is the parent of this work — reconcile it, do not + duplicate it. + +## Out of scope + +- Removing BO Agri, `01b_solve_legifrance.py`, or the OCR mirror path. Those + come out only after the shadow report proves the register covers them — + a later, separate change. +- Other countries. The cross-country retrofit is tracked separately; this + brief is FR only. +- The 02d IGP-slicing bug (`Spécificité de la zone géographique` headings not + recognised, so IGP terroir facts land in one bucket). Real, confirmed + during the spike, tracked in `CURATOR_TODO.md` — but a different fix. + +## Resolve early + +- The 15 % without a cahier: fall back to the register **single document** + (thinner, different template — needs `fiche_technique.py` and an FR keyword + table, which `scripts/_lib/be/document.py` already has for the Walloon + wines), or leave them on BO Agri? Cheapest answer is probably: leave them. +- Cahier **vintage**: the two records checked were byte-identical, but that + is n=2. The shadow report should quantify this across all 466 before anyone + argues for making the register a *primary* source. diff --git a/docs/reconciliation-log.md b/docs/reconciliation-log.md index e7e6e62..a4143cc 100644 --- a/docs/reconciliation-log.md +++ b/docs/reconciliation-log.md @@ -8,6 +8,67 @@ Newest first. --- +## 2026-08-29 — FR cahiers from the eAmbrosia register (plumbing + shadow report) + +Gave the French pipeline a second, self-service source for the cahier des +charges: the eAmbrosia EU GI register's `productSpecifications[0]` attachment, +which is the INAO cahier itself (`CDC_Batard-Montrachet.pdf`) and parses with +the unchanged stage-02 extractor. Additive only — the tier runs behind BO Agri +and the curator overrides, and it wins nothing on the current corpus. + +**Shipped** + +- `scripts/01d_resolve_register.py` + `scripts/_lib/fr/register_match.py` — + the name → `fileNumber` resolver FR needs because it is the one + INAO-sourced country with no `id_eambrosia`. **466 / 466 parents resolved** + (444 full-name, 20 alias, 2 pinned), 2 in the queue by design. Four guards: + product-type partition (Calvados the wine vs Calvados the eau-de-vie), no + fuzzy matching, one-to-one, and live-registrations-only. +- `scripts/_lib/fr/register_cahier.py` — sha256-addressed attachment cache + under `raw/inao/register/`, deliberately outside `raw/inao/cahiers/` (stage + 02 indexes every PDF there). Persists per fetch; remembers data-fact misses. +- `RegisterTier` in `scripts/01_scrape_cahiers.py` — last tier, additionally + gated on the appellation having no cahier PDF on disk so an INAO outage + cannot re-source a working record. +- `scripts/audit_fr_register_shadow.py` — read-only three-way comparison + (build / self / register) with a determinism control. +- Honest attribution end-to-end: `boagri_url` stays empty for a + register-sourced cahier, the attachment URL rides `register_attachment_url`, + and stage 03 + the map panel label it "registre GI de l'UE" (new gettext + msgid, en/es/nl filled). +- `scripts/_lib/fr/naming.py` — move-only extraction of `normalize_name` / + `candidate_keys` out of stage 02 (verified identical over 6,143 SIQO names). +- 33 unit tests across `tests/test_fr_register_match.py` + + `tests/test_fr_register_tier.py`. + +**Shadow-report verdict (all 466 parents)** + +464 resolve · 361 (77.5 %) get a cahier attachment · 352 extract · **92 +byte-identical** · 45 cosmetically different · 215 substantively different. +Determinism control 356/356. **In 199 of the 215 substantive differences the +newer text is BO Agri's** — the register is systematically an older vintage +(Anjou Villages: BO Agri "43 communes … 3 communes" vs register "24 … 2"; +Anjou and Arbois carry 2022–2023 republications). Conclusion: keep the +register a last tier; do not promote it. Full table at +`raw/inao/register/shadow-report.md`. + +**Opened** + +- ❌ `extract_aire`'s `_DEPT_HEADER_PATTERN` backtracks pathologically on + JORF-issue layouts (6 register documents time out). Pre-existing, latent, + not triggered by anything in the current build; fixing it needs its own + corpus-wide diff. +- ❌ Cité de Carcassonne + Coteaux de Narbonne are `Cancelled` in the register + but still Publié in SIQO — the mirror of the 2026-08-26 SIQO-ghost + retirement. +- 🟡 Collioure, Pouilly-Loché, Franche-Comté would gain a materially richer + `lien` from the register; needs a stub-driven re-source path. + +**Verification**: `owm-fr-baseline-2026-08-29/compare.sh` exit 0 — all 10 +surfaces byte-identical. `ruff` clean, 352 tests pass. + +--- + ## 2026-08-26 — /research-gaps vivc-ambiguous: 42-slug pin pass The queue surfaced by the corpus-walk widening (same-day action pass below), diff --git a/scripts/01_scrape_cahiers.py b/scripts/01_scrape_cahiers.py index b98ce1d..25d3c6d 100644 --- a/scripts/01_scrape_cahiers.py +++ b/scripts/01_scrape_cahiers.py @@ -15,9 +15,18 @@ The cahier text is published to the JORF and mirrored on BO Agri, which is where we ultimately download from. +A final fallback tier reads the **eAmbrosia EU GI register**, which serves +the same INAO cahier PDF as an attachment (`RegisterTier`, bound to an +appellation by the name→file-number map stage 01d writes). It runs only after +BO Agri and the curator overrides have both failed, so appellations that +already resolve keep their canonical source; disable it with `--no-register`. + Re-runnable: a manifest at `raw/inao/cahiers/manifest.json` records the resolved show_texte id, BO Agri URL, and sha256 per appellation. Re-running diffs against the manifest and only re-downloads what changed upstream. +Register-sourced entries additionally carry `source_kind` + +`register_file_number` + the attachment uri, and their PDF is cached by +sha256 under `raw/inao/register/`. """ from __future__ import annotations @@ -38,6 +47,11 @@ from tqdm import tqdm ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / "scripts")) + +from _lib import eambrosia_register as er # noqa: E402 +from _lib.fr import register_cahier as rc # noqa: E402 + RAW = ROOT / "raw" SIQO_CSV = RAW / "inao" / "siqo-referentiel.csv" OUT_DIR = RAW / "inao" / "cahiers" @@ -306,14 +320,129 @@ def _download_alt_candidates( return n -def _process_override_only( - session: requests.Session, app: Appellation, manifest: dict, - override: dict, delay: float, -) -> tuple[str, int]: - """INAO didn't surface any candidates but a manual override is set. - Download the override URLs and seed a manifest entry from them so - stage 02's cross-bundle rescue can pick this AOC up.""" - sample = app.products[0] if app.products else {} +class RegisterTier: + """Last-resort cahier source: the eAmbrosia EU GI register. + + The register serves the same INAO cahier PDF that BO Agri publishes, so an + appellation BO Agri surfaces nothing for no longer needs a curator URL, a + Légifrance cookie, or OCR over a professional-org mirror. It runs strictly + *after* BO Agri and the curator overrides, and only for an appellation + with no cahier PDF on disk — a run that fails to *reach* INAO must not + re-source an appellation that already has one (an INAO outage is not an + upstream change). Recovering an appellation whose product page links the + *wrong* arrêté — a PDF that downloads fine but carries someone else's + cahier — is a stage-02 question, not a stage-01 one, and is out of scope + here. + + Requires the name → file-number map from stage 01d; without it the tier is + inert. `raw/inao/register/` holds the attachment cache, deliberately + outside `raw/inao/cahiers/` — only a PDF that actually wins a resolution + is promoted there, because stage 02 indexes every PDF in that directory. + """ + + def __init__(self, session: requests.Session, delay: float, enabled: bool = True): + self.session = session + self.delay = delay + self.resolved = rc.load_resolved() if enabled else {} + self.manifest = rc.load_manifest() if enabled else {} + self._id_map: dict[str, int] | None = None + + @property + def enabled(self) -> bool: + return bool(self.resolved) + + @property + def id_map(self) -> dict[str, int]: + """Derived from the same cached GI listing stage 01d resolved names + against, so a binding can never name a file number the id map lacks.""" + if self._id_map is None: + try: + self._id_map = er.id_map_from_rows(er.load_gi_rows(session=self.session)) + except (requests.RequestException, ValueError, OSError) as exc: + print(f"[register] GI listing unavailable ({exc}); tier inert " + f"for the rest of this run", file=sys.stderr) + self._id_map = {} + return self._id_map + + def cahier_for(self, id_appellation: str) -> rc.RegisterCahier | None: + """Fetch (or reuse) the register cahier bound to `id_appellation`.""" + binding = self.resolved.get(id_appellation) + if not binding or not binding.get("file_number"): + return None + cahier, status = rc.fetch( + binding["file_number"], self.id_map, self.manifest, + session=self.session, delay=self.delay, + ) + if cahier is None: + print(f"[register] {binding.get('name', id_appellation)}: {status}", + file=sys.stderr) + return None + return cahier + + def save(self) -> None: + if self.enabled: + rc.save_manifest(self.manifest) + + +def _promote_register_pdf(cahier: rc.RegisterCahier) -> Path: + """Copy a register attachment into the content-addressed cahiers dir so + stage 02 finds it on its normal path. + + Written through a temp file then renamed, like `download_pdf`: stage 02 + indexes every PDF in this directory, and a half-written file under a name + that asserts a sha256 it does not have would be picked up as a cahier and + never repaired (the existence check would skip it on every later run). + A size mismatch is therefore treated as absent.""" + dest = OUT_DIR / f"{cahier.sha256}.pdf" + if dest.exists() and dest.stat().st_size == cahier.bytes: + return dest + OUT_DIR.mkdir(parents=True, exist_ok=True) + tmp = OUT_DIR / f".part-register-{cahier.sha256}.pdf" + tmp.write_bytes(cahier.path.read_bytes()) + tmp.replace(dest) + return dest + + +def _apply_register(meta: dict, app: Appellation, cahier: rc.RegisterCahier, + binding: dict) -> dict: + """Fill a manifest entry from a register-sourced cahier. The provenance + keys are only ever written for entries the register actually won, so an + appellation still served by BO Agri keeps a byte-identical entry.""" + dest = _promote_register_pdf(cahier) + # `boagri_url` means "the BO Agri URL this PDF came from" all the way down + # to the wiki's "BO Agri (PDF source)" line and the map panel's cahier + # link. A register attachment did not come from BO Agri, so it gets its + # own field rather than borrowing that one. + meta["boagri_url"] = "" + meta["register_attachment_url"] = cahier.attachment_url + meta["filename"] = dest.name + meta["sha256"] = cahier.sha256 + meta["fetched_at"] = cahier.fetched_at + meta["source_kind"] = "eambrosia-register" + meta["register_file_number"] = cahier.file_number + meta["register_attachment_uri"] = cahier.attachment_uri + meta["register_attachment_name"] = cahier.attachment_name + meta["register_protected_name"] = binding.get("register_name", "") + meta["register_matched_via"] = binding.get("matched_via", "") + print(f"[register] {app.name} ({app.id_appellation}) -> " + f"{cahier.file_number} {cahier.attachment_name}", file=sys.stderr) + return meta + + +_INAO_META_KEYS = ( + "canonical_idproduit", "canonical_produit", "product_url", "show_texte_url", + "show_texte_paths", "boagri_url_candidates", "legifrance_jorftext_ids", +) + + +def _stub_meta(app: Appellation, prior: dict | None = None) -> dict: + """A manifest entry for an appellation INAO surfaced nothing for. + + Whatever INAO catalogue metadata an earlier run did record is carried + over: this run failing to reach www2.inao.gouv.fr is not evidence that the + product page, show_texte trail or Légifrance ids ceased to exist.""" + sample = app.canonical_product() if app.products else {} + prior = prior or {} meta = { "name": app.name, "canonical_idproduit": "", @@ -328,8 +457,22 @@ def _process_override_only( "boagri_url": "", "boagri_url_candidates": [], "legifrance_jorftext_ids": [], - "manual_override_note": override.get("note", ""), } + for key in _INAO_META_KEYS: + if prior.get(key): + meta[key] = prior[key] + return meta + + +def _process_override_only( + session: requests.Session, app: Appellation, manifest: dict, + override: dict, delay: float, +) -> tuple[str, int]: + """INAO didn't surface any candidates but a manual override is set. + Download the override URLs and seed a manifest entry from them so + stage 02's cross-bundle rescue can pick this AOC up.""" + meta = _stub_meta(app) + meta["manual_override_note"] = override.get("note", "") n = _download_alt_candidates(session, app.name, override["boagri_urls"], delay) try: digest, dest = download_pdf(session, override["boagri_urls"][0], OUT_DIR) @@ -344,13 +487,50 @@ def _process_override_only( return "override-only", n +def _has_usable_cahier(prior: dict) -> bool: + """True when the manifest already points this appellation at a PDF that is + still on disk.""" + sha = prior.get("sha256") or "" + return bool(sha) and (OUT_DIR / f"{sha}.pdf").exists() + + +def _register_tier( + app: Appellation, manifest: dict, meta: dict, register: RegisterTier | None, + prior: dict, fallback: tuple[str, int] | None, +): + """Final tier. Runs only once BO Agri and the curator overrides have both + failed to yield a PDF *and* the appellation has none from an earlier run. + + That second condition is what keeps a transient INAO outage from + re-sourcing a working appellation: `resolve_cahier` returns None for a + 5xx, a timeout, or an empty Drupal body just as it does for a genuinely + absent cahier, and `_download_first_pdf` returns None for a BO Agri blip + just as it does for a dead URL. Without the guard, one bad afternoon would + flip canonical resolutions to the register and churn every downstream + surface with no upstream change behind it. + + Returns `fallback` (or `(None, 0)` when fallback is None) if the register + has nothing for this appellation.""" + miss = fallback if fallback is not None else (None, 0) + if register is None or not register.enabled: + return miss + if _has_usable_cahier(prior): + return miss + cahier = register.cahier_for(app.id_appellation) + if cahier is None: + return miss + manifest[app.id_appellation] = _apply_register( + meta, app, cahier, register.resolved.get(app.id_appellation, {})) + return "register", 0 + + def _process_app( session: requests.Session, app: Appellation, manifest: dict, - overrides: dict, delay: float, + overrides: dict, delay: float, register: RegisterTier | None = None, ) -> tuple[str, int]: """Resolve and download `app`'s cahier(s). Returns (status, alt_count) where status is one of: missed, cached, fetched, legifrance-only, - override-only. + override-only, register. """ prior = manifest.get(app.id_appellation, {}) override = overrides.get(app.id_appellation) @@ -360,8 +540,15 @@ def _process_app( time.sleep(delay) if result is None: if has_override_urls: - return _process_override_only(session, app, manifest, override, delay) - return "missed", 0 + status, n = _process_override_only(session, app, manifest, override, delay) + if status != "missed": + return status, n + stub = _stub_meta(app, prior) + stub["manual_override_note"] = override.get("note", "") + return _register_tier(app, manifest, stub, register, prior, + fallback=("missed", n)) + return _register_tier(app, manifest, _stub_meta(app, prior), register, prior, + fallback=("missed", 0)) meta, pdf_urls = result if has_override_urls: pdf_urls = _prepend_override_urls(override["boagri_urls"], pdf_urls) @@ -369,6 +556,9 @@ def _process_app( meta["manual_override_note"] = override.get("note", "") if not pdf_urls: + status, n = _register_tier(app, manifest, meta, register, prior, fallback=None) + if status is not None: + return status, n meta["filename"] = "" meta["sha256"] = "" meta["fetched_at"] = datetime.now(timezone.utc).isoformat(timespec="seconds") @@ -391,7 +581,7 @@ def _process_app( return "cached", n download = _download_first_pdf(session, app.name, pdf_urls, delay) if download is None: - return "missed", 0 + return _register_tier(app, manifest, meta, register, prior, fallback=("missed", 0)) canonical_url, digest, dest = download rest = [u for u in pdf_urls if u != canonical_url] n = _download_alt_candidates(session, app.name, rest, delay) @@ -450,6 +640,17 @@ def main() -> int: action="store_true", help="only re-attempt appellations not yet in the manifest", ) + ap.add_argument( + "--no-register", + action="store_true", + help="skip the eAmbrosia register fallback tier", + ) + ap.add_argument( + "--register-delay", + type=float, + default=rc.DEFAULT_DELAY, + help="seconds between eAmbrosia register requests", + ) args = ap.parse_args() appellations = load_appellations(SIQO_CSV) @@ -477,29 +678,44 @@ def main() -> int: session = requests.Session() session.headers["User-Agent"] = UA + register = RegisterTier(requests.Session(), args.register_delay, + enabled=not args.no_register) + if register.enabled: + print(f"[register] {len(register.resolved)} name→file-number binding(s) " + f"available as the final tier", file=sys.stderr) + elif not args.no_register: + print(f"[register] no {rc.RESOLVED_PATH.relative_to(ROOT)} — " + f"run scripts/01d_resolve_register.py to enable the register tier", + file=sys.stderr) + fetched = cached = missed = extra = 0 counters = { "fetched": 0, "cached": 0, "missed": 0, - "legifrance-only": 0, "override-only": 0, + "legifrance-only": 0, "override-only": 0, "register": 0, } - for app in tqdm(appellations, desc="cahiers", leave=False): - status, alt_count = _process_app( - session, app, manifest, overrides, args.delay + try: + for app in tqdm(appellations, desc="cahiers", leave=False): + status, alt_count = _process_app( + session, app, manifest, overrides, args.delay, register + ) + counters[status] += 1 + extra += alt_count + time.sleep(args.delay) + finally: + register.save() + MANIFEST_PATH.write_text( + json.dumps(manifest, indent=2, sort_keys=True, ensure_ascii=False), + encoding="utf-8", ) - counters[status] += 1 - extra += alt_count - time.sleep(args.delay) fetched = counters["fetched"] + counters["override-only"] cached = counters["cached"] missed = counters["missed"] + counters["legifrance-only"] - MANIFEST_PATH.write_text(json.dumps(manifest, indent=2, sort_keys=True, ensure_ascii=False), encoding="utf-8") - print( - f"[done] fetched={fetched} cached={cached} extra={extra} missed={missed} " - f"manifest={MANIFEST_PATH.relative_to(ROOT)}", + f"[done] fetched={fetched} cached={cached} register={counters['register']} " + f"extra={extra} missed={missed} manifest={MANIFEST_PATH.relative_to(ROOT)}", file=sys.stderr, ) return 0 if missed == 0 else 2 diff --git a/scripts/01d_resolve_register.py b/scripts/01d_resolve_register.py new file mode 100644 index 0000000..ed469e0 --- /dev/null +++ b/scripts/01d_resolve_register.py @@ -0,0 +1,125 @@ +"""Resolve every French appellation to its eAmbrosia register file number. + +Pipeline stage 01d. + +France is INAO-sourced and carries no `id_eambrosia`, so the register — which +serves the INAO cahier des charges PDF per GI, with none of BO Agri's +hand-curated URLs, Légifrance cookie injection or OCR-over-mirrors — needs a +name-based join key before stage 01 can use it as a fallback source. + +Reads the parent appellations from `raw/inao/cahiers/manifest.json` (falling +back to the SIQO referentiel when stage 01 has not run yet) and the register's +own GI listing, and writes: + + raw/inao/register/resolved.json id_appellation -> register GI binding + raw/inao/register/unresolved.json curator queue, with a reason per entry + +Pin the residue in `scripts/_lib/fr/register_overrides.json` (checked in) and +re-run; the resolver never fuzzy-matches, so anything it cannot settle on +exact alias keys is a curator decision. +""" + +from __future__ import annotations + +import argparse +import csv +import json +import sys +from collections import Counter +from pathlib import Path + +import requests + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / "scripts")) + +from _lib import eambrosia_register as er # noqa: E402 +from _lib.fr import register_cahier as rc # noqa: E402 +from _lib.fr import register_match as rm # noqa: E402 + +MANIFEST_PATH = ROOT / "raw" / "inao" / "cahiers" / "manifest.json" +SIQO_CSV = ROOT / "raw" / "inao" / "siqo-referentiel.csv" +WINE_SIGNS = {"AOC", "AOP", "IGP"} + + +def appellations_from_manifest() -> list[dict]: + manifest = json.loads(MANIFEST_PATH.read_text(encoding="utf-8")) + return [ + {"id_appellation": k, "name": v["name"], "categorie": v.get("categorie", "")} + for k, v in manifest.items() + ] + + +def appellations_from_siqo() -> list[dict]: + """Stage-01-free fallback: the same VITICOLE / AOC-AOP-IGP / Publié + filter stage 01 applies, one row per id_appellation.""" + out: dict[str, dict] = {} + with open(SIQO_CSV, encoding="utf-8-sig", newline="") as f: + for row in csv.DictReader(f): + if row["secteur"].strip() != "VITICOLE" or row["lib_etat"].strip() != "Publié": + continue + if (row["signe_fr"].strip() or row["signe_ue"].strip()) not in WINE_SIGNS: + continue + id_app = row["id_appellation"].strip() + out.setdefault(id_app, { + "id_appellation": id_app, + "name": row["appellation"].strip(), + "categorie": row["categorie"].strip(), + }) + return list(out.values()) + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--refresh", action="store_true", help="re-fetch the register GI listing") + ap.add_argument("--strict", action="store_true", help="exit 2 when the queue is non-empty") + args = ap.parse_args() + + if MANIFEST_PATH.exists(): + appellations = appellations_from_manifest() + origin = MANIFEST_PATH + elif SIQO_CSV.exists(): + appellations = appellations_from_siqo() + origin = SIQO_CSV + else: + print(f"error: neither {MANIFEST_PATH} nor {SIQO_CSV} — run stage 00/01 first", + file=sys.stderr) + return 1 + + session = requests.Session() + rows = rm.fr_rows(er.load_gi_rows(refresh=args.refresh, session=session)) + overrides = rm.load_overrides() + print( + f"[plan] {len(appellations)} appellations from {origin.relative_to(ROOT)} " + f"vs {len(rows)} FR register GIs; {len(overrides)} curator pin(s)", + file=sys.stderr, + ) + + resolved, unresolved = rm.resolve_all(appellations, rows, overrides) + + rc.REGISTER_DIR.mkdir(parents=True, exist_ok=True) + rc.RESOLVED_PATH.write_text( + json.dumps(rm.to_json(resolved), ensure_ascii=False, indent=2), encoding="utf-8" + ) + rc.UNRESOLVED_PATH.write_text( + json.dumps(unresolved, ensure_ascii=False, indent=2), encoding="utf-8" + ) + + by_type = Counter(r.product_type for r in resolved.values()) + by_via = Counter(r.matched_via for r in resolved.values()) + by_status = Counter(r.status for r in resolved.values()) + for u in unresolved: + print(f"[queue] {u['name']} ({u['id_appellation']}): {u['reason']} — {u['detail']}", + file=sys.stderr) + print( + f"[done] resolved={len(resolved)} ({dict(by_via)}; {dict(by_type)}; " + f"{dict(by_status)}) " + f"unresolved={len(unresolved)} " + f"-> {rc.RESOLVED_PATH.relative_to(ROOT)}, {rc.UNRESOLVED_PATH.relative_to(ROOT)}", + file=sys.stderr, + ) + return 2 if (args.strict and unresolved) else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/02_extract_cahiers.py b/scripts/02_extract_cahiers.py index f1c6445..80b413a 100644 --- a/scripts/02_extract_cahiers.py +++ b/scripts/02_extract_cahiers.py @@ -38,6 +38,7 @@ from collections import defaultdict from pathlib import Path +from _lib.fr.naming import candidate_keys, normalize_name from _lib.grape_entity import ( flush_unknowns_queue, preheat_vocabulary, @@ -128,41 +129,6 @@ def slug(s: str) -> str: return s -def normalize_name(s: str) -> str: - """Loose match key — strips diacritics, casing, spacing/hyphens.""" - s = unicodedata.normalize("NFKD", s).encode("ascii", "ignore").decode() - return re.sub(r"[\W_]+", "", s).lower() - - -# AOC names regularly carry one or more aliases concatenated with " ou ", " et ", -# or comma-separated lists — e.g. "Cidre de Normandie ou Cidre normand", -# "Cognac ou Eau-de-vie de Cognac ou Eau-de-vie des Charentes", -# "Côtes de Bourg, Bourg et Bourgeais". The cahier itself usually carries only -# one of those variants as its segment header, so a strict normalize-and-equal -# match between parent name and segment header misses them. Splitting both sides -# into alias parts and matching on any shared component closes the gap without -# the false-positive risk of pure substring matching ("Bourgogne" would -# otherwise match a "Bourgogne Passe-tout-grains" segment). -_ALIAS_SPLIT_RE = re.compile(r"\s+ou\s+|\s+et\s+|,\s*", flags=re.IGNORECASE) - - -def candidate_keys(name: str) -> list[str]: - """Return a list of normalised match keys for `name` — the full normalised - form first, followed by aliases split on " ou ", " et ", and commas.""" - keys: list[str] = [] - full = normalize_name(name) - if full: - keys.append(full) - for part in _ALIAS_SPLIT_RE.split(name): - part = part.strip() - if not part: - continue - k = normalize_name(part) - if k and k not in keys: - keys.append(k) - return keys - - def _is_parent_denom(d: dict) -> bool: """True iff a SIQO denomination row represents its appellation's parent record (vs a DGC). Strict equality covers the common case; fuzzy @@ -1481,6 +1447,18 @@ def main() -> int: "latest_known_pdf": latest_pdf, "latest_known_homologated_at": latest_date, } + # Only present when stage 01's register tier won the resolution, so a + # BO Agri-sourced record keeps a byte-identical `source` block. + if meta.get("source_kind"): + record["source"]["source_kind"] = meta["source_kind"] + for key in ( + "register_file_number", + "register_attachment_uri", + "register_attachment_url", + "register_attachment_name", + "register_protected_name", + ): + record["source"][key] = meta.get(key, "") record["signe_fr"] = meta.get("signe_fr", "") record["signe_ue"] = meta.get("signe_ue", "") record["categorie"] = meta.get("categorie", "") diff --git a/scripts/03_generate_wiki.py b/scripts/03_generate_wiki.py index b194581..bdf8187 100644 --- a/scripts/03_generate_wiki.py +++ b/scripts/03_generate_wiki.py @@ -351,6 +351,8 @@ def render_page(record: dict) -> str: f" cahier: {cahier_relpath}", f" show_texte: {src['show_texte_url']}", f" boagri: {src['boagri_url']}", + *([f" eu_register: {src['register_attachment_url']}"] + if src.get("register_attachment_url") else []), f" product: {src['product_url']}", f" pdf_sha256: {src['pdf_sha256']}", f" homologated_at: {homologated_at}", @@ -444,7 +446,11 @@ def render_page(record: dict) -> str: f"- Cahier des charges (PDF): [`{cahier_relpath}`]({cahier_relpath}) — sha256 `{src['pdf_sha256'][:16]}…`", (f"- Cahier homologué le **{homologated_at}**" if homologated_at else "- _(date d'homologation non extraite)_"), f"- INAO show_texte: <{src['show_texte_url']}>", - f"- BO Agri (PDF source): <{src['boagri_url']}>", + # Register-sourced cahiers did not come from BO Agri; every other + # record keeps the line it has always had, empty URL included. + (f"- Registre GI de l'UE (PDF source): <{src['register_attachment_url']}>" + if src.get("register_attachment_url") + else f"- BO Agri (PDF source): <{src['boagri_url']}>"), f"- INAO produit (catalogue): <{src['product_url']}>", "", ] diff --git a/scripts/04_build_maps.py b/scripts/04_build_maps.py index 926c2fb..f711ab7 100644 --- a/scripts/04_build_maps.py +++ b/scripts/04_build_maps.py @@ -76,7 +76,6 @@ from _lib.augment.ro import augment_ro_records_with_national_specs from _lib.augment.si import augment_si_records_with_specifikacija from _lib.augment.sk import augment_sk_records_with_national_specs -from _lib.batch import _load_dotenv from _lib.be.geometry import BEPolygonIndex from _lib.be.region import derive_region as derive_be_region from _lib.bg.geometry import BGPolygonIndex @@ -91,6 +90,7 @@ from _lib.cz.region import derive_region as derive_cz_region from _lib.de.geometry import DEPolygonIndex from _lib.de.region import derive_region as derive_de_region +from _lib.env import load_dotenv from _lib.es.geometry import ESPolygonIndex from _lib.es.region import ( derive_ccaa as derive_es_ccaa, @@ -3213,7 +3213,7 @@ def _sources_for(record: dict) -> dict: "national_pliego_fetched_at": nat.get("fetched_at", ""), "national_pliego_added_slugs": nat.get("added_slugs") or [], } - return { + fr = { "country": "fr", "boagri": src.get("boagri_url") or "", "show_texte": src.get("show_texte_url") or "", @@ -3224,6 +3224,12 @@ def _sources_for(record: dict) -> dict: "homologation_date": (record.get("header") or {}).get("homologation_date") or "", "jorf_date": (record.get("header") or {}).get("jorf_date") or "", } + # Added only for a cahier the register tier won, so the panel never labels + # an EU-register attachment as a BO Agri document — and so every record + # still sourced from BO Agri keeps a byte-identical panel payload. + if src.get("register_attachment_url"): + fr["eu_register_cahier"] = src["register_attachment_url"] + return fr # Curator-pinned terroir-facts inheritance for wines whose canonical @@ -4229,7 +4235,7 @@ def write_seo_files(entity_entries: list[tuple[str, str, str]] | None = None) -> # environment / repo-root .env — deliberately NOT committed — so a checkout # without it simply skips the file while everything else in wiki/ still # reproduces. File name == body, no trailing newline, served as text/plain. - _load_dotenv() + load_dotenv() indexnow_key = os.environ.get("INDEXNOW_KEY", "").strip() indexnow_note = "" if indexnow_key: diff --git a/scripts/_lib/assets/app.js b/scripts/_lib/assets/app.js index 070dd4b..7594967 100644 --- a/scripts/_lib/assets/app.js +++ b/scripts/_lib/assets/app.js @@ -418,9 +418,23 @@ // front and switched by layer visibility, so the theme toggle is live — no // source remove/re-add (which would reorder layers above the appellation // polygons and drop their selection feature-state). Same CARTO / OSM credit. + // + // CARTO retired key-less raster access (2026-08): an unkeyed tile comes back + // with "API KEY REQUIRED" stamped into the PNG itself. The key is client-side + // by necessity — the browser issues the tile request — but it is injected at + // build time from CARTO_BASEMAP_KEY (repo-root .env, the same file + // scripts/deploy.sh sources) rather than committed, so it stays rotatable and + // out of git history. Unset -> no ?key= and a watermarked basemap; stage 04 + // warns. Free tier: 5M tile requests/month across raster + vector, and the + // CARTO + OpenStreetMap attribution below is the condition of it — keep it. + // CARTO is retiring raster altogether; the successor is vendored next to the + // runtime libs (scripts/_lib/vendor/openfreemap-*.json — key-free OSM vector + // tiles, a positron/dark pair that maps onto this same visibility toggle). + const CARTO_KEY = __OWM_carto_key_json__; function cartoTiles(style) { + const auth = CARTO_KEY ? '?key=' + CARTO_KEY : ''; return ['a', 'b', 'c'].map(function (s) { - return 'https://' + s + '.basemaps.cartocdn.com/' + style + '/{z}/{x}/{y}.png'; + return 'https://' + s + '.basemaps.cartocdn.com/' + style + '/{z}/{x}/{y}.png' + auth; }); } function effectiveTheme() { @@ -1733,10 +1747,12 @@ function renderSources(slug, sources) { if (!sources) sources = {}; const links = []; - if (sources.boagri) { + const cahierHref = sources.boagri || sources.eu_register_cahier; + if (cahierHref) { const homo = sources.homologation_date ? ' — ' + LABELS.src_homologated + ' ' + escapeHtml(sources.homologation_date) : ''; const jorf = sources.jorf_date ? ', ' + LABELS.src_jorf + ' ' + escapeHtml(sources.jorf_date) : ''; - links.push(`
  • ${LABELS.src_cahier}${homo}${jorf}
    • `); + const cahierLabel = sources.boagri ? LABELS.src_cahier : LABELS.src_cahier_eu_register; + links.push(`
  • ${cahierLabel}${homo}${jorf}
    • `); } if (sources.show_texte) { links.push(`
  • ${LABELS.src_show_texte}
    • `); @@ -1987,7 +2003,9 @@ ? [eambrosiaReg, PROV_EU_DOC_TERM[country] || 'single document', false, extra] : null; if (country === 'fr') { - if (s.boagri) { + // The register serves the INAO cahier itself, so the authoring body is + // INAO either way; only the link label differs (see renderSources). + if (s.boagri || s.eu_register_cahier) { const d = s.homologation_date || s.jorf_date || ''; return ['INAO', 'cahier des charges', false, d ? ` (${escapeHtml(d)})` : '']; } diff --git a/scripts/_lib/batch.py b/scripts/_lib/batch.py index 8626ab3..55bf773 100644 --- a/scripts/_lib/batch.py +++ b/scripts/_lib/batch.py @@ -37,24 +37,14 @@ import requests -ROOT = Path(__file__).resolve().parents[2] +from _lib.env import load_dotenv def _load_dotenv() -> None: """Populate os.environ from a repo-root .env (KEY=VALUE lines); existing environment variables win. Lets `--batch` pick up API keys without a manual export.""" - env = ROOT / ".env" - if not env.exists(): - return - for line in env.read_text(encoding="utf-8").splitlines(): - line = line.strip() - if not line or line.startswith("#") or "=" not in line: - continue - if line.startswith("export "): - line = line[7:] - key, val = line.split("=", 1) - os.environ.setdefault(key.strip(), val.strip().strip('"').strip("'")) + load_dotenv() POLL_INTERVAL_S = 20 diff --git a/scripts/_lib/eambrosia_register.py b/scripts/_lib/eambrosia_register.py index 6c6c26f..490e34a 100644 --- a/scripts/_lib/eambrosia_register.py +++ b/scripts/_lib/eambrosia_register.py @@ -50,6 +50,25 @@ ROOT = Path(__file__).resolve().parents[2] ID_MAP_CACHE = ROOT / "raw" / "eambrosia-register" / "filenumber-id-map.json" +GI_INDEX_CACHE = ROOT / "raw" / "eambrosia-register" / "gi-index.json" + +# The bulk filter row carries far more than the id map needs — the fields +# below are what a name-based resolver has to see: the protected name to +# match on, and the country / product-type / status axes that partition the +# candidate pool so a name shared by a wine and a spirit cannot cross-bind. +_GI_ROW_FIELDS = ( + "id", + "fileName", + "protectedName", + "countryId", + "countries", + "qualityProductType", + "geographicalIndicatorTypeCode", + "productCategory", + "status", + "showInRegister", + "registrationDate", +) def load_id_map(refresh: bool = False, session: requests.Session | None = None) -> dict[str, int]: @@ -81,6 +100,56 @@ def load_id_map(refresh: bool = False, session: requests.Session | None = None) return id_map +def load_gi_rows(refresh: bool = False, session: requests.Session | None = None) -> list[dict]: + """Return the trimmed register listing (one row per GI), cached on disk. + + Same single ~4 MB POST that backs `load_id_map`, but keeping the + `protectedName` / country / product-type columns a name-based resolver + needs. Seeds the id-map cache from the same response when it is cold, so + a cold start costs one request, not two.""" + if GI_INDEX_CACHE.exists() and not refresh: + try: + return json.loads(GI_INDEX_CACHE.read_text(encoding="utf-8")) + except (ValueError, OSError): + pass + s = session or requests.Session() + r = s.post( + _FILTER_URL, + headers={**_JSON_HEADERS, "Content-Type": "application/json"}, + json={"first": 0, "rows": 100000, "showTSGs": "false", "filters": []}, + timeout=300, + ) + r.raise_for_status() + rows = [ + {k: row.get(k) for k in _GI_ROW_FIELDS} + for row in r.json().get("results", []) + if row.get("fileName") and row.get("id") is not None + ] + GI_INDEX_CACHE.parent.mkdir(parents=True, exist_ok=True) + GI_INDEX_CACHE.write_text( + json.dumps(rows, ensure_ascii=False, indent=2, sort_keys=True), encoding="utf-8" + ) + # Keep the two caches on one vintage: a `--refresh` that renewed only the + # listing would leave a newly-registered GI resolvable by name but not by + # id for every other consumer of `load_id_map`. + ID_MAP_CACHE.write_text( + json.dumps(id_map_from_rows(rows), ensure_ascii=False, sort_keys=True), + encoding="utf-8", + ) + return rows + + +def id_map_from_rows(rows: list[dict]) -> dict[str, int]: + """{file_number: internal_id} derived from a `load_gi_rows` listing. + + Prefer this over `load_id_map` when the caller already holds (or resolved + names against) a `gi-index.json` vintage: the two caches are written + independently, so a file number added to the register after the id-map + cache was last written would otherwise resolve to a name but not to an + id.""" + return {row["fileName"]: row["id"] for row in rows if row.get("id") is not None} + + def gi_detail( file_number: str, id_map: dict[str, int], session: requests.Session | None = None, ) -> dict | None: diff --git a/scripts/_lib/env.py b/scripts/_lib/env.py new file mode 100644 index 0000000..eb5d665 --- /dev/null +++ b/scripts/_lib/env.py @@ -0,0 +1,56 @@ +"""Repo-root `.env` loading, shared by the stages that need a credential. + +The LLM stages read provider keys straight from the environment; stage 04 +needs the basemap key at *render* time (it is baked into the emitted app.js), +so it goes through the same `.env` that `scripts/deploy.sh` sources. +""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] + +_WARNED = False + + +def load_dotenv() -> None: + """Populate os.environ from a repo-root .env (KEY=VALUE lines); existing + environment variables win.""" + env = ROOT / ".env" + if not env.exists(): + return + for line in env.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + if line.startswith("export "): + line = line[7:] + key, val = line.split("=", 1) + os.environ.setdefault(key.strip(), val.strip().strip('"').strip("'")) + + +def carto_basemap_key() -> str: + """CARTO Basemaps key for the raster tile URLs. + + Client-side by necessity — the browser issues the tile request, so the key + is public once the map ships; it lives in `.env` (gitignored) rather than + in the source so it is rotatable and stays out of git history. + + Empty when unset: the map still builds and every polygon still renders, but + CARTO stamps "API KEY REQUIRED" across every basemap tile, so warn loudly — + a silent watermark is exactly the failure this indirection can reintroduce. + """ + global _WARNED + load_dotenv() + key = os.environ.get("CARTO_BASEMAP_KEY", "").strip() + if not key and not _WARNED: + _WARNED = True + print( + "WARNING: CARTO_BASEMAP_KEY is unset — the basemap will render with " + "CARTO's 'API KEY REQUIRED' watermark. Set it in the repo-root .env.", + file=sys.stderr, + ) + return key diff --git a/scripts/_lib/fr/__init__.py b/scripts/_lib/fr/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/_lib/fr/naming.py b/scripts/_lib/fr/naming.py new file mode 100644 index 0000000..9e1b609 --- /dev/null +++ b/scripts/_lib/fr/naming.py @@ -0,0 +1,47 @@ +"""Name-matching keys for French appellation names. + +Shared by stage 02 (matching a SIQO appellation to its cahier segment +header) and the eAmbrosia register resolver (matching a SIQO appellation to +a register `protectedName`). Extracted verbatim from +`scripts/02_extract_cahiers.py` so both callers fold aliases identically. +""" + +from __future__ import annotations + +import re +import unicodedata + + +def normalize_name(s: str) -> str: + """Loose match key — strips diacritics, casing, spacing/hyphens.""" + s = unicodedata.normalize("NFKD", s).encode("ascii", "ignore").decode() + return re.sub(r"[\W_]+", "", s).lower() + + +# AOC names regularly carry one or more aliases concatenated with " ou ", " et ", +# or comma-separated lists — e.g. "Cidre de Normandie ou Cidre normand", +# "Cognac ou Eau-de-vie de Cognac ou Eau-de-vie des Charentes", +# "Côtes de Bourg, Bourg et Bourgeais". The cahier itself usually carries only +# one of those variants as its segment header, so a strict normalize-and-equal +# match between parent name and segment header misses them. Splitting both sides +# into alias parts and matching on any shared component closes the gap without +# the false-positive risk of pure substring matching ("Bourgogne" would +# otherwise match a "Bourgogne Passe-tout-grains" segment). +_ALIAS_SPLIT_RE = re.compile(r"\s+ou\s+|\s+et\s+|,\s*", flags=re.IGNORECASE) + + +def candidate_keys(name: str) -> list[str]: + """Return a list of normalised match keys for `name` — the full normalised + form first, followed by aliases split on " ou ", " et ", and commas.""" + keys: list[str] = [] + full = normalize_name(name) + if full: + keys.append(full) + for part in _ALIAS_SPLIT_RE.split(name): + part = part.strip() + if not part: + continue + k = normalize_name(part) + if k and k not in keys: + keys.append(k) + return keys diff --git a/scripts/_lib/fr/register_cahier.py b/scripts/_lib/fr/register_cahier.py new file mode 100644 index 0000000..36fed03 --- /dev/null +++ b/scripts/_lib/fr/register_cahier.py @@ -0,0 +1,207 @@ +"""Fetch + cache the eAmbrosia register's national cahier des charges PDF. + +The register serves the INAO cahier itself (`productSpecifications[0]` — +attachment names are literally `CDC_Batard-Montrachet.pdf`), which the +existing stage-02 extractor parses unchanged. This module is the polite, +content-addressed cache in front of that endpoint, shared by stage 01's +last-resort tier and the shadow audit so neither re-hits the API for a +document the other already pulled. + +Attachments are cached OUTSIDE `raw/inao/cahiers/` on purpose: stage 02's +cross-bundle rescue indexes every PDF in that directory, so dropping a few +hundred register cahiers there would silently rewrite `latest_known_pdf` +for unrelated appellations. Only a cahier that actually wins a resolution +is promoted into the cahiers directory, by stage 01. +""" + +from __future__ import annotations + +import hashlib +import json +import sys +import time +from dataclasses import asdict, dataclass +from datetime import datetime, timezone +from pathlib import Path + +import requests + +from .. import eambrosia_register as er + +ROOT = Path(__file__).resolve().parents[3] +REGISTER_DIR = ROOT / "raw" / "inao" / "register" +PDF_DIR = REGISTER_DIR / "cahiers" +MANIFEST_PATH = REGISTER_DIR / "manifest.json" +RESOLVED_PATH = REGISTER_DIR / "resolved.json" +UNRESOLVED_PATH = REGISTER_DIR / "unresolved.json" + +DEFAULT_DELAY = 1.0 + +# Outcomes that are a property of the register's own data rather than of the +# network, so they are worth remembering: re-asking costs a request and the +# answer will not change until the Commission attaches the document. A +# transient `fetch-failed` is deliberately NOT recorded. +STICKY_MISSES = ("no-cahier", "unresolved") + + +@dataclass +class RegisterCahier: + file_number: str + attachment_uri: str + attachment_url: str + attachment_name: str + single_doc_ref: str + filename: str + sha256: str + bytes: int + fetched_at: str + + @property + def path(self) -> Path: + return PDF_DIR / self.filename + + +def load_manifest() -> dict[str, dict]: + if MANIFEST_PATH.exists(): + try: + return json.loads(MANIFEST_PATH.read_text(encoding="utf-8")) + except (ValueError, OSError): + pass + return {} + + +def save_manifest(manifest: dict[str, dict]) -> None: + MANIFEST_PATH.parent.mkdir(parents=True, exist_ok=True) + MANIFEST_PATH.write_text( + json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True), encoding="utf-8" + ) + + +def load_resolved() -> dict[str, dict]: + if RESOLVED_PATH.exists(): + return json.loads(RESOLVED_PATH.read_text(encoding="utf-8")) + return {} + + +_CAHIER_FIELDS = tuple(RegisterCahier.__dataclass_fields__) + + +def _from_entry(entry: dict) -> RegisterCahier: + return RegisterCahier(**{k: entry[k] for k in _CAHIER_FIELDS if k in entry}) + + +def cached(file_number: str, manifest: dict[str, dict]) -> RegisterCahier | None: + """Return the cached cahier for `file_number`, or None when the entry is + absent, records a miss, or its PDF no longer hashes to its own name. + + The PDF is re-hashed rather than merely existence-checked: it is fed to + the stage-02 extractor as if it were the regulator's document, so a + truncated or corrupted cache file has to fail loudly, not quietly.""" + entry = manifest.get(file_number) + if not entry or entry.get("status", "ok") != "ok" or not entry.get("filename"): + return None + rc = _from_entry(entry) + if not rc.path.exists(): + return None + payload = rc.path.read_bytes() + if len(payload) != rc.bytes or hashlib.sha256(payload).hexdigest() != rc.sha256: + print(f"[register] {file_number}: cached {rc.filename} failed its sha256 " + f"— re-fetching", file=sys.stderr) + return None + return rc + + +def recorded_miss(file_number: str, manifest: dict[str, dict]) -> str | None: + """A remembered `STICKY_MISSES` outcome for `file_number`, if any.""" + status = (manifest.get(file_number) or {}).get("status") + return status if status in STICKY_MISSES else None + + +def _record_miss(file_number: str, status: str, manifest: dict[str, dict]) -> None: + if status not in STICKY_MISSES: + return + manifest[file_number] = { + "file_number": file_number, + "status": status, + "checked_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), + } + save_manifest(manifest) + + +def fetch( + file_number: str, + id_map: dict[str, int], + manifest: dict[str, dict], + session: requests.Session | None = None, + refresh: bool = False, + delay: float = DEFAULT_DELAY, +) -> tuple[RegisterCahier | None, str]: + """Fetch `file_number`'s cahier attachment into the cache. + + Returns (cahier, status) where status is one of `cached`, `fetched`, + `no-cahier` (the GI resolves but publishes no `productSpecifications` + attachment), `unresolved` (the GI is absent from the register listing), + `detail-failed` or `fetch-failed`. + + Only the first two misses are remembered — they are properties of the + register's data. A failed request is not: recording it would let one bad + network moment permanently blank the tier.""" + if not refresh: + hit = cached(file_number, manifest) + if hit is not None: + return hit, "cached" + miss = recorded_miss(file_number, manifest) + if miss is not None: + return None, miss + + if file_number not in id_map: + # An empty id map means the listing itself could not be loaded, which + # says nothing about this GI. + if id_map: + _record_miss(file_number, "unresolved", manifest) + return None, "unresolved" + + s = session or requests.Session() + refs = er.attachment_refs(file_number, id_map, session=s) + time.sleep(delay) + if refs is None: + return None, "detail-failed" + uri = refs.get("cahier_uri") + if not uri: + _record_miss(file_number, "no-cahier", manifest) + return None, "no-cahier" + + PDF_DIR.mkdir(parents=True, exist_ok=True) + tmp = PDF_DIR / f".part-{file_number}.pdf" + ok = er.fetch_attachment(uri, tmp, session=s) + time.sleep(delay) + if not ok: + tmp.unlink(missing_ok=True) + return None, "fetch-failed" + + payload = tmp.read_bytes() + digest = hashlib.sha256(payload).hexdigest() + dest = PDF_DIR / f"{digest}.pdf" + if dest.exists(): + tmp.unlink() + else: + tmp.rename(dest) + + rc = RegisterCahier( + file_number=file_number, + attachment_uri=str(uri), + attachment_url=er.ATTACHMENT_URL.format(uri=uri), + attachment_name=refs.get("cahier_name") or "", + single_doc_ref=refs.get("single_doc_ref") or "", + filename=dest.name, + sha256=digest, + bytes=len(payload), + fetched_at=datetime.now(timezone.utc).isoformat(timespec="seconds"), + ) + manifest[file_number] = {"status": "ok", **asdict(rc)} + # Persist per fetch, not per run: the PDFs are content-addressed, so + # without the file_number → sha256 row an interrupted crawl leaves them on + # disk but unreachable, and the next run re-hits the anti-bot-gated + # endpoint for every one of them. + save_manifest(manifest) + return rc, "fetched" diff --git a/scripts/_lib/fr/register_match.py b/scripts/_lib/fr/register_match.py new file mode 100644 index 0000000..2d9e916 --- /dev/null +++ b/scripts/_lib/fr/register_match.py @@ -0,0 +1,334 @@ +"""Resolve a French SIQO appellation to its eAmbrosia register file number. + +France is the one country in the corpus sourced from INAO rather than from +eAmbrosia, so its records carry no `id_eambrosia` join key. The register is +still the cheapest self-service source for a cahier des charges PDF +(`productSpecifications[0].uri` — the INAO cahier itself, not the thinner EU +single document), which makes a name-based resolver the missing link. + +The resolver is deliberately conservative: a wrong bind attaches another +appellation's cahier, which is worse than a gap. Four guards: + +* **Product-type partition.** The candidate pool is split by the register's + `qualityProductType`, keyed off the SIQO `categorie`. "Calvados" is both a + Wine PGI (the Normandy IGP wine) and a Spirit-drink PGI (the eau-de-vie); + without the partition either could claim the other's cahier. A SIQO row with + no categorie at all searches every partition and must come back with one GI; + a categorie the table does not know goes to the curator rather than widening + the pool. +* **No fuzzy matching.** Match keys are the exact normalised alias parts of + the SIQO name (`_lib.fr.naming.candidate_keys`) against the register name + split on its own `/` synonym separator. A key that hits more than one + register GI resolves to nothing. +* **One-to-one.** Two appellations claiming the same file number is a symptom + of a bad key, so the auto-matched claimants go back to the queue (a curator + pin wins, since pinning is the deliberate answer to exactly that question). +* **Live registrations only.** A GI the Commission has struck off cannot be an + appellation's current specification, so a `Cancelled` row is refused rather + than bound. + +Everything the resolver could not settle lands in an unresolved queue, and a +curator pins the residue in `register_overrides.json`. +""" + +from __future__ import annotations + +import json +import re +from collections import defaultdict +from dataclasses import asdict, dataclass +from pathlib import Path + +from .naming import candidate_keys, normalize_name + +OVERRIDES_PATH = Path(__file__).resolve().parent / "register_overrides.json" + +# SIQO `categorie` → the register's `qualityProductType`. The SIQO VITICOLE +# sector bundles wine with vine/fruit spirits and cider, which the register +# files under three different product types. +CATEGORIE_PRODUCT_TYPE: dict[str, str] = { + "Vin tranquille": "Wine", + "Vin de triees successives": "Wine", + "Vin de sélection de grains nobles": "Wine", + "Vin doux naturel": "Wine", + "Vin de liqueur": "Wine", + "Vin mousseux": "Wine", + 'Vin mousseux "Crémant"': "Wine", + "Vin primeur": "Wine", + "Vin jaune": "Wine", + "Vin de paille": "Wine", + "Vin de raisins surmuris": "Wine", + "Vin de vendanges tardives": "Wine", + "Vin sur lie": "Wine", + "Vin pétillant": "Wine", + 'Vin mousseux "Méthode ancestrale"': "Wine", + "Eaux-de-vie de vin": "Spirit drink", + "Eaux-de-vie de marc de raisin": "Spirit drink", + "Eaux-de-vie de fruits": "Spirit drink", + "Eaux-de-vie de cidre et de poiré": "Spirit drink", + "Autres boissons spiritueuses": "Spirit drink", + "Whisky ou Whiskey": "Spirit drink", + "Rhum": "Spirit drink", + "Cidre": "Food", + "Poiré": "Food", +} + +# A GI the Commission has struck off cannot be an appellation's current +# specification, so binding to one is a curator decision, not an automatic +# match. Pending registrations (Applied / Published) are fine — the +# attachment is the submitted cahier. +BINDABLE_STATUSES = ("Registered", "Published", "Applied") + +# The register writes synonym lists with a slash — "Bourg / Côtes de Bourg / +# Bourgeais", "Alsace / Vin d'Alsace" — where SIQO writes " ou ". The slash is +# its ONLY separator: splitting a register name the way a SIQO name is split +# would manufacture keys out of ordinary words, and out of the parenthetical +# region lists some GIs carry ("… du Sud-Ouest (Chalosse, Gascogne, Gers, +# Landes, Périgord, Quercy)" would otherwise offer `gascogne` and `quercy` as +# bindable keys). The parenthetical is dropped for the same reason. +_REGISTER_SPLIT_RE = re.compile(r"\s*/\s*") +_PARENTHETICAL_RE = re.compile(r"\s*\([^)]*\)") + +# An alias fragment shorter than this is a preposition remnant or a bare +# commune word, too generic to bind an appellation on. +_MIN_ALIAS_KEY_LEN = 4 + +_STATUS_RANK = {"Registered": 0, "Published": 1, "Applied": 2, "Cancelled": 3} + + +@dataclass(frozen=True) +class Resolution: + """One appellation → register GI binding.""" + + id_appellation: str + name: str + file_number: str + register_name: str + product_type: str + gi_type: str + status: str + matched_key: str + matched_via: str # "full" | "alias" | "override" + partition: str # "categorie" (SIQO-derived) | "any" | "override" + + +def register_keys(protected_name: str) -> list[str]: + """Match keys for a register `protectedName`: the whole name, then each + `/`-separated synonym, parentheticals removed.""" + name = protected_name or "" + keys: list[str] = [] + full = normalize_name(name) + if full: + keys.append(full) + for chunk in _REGISTER_SPLIT_RE.split(_PARENTHETICAL_RE.sub("", name)): + key = normalize_name(chunk) + if key and key not in keys: + keys.append(key) + return keys + + +def product_type_for(categorie: str) -> str | None: + """Register product type for a SIQO `categorie`; None when unmapped.""" + return CATEGORIE_PRODUCT_TYPE.get((categorie or "").strip()) + + +def fr_rows(rows: list[dict]) -> list[dict]: + """The French GI rows the register shows, any product type. + + `countryId` is a comma-joined list for cross-border GIs — the Basque + cider PDO is `es,fr` — so membership, not equality, is the test.""" + return [ + r for r in rows + if "fr" in (r.get("countryId") or "").split(",") + and r.get("showInRegister") and r.get("fileName") + ] + + +def build_index(rows: list[dict]) -> dict[str, dict[str, list[dict]]]: + """product_type → match key → register rows.""" + index: dict[str, dict[str, list[dict]]] = defaultdict(lambda: defaultdict(list)) + for row in rows: + bucket = index[row.get("qualityProductType") or ""] + for key in register_keys(row.get("protectedName") or ""): + bucket[key].append(row) + return index + + +def load_overrides(path: Path | None = None) -> dict[str, dict]: + """Curator pins: id_appellation → {file_number, name?, note?}. An entry + whose `file_number` is empty is a deliberate "no register GI exists" + marker and suppresses the appellation from the unresolved queue.""" + p = path or OVERRIDES_PATH + if not p.exists(): + return {} + raw = json.loads(p.read_text(encoding="utf-8")) + return {k: v for k, v in raw.items() if not k.startswith("_")} + + +def _pick(candidates: list[dict]) -> list[dict]: + """Prefer the live registration when a name resolves to several + lifecycle rows of the same GI (a cancelled predecessor, a pending + amendment).""" + best = min(_STATUS_RANK.get(c.get("status") or "", 9) for c in candidates) + return [c for c in candidates if _STATUS_RANK.get(c.get("status") or "", 9) == best] + + +def _product_types(candidates: list[dict]) -> set[str]: + return {c.get("qualityProductType") or "" for c in candidates} + + +def _lookup(index: dict[str, dict[str, list[dict]]], pools: list[str], key: str) -> list[dict]: + """Register rows matching `key` across the given product-type pools, + deduplicated by file number.""" + seen: dict[str, dict] = {} + for pool in pools: + for row in index.get(pool, {}).get(key, ()): + seen.setdefault(row["fileName"], row) + return list(seen.values()) + + +def _resolution(id_app: str, name: str, row: dict, key: str, via: str, partition: str) -> Resolution: + return Resolution( + id_appellation=id_app, + name=name, + file_number=row["fileName"], + register_name=row.get("protectedName") or "", + product_type=row.get("qualityProductType") or "", + gi_type=row.get("geographicalIndicatorTypeCode") or "", + status=row.get("status") or "", + matched_key=key, + matched_via=via, + partition=partition, + ) + + +def resolve_all( + appellations: list[dict], + rows: list[dict], + overrides: dict[str, dict] | None = None, +) -> tuple[dict[str, Resolution], list[dict]]: + """Resolve every appellation to a register file number. + + `appellations` are `{id_appellation, name, categorie}` dicts — the parent + rows of `raw/inao/cahiers/manifest.json`. Returns (resolved, unresolved) + where unresolved entries carry a `reason` a curator can act on. + """ + overrides = overrides if overrides is not None else load_overrides() + by_file_number = {r["fileName"]: r for r in rows} + index = build_index(rows) + + resolved: dict[str, Resolution] = {} + unresolved: list[dict] = [] + + for app in appellations: + id_app = str(app["id_appellation"]) + name = app["name"] + categorie = app.get("categorie") or "" + pin = overrides.get(id_app) + + if pin is not None: + file_number = (pin.get("file_number") or "").strip() + if not file_number: + continue # curator-confirmed absent from the register + row = by_file_number.get(file_number) + if row is None: + unresolved.append({ + "id_appellation": id_app, "name": name, "categorie": categorie, + "reason": "override-file-number-not-in-register", + "detail": file_number, + }) + continue + resolved[id_app] = _resolution( + id_app, name, row, file_number, "override", "override") + continue + + # A SIQO row with no `categorie` at all (a handful of AOCs whose + # canonical product carries none) cannot pick a partition, so it + # searches all of them; a *populated* categorie the table does not know + # is a new SIQO value, and widening the pool for it would trade a + # loud gap for a quiet cross-bind, so it goes to the curator instead. + product_type = product_type_for(categorie) + if product_type is None and categorie: + unresolved.append({ + "id_appellation": id_app, "name": name, "categorie": categorie, + "reason": "unmapped-categorie", + "detail": "add it to CATEGORIE_PRODUCT_TYPE in register_match.py", + }) + continue + partition = "categorie" if product_type else "any" + pools = [product_type] if product_type else sorted(index) + + keys = candidate_keys(name) + hit = None + for i, key in enumerate(keys): + if i and len(key) < _MIN_ALIAS_KEY_LEN: + continue + candidates = _lookup(index, pools, key) + if not candidates: + continue + # With no partition to lean on, the product type itself has to be + # unambiguous: status ranking must not be allowed to quietly pick + # the wine over the eau-de-vie of the same name. + best = candidates if len(_product_types(candidates)) > 1 else _pick(candidates) + if len(best) > 1: + unresolved.append({ + "id_appellation": id_app, "name": name, "categorie": categorie, + "reason": "ambiguous", + "detail": f"key {key!r} matches {sorted(c['fileName'] for c in best)}", + }) + hit = "ambiguous" + break + if (best[0].get("status") or "") not in BINDABLE_STATUSES: + unresolved.append({ + "id_appellation": id_app, "name": name, "categorie": categorie, + "reason": "withdrawn-registration", + "detail": f"{best[0]['fileName']} is {best[0].get('status')!r} — " + f"pin it in register_overrides.json to bind it anyway", + }) + hit = "ambiguous" + break + hit = _resolution( + id_app, name, best[0], key, "full" if i == 0 else "alias", partition) + break + + if hit is None: + unresolved.append({ + "id_appellation": id_app, "name": name, "categorie": categorie, + "reason": "no-name-match", + "detail": f"{'/'.join(pools)} partition, keys tried: {keys}", + }) + elif hit != "ambiguous": + resolved[id_app] = hit + + _drop_collisions(resolved, unresolved) + unresolved.sort(key=lambda u: u["name"].lower()) + return resolved, unresolved + + +def _drop_collisions(resolved: dict[str, Resolution], unresolved: list[dict]) -> None: + """Two appellations binding one register GI means at least one bind is + wrong, so the auto-matched claimants go back to the curator. A curator pin + is the deliberate answer to exactly this question, so it survives — losing + it would leave no way to force a binding at all.""" + claims: dict[str, list[str]] = defaultdict(list) + for id_app, res in resolved.items(): + claims[res.file_number].append(id_app) + for file_number, ids in claims.items(): + if len(ids) < 2: + continue + pinned = [i for i in ids if resolved[i].matched_via == "override"] + losers = [i for i in ids if i not in pinned] if len(pinned) == 1 else ids + for id_app in losers: + res = resolved.pop(id_app) + unresolved.append({ + "id_appellation": id_app, "name": res.name, "categorie": "", + "reason": "collision-with-pin" if pinned else "collision", + "detail": f"{file_number} also claimed by {sorted(set(ids) - {id_app})}", + }) + + +def to_json(resolved: dict[str, Resolution]) -> dict[str, dict]: + def _key(kv: tuple[str, Resolution]) -> tuple[int, str]: + return (int(kv[0]) if kv[0].isdigit() else 1 << 30, kv[0]) + + return {k: asdict(v) for k, v in sorted(resolved.items(), key=_key)} diff --git a/scripts/_lib/fr/register_overrides.json b/scripts/_lib/fr/register_overrides.json new file mode 100644 index 0000000..8bfa319 --- /dev/null +++ b/scripts/_lib/fr/register_overrides.json @@ -0,0 +1,15 @@ +{ + "_README": "Curator pins for the FR SIQO appellation -> eAmbrosia register file-number resolver (scripts/01d_resolve_register.py). Keyed by INAO id_appellation. `file_number` is the register `fileName` (PDO-FR-…/PGI-FR-…); set it to \"\" to record a verified absence, which drops the appellation from the unresolved queue instead of leaving it as noise. Look a name up at https://ec.europa.eu/geographical-indications-register/ — the resolver deliberately refuses to fuzzy-match, so every residue lands here.", + "335": { + "name": "Calvados Domfontais", + "file_number": "PGI-FR-01837", + "register_name": "Calvados Domfrontais", + "note": "SIQO spells the denomination 'Domfontais'; the register (and the cahier itself) spell it 'Domfrontais'. Same GI — Calvados Domfrontais AOC, Orne." + }, + "1091": { + "name": "Marc d'Alsace Gewurztraminer", + "file_number": "PGI-FR-01836", + "register_name": "Marc d'Alsace Gewurztraminer", + "note": "Resolves on the all-partition fallback too (SIQO carries no categorie on the manifest row); pinned so the binding is explicit rather than inferred." + } +} diff --git a/scripts/_lib/map_template.py b/scripts/_lib/map_template.py index 44de0b5..8e68566 100644 --- a/scripts/_lib/map_template.py +++ b/scripts/_lib/map_template.py @@ -20,6 +20,7 @@ from pathlib import Path from _lib.content_block import RenderCtx, esc, render_content_block +from _lib.env import carto_basemap_key from _lib.i18n import load_translations from _lib.wikidata import wikidata_url @@ -150,6 +151,7 @@ def build_labels(_: Callable[[str], str]) -> dict[str, str]: "stack_header": _("{n} appellations à ce point"), "stack_cycle_hint": _("Cliquer à nouveau pour parcourir les autres"), "src_cahier": _("Cahier des charges (BO Agri, PDF)"), + "src_cahier_eu_register": _("Cahier des charges (registre GI de l'UE, PDF)"), "src_homologated": _("homologué"), "src_jorf": _("JORF"), "src_show_texte": _("Texte officiel INAO (show_texte)"), @@ -1594,6 +1596,7 @@ def _merged_facet(field: str) -> list[tuple[str, int]]: region_labels_json=json.dumps(region_labels, ensure_ascii=False), country_labels_json=json.dumps(country_labels, ensure_ascii=False), country_flag_emoji_json=json.dumps(_COUNTRY_FLAG_EMOJI, ensure_ascii=False), + carto_key_json=json.dumps(carto_basemap_key()), ) style_body = _STYLE_CSS.replace("{{", "{").replace("}}", "}") diff --git a/scripts/_lib/vendor/README.md b/scripts/_lib/vendor/README.md index e525402..88f7850 100644 --- a/scripts/_lib/vendor/README.md +++ b/scripts/_lib/vendor/README.md @@ -41,10 +41,47 @@ sha384-MinO0mNliZ3vwppuPOUnGa+iq619pfMhLVUXfC4LHwSCvF9H+6P/KO4Q7qBOYV5V maplibr sha384-QfbOCebHNw8pQiPAOd2IFee2v2A5VYZxBk0+JGZ5H+3mfzVIp6zsQNkTsfGJot93 pmtiles-3.2.0.js ``` +## Staged (not yet wired): OpenFreeMap basemap styles + +`openfreemap-positron.json` / `openfreemap-dark.json` are **not shipped** — the +mirror step only copies `.js`/`.css`, so these sit here as build inputs for a +basemap swap that has not happened yet. + +Why they exist: in 2026-08 CARTO retired key-less access to its raster +basemaps (an unkeyed tile comes back with "API KEY REQUIRED" stamped into the +PNG), and told us raster is being retired outright in favour of vector. The map +runs on a free CARTO key today (`CARTO_BASEMAP_KEY`, see +[`app.js`](../assets/app.js)); these two styles are the key-free successor — +[OpenFreeMap](https://openfreemap.org/), OpenStreetMap data on the OpenMapTiles +schema, no account, no quota, and self-hostable if the public instance ever +goes away. + +They were chosen because they drop into the existing architecture rather than +replacing it: both carry the same `sources` / `glyphs` / `sprite`, so the two +layer sets merge into ONE style and the light/dark toggle stays a +`setLayoutProperty` visibility flip — no `setStyle`, which would reorder layers +above the appellation polygons and drop their selection feature-state. + +| File | Source URL | fetched | bytes | +|---|---|---|---| +| `openfreemap-positron.json` | https://tiles.openfreemap.org/styles/positron | 2026-08-30 | 60214 | +| `openfreemap-dark.json` | https://tiles.openfreemap.org/styles/dark | 2026-08-30 | 48790 | + +``` +221ade85123e80467a2500977ba88cf17877291ee049f4d0103cc2da5a7562f4 openfreemap-positron.json +72e31c2853a05a994c575328879fd558788a229278bd04e0faa4ac52bd659410 openfreemap-dark.json +``` + +(Re-serialised with `json.dumps(indent=2)` so the diff of a future refetch is +readable; the tile/glyph/sprite URLs are untouched.) + ## Licences - **maplibre-gl** 4.7.1 — BSD-3-Clause (© MapLibre contributors). - **pmtiles** 3.2.0 — BSD-3-Clause (© Protomaps). +- **OpenFreeMap styles** — style JSON derived from OpenMapTiles/positron + (BSD-3-Clause); the tiles they reference are © OpenStreetMap contributors + (ODbL) served by OpenFreeMap. Attribution is required wherever they render. Both permit redistribution with attribution; their licence text ships inside the distributed `.js` headers. diff --git a/scripts/_lib/vendor/openfreemap-dark.json b/scripts/_lib/vendor/openfreemap-dark.json new file mode 100644 index 0000000..bffb588 --- /dev/null +++ b/scripts/_lib/vendor/openfreemap-dark.json @@ -0,0 +1,2692 @@ +{ + "version": 8, + "sources": { + "ne2_shaded": { + "maxzoom": 6, + "tileSize": 256, + "tiles": [ + "https://tiles.openfreemap.org/natural_earth/ne2sr/{z}/{x}/{y}.png" + ], + "type": "raster" + }, + "openmaptiles": { + "type": "vector", + "url": "https://tiles.openfreemap.org/planet" + } + }, + "sprite": "https://tiles.openfreemap.org/sprites/ofm_f384/ofm", + "glyphs": "https://tiles.openfreemap.org/fonts/{fontstack}/{range}.pbf", + "layers": [ + { + "id": "background", + "type": "background", + "paint": { + "background-color": "rgb(12,12,12)" + } + }, + { + "id": "water", + "type": "fill", + "source": "openmaptiles", + "source-layer": "water", + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "MultiPolygon", + "Polygon" + ], + true, + false + ], + [ + "!=", + [ + "get", + "brunnel" + ], + "tunnel" + ] + ], + "paint": { + "fill-antialias": false, + "fill-color": "rgb(27 ,27 ,29)" + } + }, + { + "id": "landcover_ice_shelf", + "type": "fill", + "source": "openmaptiles", + "source-layer": "landcover", + "maxzoom": 8, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "MultiPolygon", + "Polygon" + ], + true, + false + ], + [ + "==", + [ + "get", + "subclass" + ], + "ice_shelf" + ] + ], + "paint": { + "fill-color": "rgb(12,12,12)", + "fill-opacity": 0.7 + } + }, + { + "id": "landcover_glacier", + "type": "fill", + "source": "openmaptiles", + "source-layer": "landcover", + "maxzoom": 8, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "MultiPolygon", + "Polygon" + ], + true, + false + ], + [ + "==", + [ + "get", + "subclass" + ], + "glacier" + ] + ], + "paint": { + "fill-color": "hsl(0,1%,2%)", + "fill-opacity": [ + "interpolate", + [ + "linear" + ], + [ + "zoom" + ], + 0, + 1, + 8, + 0.5 + ] + } + }, + { + "id": "landuse_residential", + "type": "fill", + "source": "openmaptiles", + "source-layer": "landuse", + "maxzoom": 9, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "MultiPolygon", + "Polygon" + ], + true, + false + ], + [ + "==", + [ + "get", + "class" + ], + "residential" + ] + ], + "paint": { + "fill-color": "hsl(0,2%,5%)", + "fill-opacity": 0.4 + } + }, + { + "id": "landcover_wood", + "type": "fill", + "source": "openmaptiles", + "source-layer": "landcover", + "minzoom": 10, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "MultiPolygon", + "Polygon" + ], + true, + false + ], + [ + "==", + [ + "get", + "class" + ], + "wood" + ] + ], + "paint": { + "fill-color": "rgb(32,32,32)", + "fill-opacity": [ + "interpolate", + [ + "exponential", + 0.3 + ], + [ + "zoom" + ], + 8, + 0, + 10, + 0.8, + 13, + 0.4 + ], + "fill-pattern": "wood-pattern", + "fill-translate": [ + 0, + 0 + ] + } + }, + { + "id": "landuse_park", + "type": "fill", + "source": "openmaptiles", + "source-layer": "landuse", + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "MultiPolygon", + "Polygon" + ], + true, + false + ], + [ + "==", + [ + "get", + "class" + ], + "park" + ] + ], + "paint": { + "fill-color": "rgb(32,32,32)" + } + }, + { + "id": "waterway", + "type": "line", + "source": "openmaptiles", + "source-layer": "waterway", + "filter": [ + "match", + [ + "geometry-type" + ], + [ + "LineString", + "MultiLineString" + ], + true, + false + ], + "paint": { + "line-color": "rgb(27 ,27 ,29)" + } + }, + { + "id": "water_name", + "type": "symbol", + "source": "openmaptiles", + "source-layer": "water_name", + "filter": [ + "match", + [ + "geometry-type" + ], + [ + "LineString", + "MultiLineString" + ], + true, + false + ], + "layout": { + "symbol-placement": "line", + "symbol-spacing": 500, + "text-field": [ + "case", + [ + "has", + "name:nonlatin" + ], + [ + "concat", + [ + "get", + "name:latin" + ], + "\n", + [ + "get", + "name:nonlatin" + ] + ], + [ + "coalesce", + [ + "get", + "name_en" + ], + [ + "get", + "name" + ] + ] + ], + "text-font": [ + "Noto Sans Regular" + ], + "text-rotation-alignment": "map", + "text-size": 12 + }, + "paint": { + "text-color": "hsla(0,0%,0%,0.7)", + "text-halo-color": "hsl(0,0%,27%)" + } + }, + { + "id": "building", + "type": "fill", + "source": "openmaptiles", + "source-layer": "building", + "minzoom": 12, + "filter": [ + "match", + [ + "geometry-type" + ], + [ + "MultiPolygon", + "Polygon" + ], + true, + false + ], + "paint": { + "fill-antialias": true, + "fill-color": "rgb(10,10,10)", + "fill-outline-color": "rgb(27 ,27 ,29)" + } + }, + { + "id": "aeroway-taxiway", + "type": "line", + "source": "openmaptiles", + "source-layer": "aeroway", + "minzoom": 12, + "filter": [ + "match", + [ + "get", + "class" + ], + [ + "taxiway" + ], + true, + false + ], + "layout": { + "line-cap": "round", + "line-join": "round" + }, + "paint": { + "line-color": "#181818", + "line-opacity": 1, + "line-width": [ + "interpolate", + [ + "exponential", + 1.55 + ], + [ + "zoom" + ], + 13, + 1.8, + 20, + 20 + ] + } + }, + { + "id": "aeroway-runway-casing", + "type": "line", + "source": "openmaptiles", + "source-layer": "aeroway", + "minzoom": 11, + "filter": [ + "match", + [ + "get", + "class" + ], + [ + "runway" + ], + true, + false + ], + "layout": { + "line-cap": "round", + "line-join": "round" + }, + "paint": { + "line-color": "rgba(60,60,60,0.8)", + "line-opacity": 1, + "line-width": [ + "interpolate", + [ + "exponential", + 1.5 + ], + [ + "zoom" + ], + 11, + 5, + 17, + 55 + ] + } + }, + { + "id": "aeroway-area", + "type": "fill", + "source": "openmaptiles", + "source-layer": "aeroway", + "minzoom": 4, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "MultiPolygon", + "Polygon" + ], + true, + false + ], + [ + "match", + [ + "get", + "class" + ], + [ + "runway", + "taxiway" + ], + true, + false + ] + ], + "paint": { + "fill-color": "#000", + "fill-opacity": 1 + } + }, + { + "id": "aeroway-runway", + "type": "line", + "source": "openmaptiles", + "source-layer": "aeroway", + "minzoom": 11, + "filter": [ + "all", + [ + "match", + [ + "get", + "class" + ], + [ + "runway" + ], + true, + false + ], + [ + "match", + [ + "geometry-type" + ], + [ + "LineString", + "MultiLineString" + ], + true, + false + ] + ], + "layout": { + "line-cap": "round", + "line-join": "round" + }, + "paint": { + "line-color": "#000", + "line-opacity": 1, + "line-width": [ + "interpolate", + [ + "exponential", + 1.5 + ], + [ + "zoom" + ], + 11, + 4, + 17, + 50 + ] + } + }, + { + "id": "road_area_pier", + "type": "fill", + "source": "openmaptiles", + "source-layer": "transportation", + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "MultiPolygon", + "Polygon" + ], + true, + false + ], + [ + "==", + [ + "get", + "class" + ], + "pier" + ] + ], + "paint": { + "fill-antialias": true, + "fill-color": "rgb(12,12,12)" + } + }, + { + "id": "road_pier", + "type": "line", + "source": "openmaptiles", + "source-layer": "transportation", + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "LineString", + "MultiLineString" + ], + true, + false + ], + [ + "match", + [ + "get", + "class" + ], + [ + "pier" + ], + true, + false + ] + ], + "layout": { + "line-cap": "round", + "line-join": "round" + }, + "paint": { + "line-color": "rgb(12,12,12)", + "line-width": [ + "interpolate", + [ + "exponential", + 1.2 + ], + [ + "zoom" + ], + 15, + 1, + 17, + 4 + ] + } + }, + { + "id": "highway_path", + "type": "line", + "source": "openmaptiles", + "source-layer": "transportation", + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "LineString", + "MultiLineString" + ], + true, + false + ], + [ + "==", + [ + "get", + "class" + ], + "path" + ] + ], + "layout": { + "line-cap": "round", + "line-join": "round" + }, + "paint": { + "line-color": "rgb(27 ,27 ,29)", + "line-dasharray": [ + 1.5, + 1.5 + ], + "line-opacity": 0.9, + "line-width": [ + "interpolate", + [ + "exponential", + 1.2 + ], + [ + "zoom" + ], + 13, + 1, + 20, + 10 + ] + } + }, + { + "id": "highway_minor", + "type": "line", + "source": "openmaptiles", + "source-layer": "transportation", + "minzoom": 8, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "LineString", + "MultiLineString" + ], + true, + false + ], + [ + "match", + [ + "get", + "class" + ], + [ + "minor", + "service", + "track" + ], + true, + false + ] + ], + "layout": { + "line-cap": "round", + "line-join": "round" + }, + "paint": { + "line-color": "#181818", + "line-opacity": 0.9, + "line-width": [ + "interpolate", + [ + "exponential", + 1.55 + ], + [ + "zoom" + ], + 13, + 1.8, + 20, + 20 + ] + } + }, + { + "id": "highway_major_casing", + "type": "line", + "source": "openmaptiles", + "source-layer": "transportation", + "minzoom": 11, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "LineString", + "MultiLineString" + ], + true, + false + ], + [ + "match", + [ + "get", + "class" + ], + [ + "primary", + "secondary", + "tertiary", + "trunk" + ], + true, + false + ] + ], + "layout": { + "line-cap": "butt", + "line-join": "miter" + }, + "paint": { + "line-color": "rgba(60,60,60,0.8)", + "line-dasharray": [ + 12, + 0 + ], + "line-width": [ + "interpolate", + [ + "exponential", + 1.3 + ], + [ + "zoom" + ], + 10, + 3, + 20, + 23 + ] + } + }, + { + "id": "highway_major_inner", + "type": "line", + "source": "openmaptiles", + "source-layer": "transportation", + "minzoom": 11, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "LineString", + "MultiLineString" + ], + true, + false + ], + [ + "match", + [ + "get", + "class" + ], + [ + "primary", + "secondary", + "tertiary", + "trunk" + ], + true, + false + ] + ], + "layout": { + "line-cap": "round", + "line-join": "round" + }, + "paint": { + "line-color": "hsl(0,0%,7%)", + "line-width": [ + "interpolate", + [ + "exponential", + 1.3 + ], + [ + "zoom" + ], + 10, + 2, + 20, + 20 + ] + } + }, + { + "id": "highway_major_subtle", + "type": "line", + "source": "openmaptiles", + "source-layer": "transportation", + "minzoom": 6, + "maxzoom": 11, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "LineString", + "MultiLineString" + ], + true, + false + ], + [ + "match", + [ + "get", + "class" + ], + [ + "primary", + "secondary", + "tertiary", + "trunk" + ], + true, + false + ] + ], + "layout": { + "line-cap": "round", + "line-join": "round" + }, + "paint": { + "line-color": "#2a2a2a", + "line-width": [ + "interpolate", + [ + "linear" + ], + [ + "zoom" + ], + 6, + 0, + 8, + 2 + ] + } + }, + { + "id": "highway_motorway_casing", + "type": "line", + "source": "openmaptiles", + "source-layer": "transportation", + "minzoom": 6, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "LineString", + "MultiLineString" + ], + true, + false + ], + [ + "==", + [ + "get", + "class" + ], + "motorway" + ] + ], + "layout": { + "line-cap": "butt", + "line-join": "miter" + }, + "paint": { + "line-color": "rgba(60,60,60,0.8)", + "line-dasharray": [ + 2, + 0 + ], + "line-opacity": 1, + "line-width": [ + "interpolate", + [ + "exponential", + 1.4 + ], + [ + "zoom" + ], + 5.8, + 0, + 6, + 3, + 20, + 40 + ] + } + }, + { + "id": "highway_motorway_inner", + "type": "line", + "source": "openmaptiles", + "source-layer": "transportation", + "minzoom": 6, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "LineString", + "MultiLineString" + ], + true, + false + ], + [ + "==", + [ + "get", + "class" + ], + "motorway" + ] + ], + "layout": { + "line-cap": "round", + "line-join": "round" + }, + "paint": { + "line-color": [ + "interpolate", + [ + "linear" + ], + [ + "zoom" + ], + 5.8, + "hsla(0,0%,85%,0.53)", + 6, + "#000" + ], + "line-width": [ + "interpolate", + [ + "exponential", + 1.4 + ], + [ + "zoom" + ], + 4, + 2, + 6, + 1.3, + 20, + 30 + ] + } + }, + { + "id": "road_oneway", + "type": "symbol", + "source": "openmaptiles", + "source-layer": "transportation", + "minzoom": 15, + "filter": [ + "==", + [ + "get", + "oneway" + ], + 1 + ], + "layout": { + "icon-image": "oneway", + "icon-padding": 2, + "icon-rotate": 0, + "icon-rotation-alignment": "map", + "icon-size": [ + "interpolate", + [ + "linear" + ], + [ + "zoom" + ], + 15, + 0.5, + 19, + 1 + ], + "symbol-placement": "line", + "symbol-spacing": 200 + }, + "paint": { + "icon-opacity": 0.5 + } + }, + { + "id": "road_oneway_opposite", + "type": "symbol", + "source": "openmaptiles", + "source-layer": "transportation", + "minzoom": 15, + "filter": [ + "==", + [ + "get", + "oneway" + ], + -1 + ], + "layout": { + "icon-image": "oneway", + "icon-padding": 2, + "icon-rotate": 180, + "icon-rotation-alignment": "map", + "icon-size": [ + "interpolate", + [ + "linear" + ], + [ + "zoom" + ], + 15, + 0.5, + 19, + 1 + ], + "symbol-placement": "line", + "symbol-spacing": 200 + }, + "paint": { + "icon-opacity": 0.5 + } + }, + { + "id": "highway_motorway_subtle", + "type": "line", + "source": "openmaptiles", + "source-layer": "transportation", + "maxzoom": 6, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "LineString", + "MultiLineString" + ], + true, + false + ], + [ + "==", + [ + "get", + "class" + ], + "motorway" + ] + ], + "layout": { + "line-cap": "round", + "line-join": "round" + }, + "paint": { + "line-color": "#181818", + "line-width": [ + "interpolate", + [ + "exponential", + 1.4 + ], + [ + "zoom" + ], + 4, + 2, + 6, + 1.3 + ] + } + }, + { + "id": "railway_transit", + "type": "line", + "source": "openmaptiles", + "source-layer": "transportation", + "minzoom": 16, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "LineString", + "MultiLineString" + ], + true, + false + ], + [ + "all", + [ + "==", + [ + "get", + "class" + ], + "transit" + ], + [ + "match", + [ + "get", + "brunnel" + ], + [ + "tunnel" + ], + false, + true + ] + ] + ], + "layout": { + "line-join": "round" + }, + "paint": { + "line-color": "rgb(35,35,35)", + "line-width": 3 + } + }, + { + "id": "railway_transit_dashline", + "type": "line", + "source": "openmaptiles", + "source-layer": "transportation", + "minzoom": 16, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "LineString", + "MultiLineString" + ], + true, + false + ], + [ + "all", + [ + "==", + [ + "get", + "class" + ], + "transit" + ], + [ + "match", + [ + "get", + "brunnel" + ], + [ + "tunnel" + ], + false, + true + ] + ] + ], + "layout": { + "line-join": "round" + }, + "paint": { + "line-color": "rgb(12,12,12)", + "line-dasharray": [ + 3, + 3 + ], + "line-width": 2 + } + }, + { + "id": "railway_minor", + "type": "line", + "source": "openmaptiles", + "source-layer": "transportation", + "minzoom": 16, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "LineString", + "MultiLineString" + ], + true, + false + ], + [ + "all", + [ + "==", + [ + "get", + "class" + ], + "rail" + ], + [ + "has", + "service" + ] + ] + ], + "layout": { + "line-join": "round" + }, + "paint": { + "line-color": "rgb(35,35,35)", + "line-width": 3 + } + }, + { + "id": "railway_minor_dashline", + "type": "line", + "source": "openmaptiles", + "source-layer": "transportation", + "minzoom": 16, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "LineString", + "MultiLineString" + ], + true, + false + ], + [ + "all", + [ + "==", + [ + "get", + "class" + ], + "rail" + ], + [ + "has", + "service" + ] + ] + ], + "layout": { + "line-join": "round" + }, + "paint": { + "line-color": "rgb(12,12,12)", + "line-dasharray": [ + 3, + 3 + ], + "line-width": 2 + } + }, + { + "id": "railway", + "type": "line", + "source": "openmaptiles", + "source-layer": "transportation", + "minzoom": 13, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "LineString", + "MultiLineString" + ], + true, + false + ], + [ + "==", + [ + "get", + "class" + ], + "rail" + ], + [ + "!", + [ + "has", + "service" + ] + ] + ], + "layout": { + "line-join": "round" + }, + "paint": { + "line-color": "rgb(35,35,35)", + "line-width": [ + "interpolate", + [ + "exponential", + 1.3 + ], + [ + "zoom" + ], + 16, + 3, + 20, + 7 + ] + } + }, + { + "id": "railway_dashline", + "type": "line", + "source": "openmaptiles", + "source-layer": "transportation", + "minzoom": 13, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "LineString", + "MultiLineString" + ], + true, + false + ], + [ + "==", + [ + "get", + "class" + ], + "rail" + ], + [ + "!", + [ + "has", + "service" + ] + ] + ], + "layout": { + "line-join": "round" + }, + "paint": { + "line-color": "rgb(12,12,12)", + "line-dasharray": [ + 3, + 3 + ], + "line-width": [ + "interpolate", + [ + "exponential", + 1.3 + ], + [ + "zoom" + ], + 16, + 2, + 20, + 6 + ] + } + }, + { + "id": "highway_name_other", + "type": "symbol", + "source": "openmaptiles", + "source-layer": "transportation_name", + "filter": [ + "all", + [ + "!=", + [ + "get", + "class" + ], + "motorway" + ], + [ + "match", + [ + "geometry-type" + ], + [ + "LineString", + "MultiLineString" + ], + true, + false + ] + ], + "layout": { + "symbol-placement": "line", + "symbol-spacing": 350, + "text-field": [ + "case", + [ + "has", + "name:nonlatin" + ], + [ + "concat", + [ + "get", + "name:latin" + ], + " ", + [ + "get", + "name:nonlatin" + ] + ], + [ + "coalesce", + [ + "get", + "name_en" + ], + [ + "get", + "name" + ] + ] + ], + "text-font": [ + "Noto Sans Regular" + ], + "text-max-angle": 30, + "text-pitch-alignment": "viewport", + "text-rotation-alignment": "map", + "text-size": 10, + "text-transform": "uppercase" + }, + "paint": { + "text-color": "rgba(80, 78, 78, 1)", + "text-halo-blur": 0, + "text-halo-color": "rgba(0, 0, 0, 1)", + "text-halo-width": 1, + "text-translate": [ + 0, + 0 + ] + } + }, + { + "id": "highway_name_motorway", + "type": "symbol", + "source": "openmaptiles", + "source-layer": "transportation_name", + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "LineString", + "MultiLineString" + ], + true, + false + ], + [ + "==", + [ + "get", + "class" + ], + "motorway" + ] + ], + "layout": { + "symbol-placement": "line", + "symbol-spacing": 350, + "text-field": [ + "to-string", + [ + "get", + "ref" + ] + ], + "text-font": [ + "Noto Sans Regular" + ], + "text-pitch-alignment": "viewport", + "text-rotation-alignment": "viewport", + "text-size": 10 + }, + "paint": { + "text-color": "hsl(0,0%,37%)", + "text-translate": [ + 0, + 2 + ] + } + }, + { + "id": "boundary_state", + "type": "line", + "source": "openmaptiles", + "source-layer": "boundary", + "filter": [ + "==", + [ + "get", + "admin_level" + ], + 4 + ], + "layout": { + "line-cap": "round", + "line-join": "round" + }, + "paint": { + "line-blur": 0.4, + "line-color": "hsl(0,0%,21%)", + "line-dasharray": [ + 2, + 2 + ], + "line-opacity": 1, + "line-width": [ + "interpolate", + [ + "exponential", + 1.3 + ], + [ + "zoom" + ], + 3, + 1, + 22, + 15 + ] + } + }, + { + "id": "boundary_country_z0-4", + "type": "line", + "source": "openmaptiles", + "source-layer": "boundary", + "maxzoom": 5, + "filter": [ + "all", + [ + "==", + [ + "get", + "admin_level" + ], + 2 + ], + [ + "!", + [ + "has", + "claimed_by" + ] + ] + ], + "layout": { + "line-cap": "round", + "line-join": "round" + }, + "paint": { + "line-blur": [ + "interpolate", + [ + "linear" + ], + [ + "zoom" + ], + 0, + 0.4, + 22, + 4 + ], + "line-color": "hsl(0,0%,23%)", + "line-opacity": 1, + "line-width": [ + "interpolate", + [ + "exponential", + 1.1 + ], + [ + "zoom" + ], + 3, + 1, + 22, + 20 + ] + } + }, + { + "id": "boundary_country_z5-", + "type": "line", + "source": "openmaptiles", + "source-layer": "boundary", + "minzoom": 5, + "filter": [ + "==", + [ + "get", + "admin_level" + ], + 2 + ], + "layout": { + "line-cap": "round", + "line-join": "round" + }, + "paint": { + "line-blur": [ + "interpolate", + [ + "linear" + ], + [ + "zoom" + ], + 0, + 0.4, + 22, + 4 + ], + "line-color": "hsl(0,0%,23%)", + "line-opacity": 1, + "line-width": [ + "interpolate", + [ + "exponential", + 1.1 + ], + [ + "zoom" + ], + 3, + 1, + 22, + 20 + ] + } + }, + { + "id": "place_other", + "type": "symbol", + "source": "openmaptiles", + "source-layer": "place", + "maxzoom": 14, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "MultiPoint", + "Point" + ], + true, + false + ], + [ + "match", + [ + "get", + "class" + ], + [ + "hamlet", + "isolated_dwelling", + "neighbourhood" + ], + true, + false + ] + ], + "layout": { + "text-anchor": "center", + "text-field": [ + "case", + [ + "has", + "name:nonlatin" + ], + [ + "concat", + [ + "get", + "name:latin" + ], + "\n", + [ + "get", + "name:nonlatin" + ] + ], + [ + "coalesce", + [ + "get", + "name_en" + ], + [ + "get", + "name" + ] + ] + ], + "text-font": [ + "Noto Sans Regular" + ], + "text-justify": "center", + "text-offset": [ + 0.5, + 0 + ], + "text-size": 10, + "text-transform": "uppercase" + }, + "paint": { + "text-color": "rgb(101,101,101)", + "text-halo-blur": 1, + "text-halo-color": "rgba(0,0,0,0.7)", + "text-halo-width": 1 + } + }, + { + "id": "place_suburb", + "type": "symbol", + "source": "openmaptiles", + "source-layer": "place", + "maxzoom": 15, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "MultiPoint", + "Point" + ], + true, + false + ], + [ + "==", + [ + "get", + "class" + ], + "suburb" + ] + ], + "layout": { + "text-anchor": "center", + "text-field": [ + "case", + [ + "has", + "name:nonlatin" + ], + [ + "concat", + [ + "get", + "name:latin" + ], + "\n", + [ + "get", + "name:nonlatin" + ] + ], + [ + "coalesce", + [ + "get", + "name_en" + ], + [ + "get", + "name" + ] + ] + ], + "text-font": [ + "Noto Sans Regular" + ], + "text-justify": "center", + "text-offset": [ + 0.5, + 0 + ], + "text-size": 10, + "text-transform": "uppercase" + }, + "paint": { + "text-color": "rgb(101,101,101)", + "text-halo-blur": 1, + "text-halo-color": "rgba(0,0,0,0.7)", + "text-halo-width": 1 + } + }, + { + "id": "place_village", + "type": "symbol", + "source": "openmaptiles", + "source-layer": "place", + "maxzoom": 14, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "MultiPoint", + "Point" + ], + true, + false + ], + [ + "==", + [ + "get", + "class" + ], + "village" + ] + ], + "layout": { + "icon-size": 0.4, + "text-anchor": "left", + "text-field": [ + "case", + [ + "has", + "name:nonlatin" + ], + [ + "concat", + [ + "get", + "name:latin" + ], + "\n", + [ + "get", + "name:nonlatin" + ] + ], + [ + "coalesce", + [ + "get", + "name_en" + ], + [ + "get", + "name" + ] + ] + ], + "text-font": [ + "Noto Sans Regular" + ], + "text-justify": "left", + "text-offset": [ + 0.5, + 0.2 + ], + "text-size": 10, + "text-transform": "uppercase" + }, + "paint": { + "icon-opacity": 0.7, + "text-color": "rgb(101,101,101)", + "text-halo-blur": 1, + "text-halo-color": "rgba(0,0,0,0.7)", + "text-halo-width": 1 + } + }, + { + "id": "place_town", + "type": "symbol", + "source": "openmaptiles", + "source-layer": "place", + "maxzoom": 15, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "MultiPoint", + "Point" + ], + true, + false + ], + [ + "==", + [ + "get", + "class" + ], + "town" + ] + ], + "layout": { + "icon-image": [ + "step", + [ + "zoom" + ], + "circle-11", + 9, + "" + ], + "icon-size": 0.4, + "text-anchor": [ + "step", + [ + "zoom" + ], + "left", + 8, + "center" + ], + "text-field": [ + "case", + [ + "has", + "name:nonlatin" + ], + [ + "concat", + [ + "get", + "name:latin" + ], + "\n", + [ + "get", + "name:nonlatin" + ] + ], + [ + "coalesce", + [ + "get", + "name_en" + ], + [ + "get", + "name" + ] + ] + ], + "text-font": [ + "Noto Sans Regular" + ], + "text-justify": "left", + "text-offset": [ + 0.5, + 0.2 + ], + "text-size": 10, + "text-transform": "uppercase" + }, + "paint": { + "icon-opacity": 0.7, + "text-color": "rgb(101,101,101)", + "text-halo-blur": 1, + "text-halo-color": "rgba(0,0,0,0.7)", + "text-halo-width": 1 + } + }, + { + "id": "place_city", + "type": "symbol", + "source": "openmaptiles", + "source-layer": "place", + "maxzoom": 14, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "MultiPoint", + "Point" + ], + true, + false + ], + [ + "==", + [ + "get", + "class" + ], + "city" + ], + [ + ">", + [ + "get", + "rank" + ], + 3 + ] + ], + "layout": { + "icon-image": [ + "step", + [ + "zoom" + ], + "circle-11", + 9, + "" + ], + "icon-size": 0.4, + "text-anchor": [ + "step", + [ + "zoom" + ], + "left", + 8, + "center" + ], + "text-field": [ + "case", + [ + "has", + "name:nonlatin" + ], + [ + "concat", + [ + "get", + "name:latin" + ], + "\n", + [ + "get", + "name:nonlatin" + ] + ], + [ + "coalesce", + [ + "get", + "name_en" + ], + [ + "get", + "name" + ] + ] + ], + "text-font": [ + "Noto Sans Regular" + ], + "text-justify": "left", + "text-offset": [ + 0.5, + 0.2 + ], + "text-size": 10, + "text-transform": "uppercase" + }, + "paint": { + "icon-opacity": 0.7, + "text-color": "rgb(101,101,101)", + "text-halo-blur": 1, + "text-halo-color": "rgba(0,0,0,0.7)", + "text-halo-width": 1 + } + }, + { + "id": "place_city_large", + "type": "symbol", + "source": "openmaptiles", + "source-layer": "place", + "maxzoom": 12, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "MultiPoint", + "Point" + ], + true, + false + ], + [ + "<=", + [ + "get", + "rank" + ], + 3 + ], + [ + "==", + [ + "get", + "class" + ], + "city" + ] + ], + "layout": { + "icon-image": [ + "step", + [ + "zoom" + ], + "circle-11", + 9, + "" + ], + "icon-size": 0.4, + "text-anchor": [ + "step", + [ + "zoom" + ], + "left", + 8, + "center" + ], + "text-field": [ + "case", + [ + "has", + "name:nonlatin" + ], + [ + "concat", + [ + "get", + "name:latin" + ], + "\n", + [ + "get", + "name:nonlatin" + ] + ], + [ + "coalesce", + [ + "get", + "name_en" + ], + [ + "get", + "name" + ] + ] + ], + "text-font": [ + "Noto Sans Regular" + ], + "text-justify": "left", + "text-offset": [ + 0.5, + 0.2 + ], + "text-size": 14, + "text-transform": "uppercase" + }, + "paint": { + "icon-opacity": 0.7, + "text-color": "rgb(101,101,101)", + "text-halo-blur": 1, + "text-halo-color": "rgba(0,0,0,0.7)", + "text-halo-width": 1 + } + }, + { + "id": "place_state", + "type": "symbol", + "source": "openmaptiles", + "source-layer": "place", + "maxzoom": 12, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "MultiPoint", + "Point" + ], + true, + false + ], + [ + "==", + [ + "get", + "class" + ], + "state" + ] + ], + "layout": { + "text-field": [ + "case", + [ + "has", + "name:nonlatin" + ], + [ + "concat", + [ + "get", + "name:latin" + ], + "\n", + [ + "get", + "name:nonlatin" + ] + ], + [ + "coalesce", + [ + "get", + "name_en" + ], + [ + "get", + "name" + ] + ] + ], + "text-font": [ + "Noto Sans Regular" + ], + "text-size": 10, + "text-transform": "uppercase" + }, + "paint": { + "text-color": "rgb(101,101,101)", + "text-halo-blur": 1, + "text-halo-color": "rgba(0,0,0,0.7)", + "text-halo-width": 1 + } + }, + { + "id": "place_country_other", + "type": "symbol", + "source": "openmaptiles", + "source-layer": "place", + "maxzoom": 8, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "MultiPoint", + "Point" + ], + true, + false + ], + [ + "==", + [ + "get", + "class" + ], + "country" + ], + [ + "!", + [ + "has", + "iso_a2" + ] + ] + ], + "layout": { + "text-field": [ + "case", + [ + "has", + "name:nonlatin" + ], + [ + "concat", + [ + "get", + "name:latin" + ], + "\n", + [ + "get", + "name:nonlatin" + ] + ], + [ + "coalesce", + [ + "get", + "name_en" + ], + [ + "get", + "name" + ] + ] + ], + "text-font": [ + "Noto Sans Regular" + ], + "text-size": [ + "interpolate", + [ + "linear" + ], + [ + "zoom" + ], + 0, + 9, + 1, + 11 + ], + "text-transform": "uppercase" + }, + "paint": { + "text-color": "rgb(101,101,101)", + "text-halo-color": "rgba(0,0,0,0.7)", + "text-halo-width": 1.4 + } + }, + { + "id": "place_country_minor", + "type": "symbol", + "source": "openmaptiles", + "source-layer": "place", + "maxzoom": 8, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "MultiPoint", + "Point" + ], + true, + false + ], + [ + "==", + [ + "get", + "class" + ], + "country" + ], + [ + ">=", + [ + "get", + "rank" + ], + 2 + ], + [ + "has", + "iso_a2" + ] + ], + "layout": { + "text-field": [ + "case", + [ + "has", + "name:nonlatin" + ], + [ + "concat", + [ + "get", + "name:latin" + ], + "\n", + [ + "get", + "name:nonlatin" + ] + ], + [ + "coalesce", + [ + "get", + "name_en" + ], + [ + "get", + "name" + ] + ] + ], + "text-font": [ + "Noto Sans Regular" + ], + "text-size": [ + "interpolate", + [ + "linear" + ], + [ + "zoom" + ], + 0, + 10, + 6, + 12 + ], + "text-transform": "uppercase" + }, + "paint": { + "text-color": "rgb(101,101,101)", + "text-halo-color": "rgba(0,0,0,0.7)", + "text-halo-width": 1.4 + } + }, + { + "id": "place_country_major", + "type": "symbol", + "source": "openmaptiles", + "source-layer": "place", + "maxzoom": 6, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "MultiPoint", + "Point" + ], + true, + false + ], + [ + "<=", + [ + "get", + "rank" + ], + 1 + ], + [ + "==", + [ + "get", + "class" + ], + "country" + ], + [ + "has", + "iso_a2" + ] + ], + "layout": { + "text-anchor": "center", + "text-field": [ + "case", + [ + "has", + "name:nonlatin" + ], + [ + "concat", + [ + "get", + "name:latin" + ], + "\n", + [ + "get", + "name:nonlatin" + ] + ], + [ + "coalesce", + [ + "get", + "name_en" + ], + [ + "get", + "name" + ] + ] + ], + "text-font": [ + "Noto Sans Regular" + ], + "text-size": [ + "interpolate", + [ + "exponential", + 1.4 + ], + [ + "zoom" + ], + 0, + 10, + 3, + 12, + 4, + 14 + ], + "text-transform": "uppercase" + }, + "paint": { + "text-color": "rgb(101,101,101)", + "text-halo-color": "rgba(0,0,0,0.7)", + "text-halo-width": 1.4 + } + } + ] +} diff --git a/scripts/_lib/vendor/openfreemap-positron.json b/scripts/_lib/vendor/openfreemap-positron.json new file mode 100644 index 0000000..f671bc4 --- /dev/null +++ b/scripts/_lib/vendor/openfreemap-positron.json @@ -0,0 +1,3344 @@ +{ + "version": 8, + "sources": { + "ne2_shaded": { + "maxzoom": 6, + "tileSize": 256, + "tiles": [ + "https://tiles.openfreemap.org/natural_earth/ne2sr/{z}/{x}/{y}.png" + ], + "type": "raster" + }, + "openmaptiles": { + "type": "vector", + "url": "https://tiles.openfreemap.org/planet" + } + }, + "sprite": "https://tiles.openfreemap.org/sprites/ofm_f384/ofm", + "glyphs": "https://tiles.openfreemap.org/fonts/{fontstack}/{range}.pbf", + "layers": [ + { + "id": "background", + "type": "background", + "paint": { + "background-color": "rgb(242,243,240)" + } + }, + { + "id": "park", + "type": "fill", + "source": "openmaptiles", + "source-layer": "park", + "filter": [ + "match", + [ + "geometry-type" + ], + [ + "MultiPolygon", + "Polygon" + ], + true, + false + ], + "paint": { + "fill-color": "rgb(230, 233, 229)" + } + }, + { + "id": "water", + "type": "fill", + "source": "openmaptiles", + "source-layer": "water", + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "MultiPolygon", + "Polygon" + ], + true, + false + ], + [ + "!=", + [ + "get", + "brunnel" + ], + "tunnel" + ] + ], + "paint": { + "fill-antialias": true, + "fill-color": "rgb(194, 200, 202)" + } + }, + { + "id": "landcover_ice_shelf", + "type": "fill", + "source": "openmaptiles", + "source-layer": "landcover", + "maxzoom": 8, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "MultiPolygon", + "Polygon" + ], + true, + false + ], + [ + "==", + [ + "get", + "subclass" + ], + "ice_shelf" + ] + ], + "paint": { + "fill-color": "hsl(0,0%,98%)", + "fill-opacity": 0.7 + } + }, + { + "id": "landcover_glacier", + "type": "fill", + "source": "openmaptiles", + "source-layer": "landcover", + "maxzoom": 8, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "MultiPolygon", + "Polygon" + ], + true, + false + ], + [ + "==", + [ + "get", + "subclass" + ], + "glacier" + ] + ], + "paint": { + "fill-color": "hsl(0,0%,98%)", + "fill-opacity": [ + "interpolate", + [ + "linear" + ], + [ + "zoom" + ], + 0, + 1, + 8, + 0.5 + ] + } + }, + { + "id": "landuse_residential", + "type": "fill", + "source": "openmaptiles", + "source-layer": "landuse", + "maxzoom": 16, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "MultiPolygon", + "Polygon" + ], + true, + false + ], + [ + "==", + [ + "get", + "class" + ], + "residential" + ] + ], + "paint": { + "fill-color": "rgb(234, 234, 230)", + "fill-opacity": [ + "interpolate", + [ + "exponential", + 0.6 + ], + [ + "zoom" + ], + 8, + 0.8, + 9, + 0.6 + ] + } + }, + { + "id": "landcover_wood", + "type": "fill", + "source": "openmaptiles", + "source-layer": "landcover", + "minzoom": 10, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "MultiPolygon", + "Polygon" + ], + true, + false + ], + [ + "==", + [ + "get", + "class" + ], + "wood" + ] + ], + "paint": { + "fill-color": "rgb(220,224,220)", + "fill-opacity": [ + "interpolate", + [ + "linear" + ], + [ + "zoom" + ], + 8, + 0, + 12, + 1 + ] + } + }, + { + "id": "waterway", + "type": "line", + "source": "openmaptiles", + "source-layer": "waterway", + "filter": [ + "match", + [ + "geometry-type" + ], + [ + "LineString", + "MultiLineString" + ], + true, + false + ], + "paint": { + "line-color": "hsl(195,17%,78%)" + } + }, + { + "id": "building", + "type": "fill", + "source": "openmaptiles", + "source-layer": "building", + "minzoom": 12, + "paint": { + "fill-antialias": true, + "fill-color": "rgb(234, 234, 229)", + "fill-outline-color": "rgb(219, 219, 218)" + } + }, + { + "id": "tunnel_motorway_casing", + "type": "line", + "source": "openmaptiles", + "source-layer": "transportation", + "minzoom": 6, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "LineString", + "MultiLineString" + ], + true, + false + ], + [ + "all", + [ + "==", + [ + "get", + "brunnel" + ], + "tunnel" + ], + [ + "==", + [ + "get", + "class" + ], + "motorway" + ] + ] + ], + "layout": { + "line-cap": "butt", + "line-join": "miter" + }, + "paint": { + "line-color": "rgb(213, 213, 213)", + "line-opacity": 1, + "line-width": [ + "interpolate", + [ + "exponential", + 1.4 + ], + [ + "zoom" + ], + 5.8, + 0, + 6, + 3, + 20, + 40 + ] + } + }, + { + "id": "tunnel_motorway_inner", + "type": "line", + "source": "openmaptiles", + "source-layer": "transportation", + "minzoom": 6, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "LineString", + "MultiLineString" + ], + true, + false + ], + [ + "all", + [ + "==", + [ + "get", + "brunnel" + ], + "tunnel" + ], + [ + "==", + [ + "get", + "class" + ], + "motorway" + ] + ] + ], + "layout": { + "line-cap": "round", + "line-join": "round" + }, + "paint": { + "line-color": "rgb(234,234,234)", + "line-width": [ + "interpolate", + [ + "exponential", + 1.4 + ], + [ + "zoom" + ], + 4, + 2, + 6, + 1.3, + 20, + 30 + ] + } + }, + { + "id": "aeroway-taxiway", + "type": "line", + "source": "openmaptiles", + "source-layer": "aeroway", + "minzoom": 12, + "filter": [ + "match", + [ + "get", + "class" + ], + [ + "taxiway" + ], + true, + false + ], + "layout": { + "line-cap": "round", + "line-join": "round" + }, + "paint": { + "line-color": "hsl(0,0%,88%)", + "line-opacity": 1, + "line-width": [ + "interpolate", + [ + "exponential", + 1.55 + ], + [ + "zoom" + ], + 13, + 1.8, + 20, + 20 + ] + } + }, + { + "id": "aeroway-runway-casing", + "type": "line", + "source": "openmaptiles", + "source-layer": "aeroway", + "minzoom": 11, + "filter": [ + "match", + [ + "get", + "class" + ], + [ + "runway" + ], + true, + false + ], + "layout": { + "line-cap": "round", + "line-join": "round" + }, + "paint": { + "line-color": "hsl(0,0%,88%)", + "line-opacity": 1, + "line-width": [ + "interpolate", + [ + "exponential", + 1.5 + ], + [ + "zoom" + ], + 11, + 6, + 17, + 55 + ] + } + }, + { + "id": "aeroway-area", + "type": "fill", + "source": "openmaptiles", + "source-layer": "aeroway", + "minzoom": 4, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "MultiPolygon", + "Polygon" + ], + true, + false + ], + [ + "match", + [ + "get", + "class" + ], + [ + "runway", + "taxiway" + ], + true, + false + ] + ], + "paint": { + "fill-color": "rgba(255, 255, 255, 1)", + "fill-opacity": [ + "interpolate", + [ + "linear" + ], + [ + "zoom" + ], + 13, + 0, + 14, + 1 + ] + } + }, + { + "id": "aeroway-runway", + "type": "line", + "source": "openmaptiles", + "source-layer": "aeroway", + "minzoom": 11, + "filter": [ + "all", + [ + "match", + [ + "get", + "class" + ], + [ + "runway" + ], + true, + false + ], + [ + "match", + [ + "geometry-type" + ], + [ + "LineString", + "MultiLineString" + ], + true, + false + ] + ], + "layout": { + "line-cap": "round", + "line-join": "round" + }, + "paint": { + "line-color": "rgba(255, 255, 255, 1)", + "line-opacity": 1, + "line-width": [ + "interpolate", + [ + "exponential", + 1.5 + ], + [ + "zoom" + ], + 11, + 4, + 17, + 50 + ] + } + }, + { + "id": "road_area_pier", + "type": "fill", + "source": "openmaptiles", + "source-layer": "transportation", + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "MultiPolygon", + "Polygon" + ], + true, + false + ], + [ + "==", + [ + "get", + "class" + ], + "pier" + ] + ], + "paint": { + "fill-antialias": true, + "fill-color": "rgb(242,243,240)" + } + }, + { + "id": "road_pier", + "type": "line", + "source": "openmaptiles", + "source-layer": "transportation", + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "LineString", + "MultiLineString" + ], + true, + false + ], + [ + "match", + [ + "get", + "class" + ], + [ + "pier" + ], + true, + false + ] + ], + "layout": { + "line-cap": "round", + "line-join": "round" + }, + "paint": { + "line-color": "rgb(242,243,240)", + "line-width": [ + "interpolate", + [ + "exponential", + 1.2 + ], + [ + "zoom" + ], + 15, + 1, + 17, + 4 + ] + } + }, + { + "id": "highway_path", + "type": "line", + "source": "openmaptiles", + "source-layer": "transportation", + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "LineString", + "MultiLineString" + ], + true, + false + ], + [ + "==", + [ + "get", + "class" + ], + "path" + ] + ], + "layout": { + "line-cap": "round", + "line-join": "round" + }, + "paint": { + "line-color": "rgb(234, 234, 234)", + "line-opacity": 0.9, + "line-width": [ + "interpolate", + [ + "exponential", + 1.2 + ], + [ + "zoom" + ], + 13, + 1, + 20, + 10 + ] + } + }, + { + "id": "highway_minor", + "type": "line", + "source": "openmaptiles", + "source-layer": "transportation", + "minzoom": 8, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "LineString", + "MultiLineString" + ], + true, + false + ], + [ + "match", + [ + "get", + "class" + ], + [ + "minor", + "service", + "track" + ], + true, + false + ] + ], + "layout": { + "line-cap": "round", + "line-join": "round" + }, + "paint": { + "line-color": "hsl(0,0%,88%)", + "line-opacity": 0.9, + "line-width": [ + "interpolate", + [ + "exponential", + 1.55 + ], + [ + "zoom" + ], + 13, + 1.8, + 20, + 20 + ] + } + }, + { + "id": "highway_major_casing", + "type": "line", + "source": "openmaptiles", + "source-layer": "transportation", + "minzoom": 11, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "LineString", + "MultiLineString" + ], + true, + false + ], + [ + "match", + [ + "get", + "class" + ], + [ + "primary", + "secondary", + "tertiary", + "trunk" + ], + true, + false + ] + ], + "layout": { + "line-cap": "butt", + "line-join": "miter" + }, + "paint": { + "line-color": "rgb(213, 213, 213)", + "line-dasharray": [ + 12, + 0 + ], + "line-width": [ + "interpolate", + [ + "exponential", + 1.3 + ], + [ + "zoom" + ], + 10, + 3, + 20, + 23 + ] + } + }, + { + "id": "highway_major_inner", + "type": "line", + "source": "openmaptiles", + "source-layer": "transportation", + "minzoom": 11, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "LineString", + "MultiLineString" + ], + true, + false + ], + [ + "match", + [ + "get", + "class" + ], + [ + "primary", + "secondary", + "tertiary", + "trunk" + ], + true, + false + ] + ], + "layout": { + "line-cap": "round", + "line-join": "round" + }, + "paint": { + "line-color": "#fff", + "line-width": [ + "interpolate", + [ + "exponential", + 1.3 + ], + [ + "zoom" + ], + 10, + 2, + 20, + 20 + ] + } + }, + { + "id": "highway_major_subtle", + "type": "line", + "source": "openmaptiles", + "source-layer": "transportation", + "maxzoom": 11, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "LineString", + "MultiLineString" + ], + true, + false + ], + [ + "match", + [ + "get", + "class" + ], + [ + "primary", + "secondary", + "tertiary", + "trunk" + ], + true, + false + ] + ], + "layout": { + "line-cap": "round", + "line-join": "round" + }, + "paint": { + "line-color": "hsla(0,0%,85%,0.69)", + "line-width": 2 + } + }, + { + "id": "highway_motorway_casing", + "type": "line", + "source": "openmaptiles", + "source-layer": "transportation", + "minzoom": 6, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "LineString", + "MultiLineString" + ], + true, + false + ], + [ + "all", + [ + "match", + [ + "get", + "brunnel" + ], + [ + "bridge", + "tunnel" + ], + false, + true + ], + [ + "==", + [ + "get", + "class" + ], + "motorway" + ] + ] + ], + "layout": { + "line-cap": "butt", + "line-join": "miter" + }, + "paint": { + "line-color": "rgb(213, 213, 213)", + "line-dasharray": [ + 2, + 0 + ], + "line-opacity": 1, + "line-width": [ + "interpolate", + [ + "exponential", + 1.4 + ], + [ + "zoom" + ], + 5.8, + 0, + 6, + 3, + 20, + 40 + ] + } + }, + { + "id": "highway_motorway_inner", + "type": "line", + "source": "openmaptiles", + "source-layer": "transportation", + "minzoom": 6, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "LineString", + "MultiLineString" + ], + true, + false + ], + [ + "all", + [ + "match", + [ + "get", + "brunnel" + ], + [ + "bridge", + "tunnel" + ], + false, + true + ], + [ + "==", + [ + "get", + "class" + ], + "motorway" + ] + ] + ], + "layout": { + "line-cap": "round", + "line-join": "round" + }, + "paint": { + "line-color": [ + "interpolate", + [ + "linear" + ], + [ + "zoom" + ], + 5.8, + "hsla(0,0%,85%,0.53)", + 6, + "#fff" + ], + "line-width": [ + "interpolate", + [ + "exponential", + 1.4 + ], + [ + "zoom" + ], + 4, + 2, + 6, + 1.3, + 20, + 30 + ] + } + }, + { + "id": "highway_motorway_subtle", + "type": "line", + "source": "openmaptiles", + "source-layer": "transportation", + "maxzoom": 6, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "LineString", + "MultiLineString" + ], + true, + false + ], + [ + "==", + [ + "get", + "class" + ], + "motorway" + ] + ], + "layout": { + "line-cap": "round", + "line-join": "round" + }, + "paint": { + "line-color": "hsla(0,0%,85%,0.53)", + "line-width": [ + "interpolate", + [ + "exponential", + 1.4 + ], + [ + "zoom" + ], + 4, + 2, + 6, + 1.3 + ] + } + }, + { + "id": "railway_transit", + "type": "line", + "source": "openmaptiles", + "source-layer": "transportation", + "minzoom": 16, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "LineString", + "MultiLineString" + ], + true, + false + ], + [ + "all", + [ + "==", + [ + "get", + "class" + ], + "transit" + ], + [ + "match", + [ + "get", + "brunnel" + ], + [ + "tunnel" + ], + false, + true + ] + ] + ], + "layout": { + "line-join": "round" + }, + "paint": { + "line-color": "#dddddd", + "line-width": 3 + } + }, + { + "id": "railway_transit_dashline", + "type": "line", + "source": "openmaptiles", + "source-layer": "transportation", + "minzoom": 16, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "LineString", + "MultiLineString" + ], + true, + false + ], + [ + "all", + [ + "==", + [ + "get", + "class" + ], + "transit" + ], + [ + "match", + [ + "get", + "brunnel" + ], + [ + "tunnel" + ], + false, + true + ] + ] + ], + "layout": { + "line-join": "round" + }, + "paint": { + "line-color": "#fafafa", + "line-dasharray": [ + 3, + 3 + ], + "line-width": 2 + } + }, + { + "id": "railway_service", + "type": "line", + "source": "openmaptiles", + "source-layer": "transportation", + "minzoom": 16, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "LineString", + "MultiLineString" + ], + true, + false + ], + [ + "all", + [ + "==", + [ + "get", + "class" + ], + "rail" + ], + [ + "has", + "service" + ] + ] + ], + "layout": { + "line-join": "round" + }, + "paint": { + "line-color": "#dddddd", + "line-width": 3 + } + }, + { + "id": "railway_service_dashline", + "type": "line", + "source": "openmaptiles", + "source-layer": "transportation", + "minzoom": 16, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "LineString", + "MultiLineString" + ], + true, + false + ], + [ + "==", + [ + "get", + "class" + ], + "rail" + ], + [ + "has", + "service" + ] + ], + "layout": { + "line-join": "round" + }, + "paint": { + "line-color": "#fafafa", + "line-dasharray": [ + 3, + 3 + ], + "line-width": 2 + } + }, + { + "id": "railway", + "type": "line", + "source": "openmaptiles", + "source-layer": "transportation", + "minzoom": 13, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "LineString", + "MultiLineString" + ], + true, + false + ], + [ + "all", + [ + "!", + [ + "has", + "service" + ] + ], + [ + "==", + [ + "get", + "class" + ], + "rail" + ] + ] + ], + "layout": { + "line-join": "round" + }, + "paint": { + "line-color": "#dddddd", + "line-width": [ + "interpolate", + [ + "exponential", + 1.3 + ], + [ + "zoom" + ], + 16, + 3, + 20, + 7 + ] + } + }, + { + "id": "railway_dashline", + "type": "line", + "source": "openmaptiles", + "source-layer": "transportation", + "minzoom": 13, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "LineString", + "MultiLineString" + ], + true, + false + ], + [ + "all", + [ + "!", + [ + "has", + "service" + ] + ], + [ + "==", + [ + "get", + "class" + ], + "rail" + ] + ] + ], + "layout": { + "line-join": "round" + }, + "paint": { + "line-color": "#fafafa", + "line-dasharray": [ + 3, + 3 + ], + "line-width": [ + "interpolate", + [ + "exponential", + 1.3 + ], + [ + "zoom" + ], + 16, + 2, + 20, + 6 + ] + } + }, + { + "id": "highway_motorway_bridge_casing", + "type": "line", + "source": "openmaptiles", + "source-layer": "transportation", + "minzoom": 6, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "LineString", + "MultiLineString" + ], + true, + false + ], + [ + "all", + [ + "==", + [ + "get", + "brunnel" + ], + "bridge" + ], + [ + "==", + [ + "get", + "class" + ], + "motorway" + ] + ] + ], + "layout": { + "line-cap": "butt", + "line-join": "miter" + }, + "paint": { + "line-color": "rgb(213, 213, 213)", + "line-dasharray": [ + 2, + 0 + ], + "line-opacity": 1, + "line-width": [ + "interpolate", + [ + "exponential", + 1.4 + ], + [ + "zoom" + ], + 5.8, + 0, + 6, + 5, + 20, + 45 + ] + } + }, + { + "id": "highway_motorway_bridge_inner", + "type": "line", + "source": "openmaptiles", + "source-layer": "transportation", + "minzoom": 6, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "LineString", + "MultiLineString" + ], + true, + false + ], + [ + "all", + [ + "==", + [ + "get", + "brunnel" + ], + "bridge" + ], + [ + "==", + [ + "get", + "class" + ], + "motorway" + ] + ] + ], + "layout": { + "line-cap": "round", + "line-join": "round" + }, + "paint": { + "line-color": [ + "interpolate", + [ + "linear" + ], + [ + "zoom" + ], + 5.8, + "hsla(0,0%,85%,0.53)", + 6, + "#fff" + ], + "line-width": [ + "interpolate", + [ + "exponential", + 1.4 + ], + [ + "zoom" + ], + 4, + 2, + 6, + 1.3, + 20, + 30 + ] + } + }, + { + "id": "boundary_3", + "type": "line", + "source": "openmaptiles", + "source-layer": "boundary", + "minzoom": 8, + "filter": [ + "all", + [ + ">=", + [ + "get", + "admin_level" + ], + 3 + ], + [ + "<=", + [ + "get", + "admin_level" + ], + 6 + ], + [ + "!=", + [ + "get", + "maritime" + ], + 1 + ], + [ + "!=", + [ + "get", + "disputed" + ], + 1 + ], + [ + "!", + [ + "has", + "claimed_by" + ] + ] + ], + "paint": { + "line-color": "hsl(0,0%,70%)", + "line-dasharray": [ + 1, + 1 + ], + "line-width": [ + "interpolate", + [ + "linear", + 1 + ], + [ + "zoom" + ], + 7, + 1, + 11, + 2 + ] + } + }, + { + "id": "boundary_2", + "type": "line", + "source": "openmaptiles", + "source-layer": "boundary", + "filter": [ + "all", + [ + "==", + [ + "get", + "admin_level" + ], + 2 + ], + [ + "!=", + [ + "get", + "maritime" + ], + 1 + ], + [ + "!=", + [ + "get", + "disputed" + ], + 1 + ], + [ + "!", + [ + "has", + "claimed_by" + ] + ] + ], + "layout": { + "line-cap": "round", + "line-join": "round" + }, + "paint": { + "line-color": "hsl(0,0%,70%)", + "line-opacity": [ + "interpolate", + [ + "linear" + ], + [ + "zoom" + ], + 0, + 0.4, + 4, + 1 + ], + "line-width": [ + "interpolate", + [ + "linear" + ], + [ + "zoom" + ], + 3, + 1, + 5, + 1.2, + 12, + 3 + ] + } + }, + { + "id": "boundary_disputed", + "type": "line", + "source": "openmaptiles", + "source-layer": "boundary", + "filter": [ + "all", + [ + "!=", + [ + "get", + "maritime" + ], + 1 + ], + [ + "==", + [ + "get", + "disputed" + ], + 1 + ] + ], + "paint": { + "line-color": "hsl(0,0%,70%)", + "line-dasharray": [ + 1, + 2 + ], + "line-width": [ + "interpolate", + [ + "linear" + ], + [ + "zoom" + ], + 3, + 1, + 5, + 1.2, + 12, + 3 + ] + } + }, + { + "id": "waterway_line_label", + "type": "symbol", + "source": "openmaptiles", + "source-layer": "waterway", + "minzoom": 10, + "filter": [ + "match", + [ + "geometry-type" + ], + [ + "LineString", + "MultiLineString" + ], + true, + false + ], + "layout": { + "symbol-placement": "line", + "symbol-spacing": 350, + "text-field": [ + "case", + [ + "has", + "name:nonlatin" + ], + [ + "concat", + [ + "get", + "name:latin" + ], + " ", + [ + "get", + "name:nonlatin" + ] + ], + [ + "coalesce", + [ + "get", + "name_en" + ], + [ + "get", + "name" + ] + ] + ], + "text-font": [ + "Noto Sans Italic" + ], + "text-letter-spacing": 0.2, + "text-max-width": 5, + "text-size": 14 + }, + "paint": { + "text-color": "hsl(0,0%,66%)", + "text-halo-color": "rgba(255,255,255,0.7)", + "text-halo-width": 1.5 + } + }, + { + "id": "water_name_point_label", + "type": "symbol", + "source": "openmaptiles", + "source-layer": "water_name", + "filter": [ + "match", + [ + "geometry-type" + ], + [ + "MultiPoint", + "Point" + ], + true, + false + ], + "layout": { + "text-field": [ + "case", + [ + "has", + "name:nonlatin" + ], + [ + "concat", + [ + "get", + "name:latin" + ], + "\n", + [ + "get", + "name:nonlatin" + ] + ], + [ + "coalesce", + [ + "get", + "name_en" + ], + [ + "get", + "name" + ] + ] + ], + "text-font": [ + "Noto Sans Italic" + ], + "text-letter-spacing": 0.2, + "text-max-width": 5, + "text-size": [ + "interpolate", + [ + "linear" + ], + [ + "zoom" + ], + 0, + 10, + 8, + 14 + ] + }, + "paint": { + "text-color": "#495e91", + "text-halo-color": "rgba(255,255,255,0.7)", + "text-halo-width": 1.5 + } + }, + { + "id": "water_name_line_label", + "type": "symbol", + "source": "openmaptiles", + "source-layer": "water_name", + "filter": [ + "match", + [ + "geometry-type" + ], + [ + "LineString", + "MultiLineString" + ], + true, + false + ], + "layout": { + "symbol-placement": "line", + "symbol-spacing": 350, + "text-field": [ + "case", + [ + "has", + "name:nonlatin" + ], + [ + "concat", + [ + "get", + "name:latin" + ], + " ", + [ + "get", + "name:nonlatin" + ] + ], + [ + "coalesce", + [ + "get", + "name_en" + ], + [ + "get", + "name" + ] + ] + ], + "text-font": [ + "Noto Sans Italic" + ], + "text-letter-spacing": 0.2, + "text-max-width": 5, + "text-size": 14 + }, + "paint": { + "text-color": "#495e91", + "text-halo-color": "rgba(255,255,255,0.7)", + "text-halo-width": 1.5 + } + }, + { + "id": "highway-name-path", + "type": "symbol", + "source": "openmaptiles", + "source-layer": "transportation_name", + "minzoom": 15.5, + "filter": [ + "==", + [ + "get", + "class" + ], + "path" + ], + "layout": { + "symbol-placement": "line", + "text-field": [ + "case", + [ + "has", + "name:nonlatin" + ], + [ + "concat", + [ + "get", + "name:latin" + ], + " ", + [ + "get", + "name:nonlatin" + ] + ], + [ + "coalesce", + [ + "get", + "name_en" + ], + [ + "get", + "name" + ] + ] + ], + "text-font": [ + "Noto Sans Regular" + ], + "text-rotation-alignment": "map", + "text-size": [ + "interpolate", + [ + "linear" + ], + [ + "zoom" + ], + 13, + 12, + 14, + 13 + ] + }, + "paint": { + "text-color": "hsl(30,0%,62%)", + "text-halo-color": "#f8f4f0", + "text-halo-width": 0.5 + } + }, + { + "id": "highway-name-minor", + "type": "symbol", + "source": "openmaptiles", + "source-layer": "transportation_name", + "minzoom": 15, + "filter": [ + "all", + [ + "match", + [ + "geometry-type" + ], + [ + "LineString", + "MultiLineString" + ], + true, + false + ], + [ + "match", + [ + "get", + "class" + ], + [ + "minor", + "service", + "track" + ], + true, + false + ] + ], + "layout": { + "symbol-placement": "line", + "text-field": [ + "case", + [ + "has", + "name:nonlatin" + ], + [ + "concat", + [ + "get", + "name:latin" + ], + " ", + [ + "get", + "name:nonlatin" + ] + ], + [ + "coalesce", + [ + "get", + "name_en" + ], + [ + "get", + "name" + ] + ] + ], + "text-font": [ + "Noto Sans Regular" + ], + "text-rotation-alignment": "map", + "text-size": [ + "interpolate", + [ + "linear" + ], + [ + "zoom" + ], + 13, + 12, + 14, + 13 + ] + }, + "paint": { + "text-color": "#666", + "text-halo-blur": 0.5, + "text-halo-width": 1 + } + }, + { + "id": "highway-name-major", + "type": "symbol", + "source": "openmaptiles", + "source-layer": "transportation_name", + "minzoom": 12.2, + "filter": [ + "match", + [ + "get", + "class" + ], + [ + "primary", + "secondary", + "tertiary", + "trunk" + ], + true, + false + ], + "layout": { + "symbol-placement": "line", + "text-field": [ + "case", + [ + "has", + "name:nonlatin" + ], + [ + "concat", + [ + "get", + "name:latin" + ], + " ", + [ + "get", + "name:nonlatin" + ] + ], + [ + "coalesce", + [ + "get", + "name_en" + ], + [ + "get", + "name" + ] + ] + ], + "text-font": [ + "Noto Sans Regular" + ], + "text-rotation-alignment": "map", + "text-size": [ + "interpolate", + [ + "linear" + ], + [ + "zoom" + ], + 13, + 12, + 14, + 13 + ] + }, + "paint": { + "text-color": "#666", + "text-halo-blur": 0.5, + "text-halo-width": 1 + } + }, + { + "id": "highway-shield-non-us", + "type": "symbol", + "source": "openmaptiles", + "source-layer": "transportation_name", + "minzoom": 11, + "filter": [ + "all", + [ + "<=", + [ + "get", + "ref_length" + ], + 6 + ], + [ + "match", + [ + "geometry-type" + ], + [ + "LineString", + "MultiLineString" + ], + true, + false + ], + [ + "match", + [ + "get", + "network" + ], + [ + "us-highway", + "us-interstate", + "us-state" + ], + false, + true + ] + ], + "layout": { + "icon-image": [ + "concat", + "road_", + [ + "get", + "ref_length" + ] + ], + "icon-rotation-alignment": "viewport", + "icon-size": 1, + "symbol-placement": [ + "step", + [ + "zoom" + ], + "point", + 11, + "line" + ], + "symbol-spacing": 200, + "text-field": [ + "to-string", + [ + "get", + "ref" + ] + ], + "text-font": [ + "Noto Sans Regular" + ], + "text-rotation-alignment": "viewport", + "text-size": 10 + } + }, + { + "id": "highway-shield-us-interstate", + "type": "symbol", + "source": "openmaptiles", + "source-layer": "transportation_name", + "minzoom": 11, + "filter": [ + "all", + [ + "<=", + [ + "get", + "ref_length" + ], + 6 + ], + [ + "match", + [ + "geometry-type" + ], + [ + "LineString", + "MultiLineString" + ], + true, + false + ], + [ + "match", + [ + "get", + "network" + ], + [ + "us-interstate" + ], + true, + false + ] + ], + "layout": { + "icon-image": [ + "concat", + [ + "get", + "network" + ], + "_", + [ + "get", + "ref_length" + ] + ], + "icon-rotation-alignment": "viewport", + "icon-size": 1, + "symbol-placement": [ + "step", + [ + "zoom" + ], + "point", + 7, + "line", + 8, + "line" + ], + "symbol-spacing": 200, + "text-field": [ + "to-string", + [ + "get", + "ref" + ] + ], + "text-font": [ + "Noto Sans Regular" + ], + "text-rotation-alignment": "viewport", + "text-size": 10 + } + }, + { + "id": "road_shield_us", + "type": "symbol", + "source": "openmaptiles", + "source-layer": "transportation_name", + "minzoom": 12, + "filter": [ + "all", + [ + "<=", + [ + "get", + "ref_length" + ], + 6 + ], + [ + "match", + [ + "geometry-type" + ], + [ + "LineString", + "MultiLineString" + ], + true, + false + ], + [ + "match", + [ + "get", + "network" + ], + [ + "us-highway", + "us-state" + ], + true, + false + ] + ], + "layout": { + "icon-image": [ + "concat", + [ + "get", + "network" + ], + "_", + [ + "get", + "ref_length" + ] + ], + "icon-rotation-alignment": "viewport", + "icon-size": 1, + "symbol-placement": [ + "step", + [ + "zoom" + ], + "point", + 11, + "line" + ], + "symbol-spacing": 200, + "text-field": [ + "to-string", + [ + "get", + "ref" + ] + ], + "text-font": [ + "Noto Sans Regular" + ], + "text-rotation-alignment": "viewport", + "text-size": 10 + } + }, + { + "id": "airport", + "type": "symbol", + "source": "openmaptiles", + "source-layer": "aerodrome_label", + "minzoom": 11, + "filter": [ + "all", + [ + "has", + "iata" + ] + ], + "layout": { + "icon-image": "airport_11", + "icon-size": 1, + "text-anchor": "top", + "text-field": [ + "case", + [ + "has", + "name:nonlatin" + ], + [ + "concat", + [ + "get", + "name:latin" + ], + "\n", + [ + "get", + "name:nonlatin" + ] + ], + [ + "coalesce", + [ + "get", + "name_en" + ], + [ + "get", + "name" + ] + ] + ], + "text-font": [ + "Noto Sans Regular" + ], + "text-max-width": 9, + "text-offset": [ + 0, + 0.6 + ], + "text-optional": true, + "text-padding": 2, + "text-size": 12 + }, + "paint": { + "text-color": "#666", + "text-halo-blur": 0.5, + "text-halo-color": "#ffffff", + "text-halo-width": 1 + } + }, + { + "id": "label_other", + "type": "symbol", + "source": "openmaptiles", + "source-layer": "place", + "minzoom": 8, + "filter": [ + "match", + [ + "get", + "class" + ], + [ + "city", + "continent", + "country", + "state", + "town", + "village" + ], + false, + true + ], + "layout": { + "text-field": [ + "case", + [ + "has", + "name:nonlatin" + ], + [ + "concat", + [ + "get", + "name:latin" + ], + "\n", + [ + "get", + "name:nonlatin" + ] + ], + [ + "coalesce", + [ + "get", + "name_en" + ], + [ + "get", + "name" + ] + ] + ], + "text-font": [ + "Noto Sans Italic" + ], + "text-letter-spacing": 0.1, + "text-max-width": 9, + "text-size": [ + "interpolate", + [ + "linear" + ], + [ + "zoom" + ], + 8, + 9, + 12, + 10 + ], + "text-transform": "uppercase" + }, + "paint": { + "text-color": "#333", + "text-halo-blur": 1, + "text-halo-color": "#fff", + "text-halo-width": 1 + } + }, + { + "id": "label_village", + "type": "symbol", + "source": "openmaptiles", + "source-layer": "place", + "minzoom": 9, + "filter": [ + "==", + [ + "get", + "class" + ], + "village" + ], + "layout": { + "icon-allow-overlap": true, + "icon-image": [ + "step", + [ + "zoom" + ], + "circle_11_black", + 10, + "" + ], + "icon-optional": false, + "icon-size": 0.2, + "text-anchor": "bottom", + "text-field": [ + "case", + [ + "has", + "name:nonlatin" + ], + [ + "concat", + [ + "get", + "name:latin" + ], + "\n", + [ + "get", + "name:nonlatin" + ] + ], + [ + "coalesce", + [ + "get", + "name_en" + ], + [ + "get", + "name" + ] + ] + ], + "text-font": [ + "Noto Sans Regular" + ], + "text-max-width": 8, + "text-size": [ + "interpolate", + [ + "exponential", + 1.2 + ], + [ + "zoom" + ], + 7, + 10, + 11, + 12 + ] + }, + "paint": { + "text-color": "#000", + "text-halo-blur": 1, + "text-halo-color": "#fff", + "text-halo-width": 1 + } + }, + { + "id": "label_town", + "type": "symbol", + "source": "openmaptiles", + "source-layer": "place", + "minzoom": 6, + "filter": [ + "==", + [ + "get", + "class" + ], + "town" + ], + "layout": { + "icon-allow-overlap": true, + "icon-image": [ + "step", + [ + "zoom" + ], + "circle_11_black", + 10, + "" + ], + "icon-optional": false, + "icon-size": 0.2, + "text-anchor": "bottom", + "text-field": [ + "case", + [ + "has", + "name:nonlatin" + ], + [ + "concat", + [ + "get", + "name:latin" + ], + "\n", + [ + "get", + "name:nonlatin" + ] + ], + [ + "coalesce", + [ + "get", + "name_en" + ], + [ + "get", + "name" + ] + ] + ], + "text-font": [ + "Noto Sans Regular" + ], + "text-max-width": 8, + "text-size": [ + "interpolate", + [ + "exponential", + 1.2 + ], + [ + "zoom" + ], + 7, + 12, + 11, + 14 + ] + }, + "paint": { + "text-color": "#000", + "text-halo-blur": 1, + "text-halo-color": "#fff", + "text-halo-width": 1 + } + }, + { + "id": "label_state", + "type": "symbol", + "source": "openmaptiles", + "source-layer": "place", + "minzoom": 5, + "maxzoom": 8, + "filter": [ + "==", + [ + "get", + "class" + ], + "state" + ], + "layout": { + "text-field": [ + "case", + [ + "has", + "name:nonlatin" + ], + [ + "concat", + [ + "get", + "name:latin" + ], + "\n", + [ + "get", + "name:nonlatin" + ] + ], + [ + "coalesce", + [ + "get", + "name_en" + ], + [ + "get", + "name" + ] + ] + ], + "text-font": [ + "Noto Sans Italic" + ], + "text-letter-spacing": 0.2, + "text-max-width": 9, + "text-size": [ + "interpolate", + [ + "linear" + ], + [ + "zoom" + ], + 5, + 10, + 8, + 14 + ], + "text-transform": "uppercase" + }, + "paint": { + "text-color": "#333", + "text-halo-blur": 1, + "text-halo-color": "#fff", + "text-halo-width": 1 + } + }, + { + "id": "label_city", + "type": "symbol", + "source": "openmaptiles", + "source-layer": "place", + "minzoom": 3, + "filter": [ + "all", + [ + "==", + [ + "get", + "class" + ], + "city" + ], + [ + "!=", + [ + "get", + "capital" + ], + 2 + ] + ], + "layout": { + "icon-allow-overlap": true, + "icon-image": [ + "step", + [ + "zoom" + ], + "circle_11_black", + 9, + "" + ], + "icon-optional": false, + "icon-size": 0.4, + "text-anchor": "bottom", + "text-field": [ + "case", + [ + "has", + "name:nonlatin" + ], + [ + "concat", + [ + "get", + "name:latin" + ], + "\n", + [ + "get", + "name:nonlatin" + ] + ], + [ + "coalesce", + [ + "get", + "name_en" + ], + [ + "get", + "name" + ] + ] + ], + "text-font": [ + "Noto Sans Regular" + ], + "text-max-width": 8, + "text-offset": [ + 0, + -0.1 + ], + "text-size": [ + "interpolate", + [ + "exponential", + 1.2 + ], + [ + "zoom" + ], + 4, + 11, + 7, + 13, + 11, + 18 + ] + }, + "paint": { + "text-color": "#000", + "text-halo-blur": 1, + "text-halo-color": "#fff", + "text-halo-width": 1 + } + }, + { + "id": "label_city_capital", + "type": "symbol", + "source": "openmaptiles", + "source-layer": "place", + "minzoom": 3, + "filter": [ + "all", + [ + "==", + [ + "get", + "class" + ], + "city" + ], + [ + "==", + [ + "get", + "capital" + ], + 2 + ] + ], + "layout": { + "icon-allow-overlap": true, + "icon-image": [ + "step", + [ + "zoom" + ], + "circle_11_black", + 9, + "" + ], + "icon-optional": false, + "icon-size": 0.5, + "text-anchor": "bottom", + "text-field": [ + "case", + [ + "has", + "name:nonlatin" + ], + [ + "concat", + [ + "get", + "name:latin" + ], + "\n", + [ + "get", + "name:nonlatin" + ] + ], + [ + "coalesce", + [ + "get", + "name_en" + ], + [ + "get", + "name" + ] + ] + ], + "text-font": [ + "Noto Sans Bold" + ], + "text-max-width": 8, + "text-offset": [ + 0, + -0.2 + ], + "text-size": [ + "interpolate", + [ + "exponential", + 1.2 + ], + [ + "zoom" + ], + 4, + 12, + 7, + 14, + 11, + 20 + ] + }, + "paint": { + "text-color": "#000", + "text-halo-blur": 1, + "text-halo-color": "#fff", + "text-halo-width": 1 + } + }, + { + "id": "label_country_3", + "type": "symbol", + "source": "openmaptiles", + "source-layer": "place", + "minzoom": 2, + "maxzoom": 9, + "filter": [ + "all", + [ + "==", + [ + "get", + "class" + ], + "country" + ], + [ + ">=", + [ + "get", + "rank" + ], + 3 + ] + ], + "layout": { + "text-field": [ + "case", + [ + "has", + "name:nonlatin" + ], + [ + "concat", + [ + "get", + "name:latin" + ], + "\n", + [ + "get", + "name:nonlatin" + ] + ], + [ + "coalesce", + [ + "get", + "name_en" + ], + [ + "get", + "name" + ] + ] + ], + "text-font": [ + "Noto Sans Bold" + ], + "text-max-width": 6.25, + "text-size": [ + "interpolate", + [ + "linear" + ], + [ + "zoom" + ], + 3, + 9, + 7, + 17 + ] + }, + "paint": { + "text-color": "#000", + "text-halo-blur": 1, + "text-halo-color": "#fff", + "text-halo-width": 1 + } + }, + { + "id": "label_country_2", + "type": "symbol", + "source": "openmaptiles", + "source-layer": "place", + "maxzoom": 9, + "filter": [ + "all", + [ + "==", + [ + "get", + "class" + ], + "country" + ], + [ + "==", + [ + "get", + "rank" + ], + 2 + ] + ], + "layout": { + "text-field": [ + "case", + [ + "has", + "name:nonlatin" + ], + [ + "concat", + [ + "get", + "name:latin" + ], + "\n", + [ + "get", + "name:nonlatin" + ] + ], + [ + "coalesce", + [ + "get", + "name_en" + ], + [ + "get", + "name" + ] + ] + ], + "text-font": [ + "Noto Sans Bold" + ], + "text-max-width": 6.25, + "text-size": [ + "interpolate", + [ + "linear" + ], + [ + "zoom" + ], + 2, + 9, + 5, + 17 + ] + }, + "paint": { + "text-color": "#000", + "text-halo-blur": 1, + "text-halo-color": "#fff", + "text-halo-width": 1 + } + }, + { + "id": "label_country_1", + "type": "symbol", + "source": "openmaptiles", + "source-layer": "place", + "maxzoom": 9, + "filter": [ + "all", + [ + "==", + [ + "get", + "class" + ], + "country" + ], + [ + "==", + [ + "get", + "rank" + ], + 1 + ] + ], + "layout": { + "text-field": [ + "case", + [ + "has", + "name:nonlatin" + ], + [ + "concat", + [ + "get", + "name:latin" + ], + "\n", + [ + "get", + "name:nonlatin" + ] + ], + [ + "coalesce", + [ + "get", + "name_en" + ], + [ + "get", + "name" + ] + ] + ], + "text-font": [ + "Noto Sans Bold" + ], + "text-max-width": 6.25, + "text-size": [ + "interpolate", + [ + "linear" + ], + [ + "zoom" + ], + 1, + 9, + 4, + 17 + ] + }, + "paint": { + "text-color": "#000", + "text-halo-blur": 1, + "text-halo-color": "#fff", + "text-halo-width": 1 + } + } + ] +} diff --git a/scripts/audit_fr_register_shadow.py b/scripts/audit_fr_register_shadow.py new file mode 100644 index 0000000..ff38e77 --- /dev/null +++ b/scripts/audit_fr_register_shadow.py @@ -0,0 +1,489 @@ +"""Shadow-compare the eAmbrosia register cahier against the in-build FR record. + +Read-only with respect to the build: fetches each appellation's register +cahier into `raw/inao/register/cahiers/`, extracts it with the *unchanged* +stage-02 parser in memory, and reports whether the result matches the record +already in `raw/inao/cahier-extracted/`. Nothing under `raw/inao/cahiers/` +or `raw/inao/cahier-extracted/` is written, so the comparison can be shipped +before any behaviour change. + +Three columns per appellation: + +* **build** — the record on disk today (BO Agri / curator URL / OCR mirror). +* **self** — re-extracting the build's own PDF right now. This is the + determinism control: if `self` already differs from `build`, a `register` + difference on the same appellation says nothing about the register. +* **register** — extracting the register attachment. + +`lien_au_terroir` is the headline comparison (it is the largest verbatim +block the wiki renders), with section / commune / grape counts and the +homologation date alongside so a *vintage* difference is distinguishable +from a *parse* difference. + + .venv/bin/python scripts/audit_fr_register_shadow.py # all 466 + .venv/bin/python scripts/audit_fr_register_shadow.py --only Chablis +""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib +import json +import re +import signal +import subprocess +import sys +from collections import Counter +from datetime import datetime, timezone +from pathlib import Path + +import requests +from tqdm import tqdm + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / "scripts")) + +from _lib import eambrosia_register as er # noqa: E402 +from _lib.fr import register_cahier as rc # noqa: E402 + +m02 = importlib.import_module("02_extract_cahiers") + +CAHIERS = ROOT / "raw" / "inao" / "cahiers" +MANIFEST_PATH = CAHIERS / "manifest.json" +EXTRACTED = ROOT / "raw" / "inao" / "cahier-extracted" +INDEX_PATH = EXTRACTED / "_index.json" +TEXT_CACHE = rc.REGISTER_DIR / ".text" +REPORT_JSON = rc.REGISTER_DIR / "shadow-report.json" +REPORT_MD = rc.REGISTER_DIR / "shadow-report.md" + + +def _sha(s: str) -> str: + return hashlib.sha256(s.encode("utf-8")).hexdigest() + + +_WS_RE = re.compile(r"\s+") + + +def _words(s: str) -> Counter: + return Counter(_WS_RE.sub(" ", s).strip().split(" ")) + + +def _similarity(a: str, b: str) -> float: + """Word-multiset Dice coefficient — linear, so it survives the 340 KB + Alsace-grand-cru bundle, and sensitive enough to separate a cosmetic + delta (a guillemet space, `mélant` → `mêlant`) from a rewritten section.""" + wa, wb = _words(a), _words(b) + total = sum(wa.values()) + sum(wb.values()) + if not total: + return 1.0 + return 2 * sum((wa & wb).values()) / total + + +def _norm_sha(s: str) -> str: + """Hash with all whitespace collapsed. Two PDFs of the same cahier laid + out at different column widths wrap differently, so a byte comparison + over-reports; this separates 'same text, different wrapping' from a real + textual (vintage) difference.""" + return _sha(_WS_RE.sub(" ", s).strip()) + + +def _pdftotext(pdf: Path) -> str: + """`pdftotext -layout` with a cache next to the register PDFs. Deliberately + NOT stage 02's cache, which lives inside the cahiers directory — the + filenames are sha256 either way, so the two would otherwise share entries. + + An empty result is never cached: it means pdftotext failed or the PDF has + no text layer, and freezing that would turn a transient problem into a + permanent `register-extract-failed` verdict.""" + TEXT_CACHE.mkdir(parents=True, exist_ok=True) + cached = TEXT_CACHE / f"{pdf.stem}.txt" + if cached.exists(): + return cached.read_text(encoding="utf-8") + proc = subprocess.run( + ["pdftotext", "-layout", str(pdf), "-"], + capture_output=True, text=True, encoding="utf-8", errors="replace", check=False, + ) + out = proc.stdout or "" + if not out.strip(): + print(f"[warn] pdftotext produced nothing for {pdf.name} " + f"(rc={proc.returncode}) — not cached", file=sys.stderr) + return out + cached.write_text(out, encoding="utf-8") + return out + + +class _ExtractTimeout(Exception): + pass + + +def _extract(name: str, text: str, limit: int) -> dict | None: + """`extract_one` under a wall-clock guard. + + `extract_aire`'s department-header regex backtracks pathologically on some + layouts — the register's JORF-issue attachment for Pouilly-Fumé takes + minutes on a 2 KB area section (see CURATOR_TODO). That is a pre-existing + extractor bug, not a register one, but an unguarded call lets a single + document stall a 466-appellation sweep.""" + if limit <= 0: + return m02.extract_one(name, text) + + def _fire(_sig, _frm): + raise _ExtractTimeout + + previous = signal.signal(signal.SIGALRM, _fire) + signal.alarm(limit) + try: + return m02.extract_one(name, text) + except _ExtractTimeout: + raise + finally: + signal.alarm(0) + signal.signal(signal.SIGALRM, previous) + + +def _shape(record: dict | None) -> dict: + """The comparable surface of an extraction.""" + if record is None: + return {"ok": False} + lien = record.get("lien_au_terroir") or "" + aire = (record.get("aire") or {}).get("aire_geographique") or {} + return { + "ok": True, + "kind": record.get("kind", ""), + "sections": len(record.get("sections") or {}), + "section_keys": sorted((record.get("sections") or {}).keys()), + "lien_chars": len(lien), + "lien_sha256": _sha(lien), + "lien_norm_sha256": _norm_sha(lien), + "lien": lien, + "communes": sum(len(v) for v in aire.values()), + "departements": len(aire), + "bundle_size": record.get("bundle_size", 0), + } + + +def _drop_text(shape: dict) -> dict: + shape.pop("lien", None) + return shape + + +def _build_shape(on_disk: dict) -> dict: + shape = _shape(on_disk) + shape["grapes"] = len(((on_disk.get("grapes") or {}).get("details")) or []) + return shape + + +# A delta at or above this word-overlap is typographic, not editorial: the two +# PDFs carry the same cahier, laid out differently or with a stray accent +# fixed. Below it the sections genuinely diverge. +COSMETIC_SIMILARITY = 0.99 + + +def _parent_slugs() -> dict[str, str]: + index = json.loads(INDEX_PATH.read_text(encoding="utf-8")) + return { + e["id_appellation"]: e["slug"] + for e in index.values() + if not e.get("is_sub_denomination") + } + + +def _verdict(reg: dict, build: dict, self_: dict) -> str: + if not reg.get("ok"): + return "register-extract-failed" + if not build.get("ok"): + return "no-build-record" + if not reg["lien_chars"] and not build["lien_chars"]: + # Two empty liens hash the same; calling that "identical" would let a + # wrong bind whose section X did not parse count as a match. + return "no-lien-either-side" + if reg["lien_sha256"] == build["lien_sha256"]: + return "identical" + if reg["lien_norm_sha256"] == build["lien_norm_sha256"]: + return "identical-normalised" + if self_.get("ok") and self_["lien_sha256"] != build["lien_sha256"]: + return "differs-build-not-reproducible" + if not reg["lien_chars"]: + return "differs-register-lien-empty" + return "differs" + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--only", action="append", default=[], + help="appellation name substring (repeatable)") + ap.add_argument("--limit", type=int, default=0) + ap.add_argument("--delay", type=float, default=rc.DEFAULT_DELAY, + help="seconds between register requests") + ap.add_argument("--refresh", action="store_true", help="re-fetch cached attachments") + ap.add_argument("--no-selfcheck", action="store_true", + help="skip re-extracting the build's own PDF") + ap.add_argument("--extract-timeout", type=int, default=20, + help="seconds before an extraction is abandoned (0 = no guard)") + args = ap.parse_args() + + for path in (MANIFEST_PATH, INDEX_PATH, rc.RESOLVED_PATH): + if not path.exists(): + print(f"error: {path.relative_to(ROOT)} missing " + f"(run stages 01, 02 and 01d first)", file=sys.stderr) + return 1 + + manifest = json.loads(MANIFEST_PATH.read_text(encoding="utf-8")) + resolved = rc.load_resolved() + slugs = _parent_slugs() + + items = sorted(manifest.items(), key=lambda kv: kv[1]["name"].lower()) + if args.only: + needles = [s.lower() for s in args.only] + items = [(k, v) for k, v in items if any(n in v["name"].lower() for n in needles)] + if args.limit: + items = items[: args.limit] + + session = requests.Session() + id_map = er.id_map_from_rows(er.load_gi_rows(session=session)) + reg_manifest = rc.load_manifest() + + rows: list[dict] = [] + verdicts: Counter[str] = Counter() + fetch_status: Counter[str] = Counter() + + for id_app, meta in tqdm(items, desc="shadow", leave=False): + name = meta["name"] + slug = slugs.get(id_app, "") + on_disk_path = EXTRACTED / f"{slug}.json" if slug else None + on_disk = ( + json.loads(on_disk_path.read_text(encoding="utf-8")) + if on_disk_path and on_disk_path.exists() else None + ) + binding = resolved.get(id_app) + + row: dict = { + "id_appellation": id_app, + "name": name, + "slug": slug, + "file_number": (binding or {}).get("file_number", ""), + "register_name": (binding or {}).get("register_name", ""), + "matched_via": (binding or {}).get("matched_via", ""), + "build": _build_shape(on_disk) if on_disk else {"ok": False}, + "build_source": { + "filename": ((on_disk or {}).get("source") or {}).get("filename", ""), + "boagri_url": ((on_disk or {}).get("source") or {}).get("boagri_url", ""), + "homologated_at": ((on_disk or {}).get("source") or {}).get( + "homologated_at", ""), + "rescued_from_pdf": ((on_disk or {}).get("source") or {}).get( + "rescued_from_pdf", ""), + }, + } + + if not binding: + row["fetch_status"] = "unresolved-name" + row["register"] = {"ok": False} + row["self"] = {"ok": False} + row["verdict"] = "unresolved-name" + fetch_status["unresolved-name"] += 1 + verdicts["unresolved-name"] += 1 + _drop_text(row["build"]) + rows.append(row) + continue + + cahier, status = rc.fetch( + binding["file_number"], id_map, reg_manifest, + session=session, refresh=args.refresh, delay=args.delay, + ) + fetch_status[status] += 1 + row["fetch_status"] = status + if cahier is None: + row["register"] = {"ok": False} + row["self"] = {"ok": False} + row["verdict"] = status + verdicts[status] += 1 + _drop_text(row["build"]) + rows.append(row) + continue + + row["register_attachment"] = { + "uri": cahier.attachment_uri, + "url": cahier.attachment_url, + "name": cahier.attachment_name, + "sha256": cahier.sha256, + "bytes": cahier.bytes, + } + reg_text = _pdftotext(cahier.path) + try: + reg_record = _extract(name, reg_text, args.extract_timeout) + except _ExtractTimeout: + print(f"[timeout] {name}: register extraction exceeded " + f"{args.extract_timeout}s", file=sys.stderr) + row["register"] = {"ok": False, "timeout": True} + row["self"] = {"ok": False} + row["verdict"] = "register-extract-timeout" + verdicts["register-extract-timeout"] += 1 + _drop_text(row["build"]) + rows.append(row) + continue + row["register"] = _shape(reg_record) + if row["register"].get("ok") and row["build"].get("ok"): + row["lien_similarity"] = round( + _similarity(row["build"]["lien"], row["register"]["lien"]), 4) + if reg_record is not None: + # Same rule stage 02 applies to the build value: date the matched + # segment, not the whole file. For a GI whose attachment is a + # whole Journal-officiel issue, a whole-file scan would report a + # neighbouring appellation's homologation date. + segment = m02.find_segment(m02.split_bundle(reg_text), name) or reg_text + row["register"]["homologated_at"] = m02.homologation_date(segment) or "" + + self_shape: dict = {"ok": False} + build_pdf = row["build_source"]["filename"] + if not args.no_selfcheck and build_pdf and (CAHIERS / build_pdf).exists(): + try: + self_shape = _shape( + _extract(name, m02.pdftotext(CAHIERS / build_pdf), args.extract_timeout) + ) + except (subprocess.CalledProcessError, OSError, _ExtractTimeout): + self_shape = {"ok": False} + row["self"] = self_shape + + row["verdict"] = _verdict(row["register"], row["build"], self_shape) + verdicts[row["verdict"]] += 1 + for key in ("build", "register", "self"): + _drop_text(row[key]) + rows.append(row) + + rc.save_manifest(reg_manifest) + generated_at = datetime.now(timezone.utc).isoformat(timespec="seconds") + REPORT_JSON.write_text(json.dumps( + {"generated_at": generated_at, "n": len(rows), + "verdicts": dict(verdicts), "fetch_status": dict(fetch_status), "rows": rows}, + ensure_ascii=False, indent=2), encoding="utf-8") + REPORT_MD.write_text(_markdown(rows, verdicts, fetch_status, generated_at), + encoding="utf-8") + + print(f"[verdicts] {dict(verdicts)}", file=sys.stderr) + print(f"[fetch] {dict(fetch_status)}", file=sys.stderr) + print(f"[done] {len(rows)} appellations -> {REPORT_JSON.relative_to(ROOT)}, " + f"{REPORT_MD.relative_to(ROOT)}", file=sys.stderr) + return 0 + + +def _vintage(row: dict) -> str: + """Which side carries the more recent homologation, when both are dated.""" + b = row["build_source"].get("homologated_at") or "" + r = row["register"].get("homologated_at") or "" + if not b or not r: + return "unknown" + if b == r: + return "same" + return "register older" if r < b else "register newer" + + +def _markdown(rows, verdicts, fetch_status, generated_at) -> str: + n = len(rows) + with_cahier = sum(1 for r in rows if r["register"].get("ok")) + lines = [ + "# FR eAmbrosia register — shadow report", + "", + f"Generated {generated_at}. {n} parent appellations.", + "", + "Read-only comparison of the register's national cahier attachment " + "(`productSpecifications[0]`) against the record currently in " + "`raw/inao/cahier-extracted/`, both parsed by the unchanged stage-02 " + "extractor. `self` re-extracts the build's own PDF as a determinism " + "control.", + "", + "## Coverage", + "", + "| step | count | share |", + "|---|---:|---:|", + f"| parent appellations | {n} | 100.0 % |", + f"| resolved to a register file number | " + f"{sum(1 for r in rows if r['file_number'])} | " + f"{100.0 * sum(1 for r in rows if r['file_number']) / max(n, 1):.1f} % |", + f"| register serves a cahier PDF | " + f"{sum(1 for r in rows if r.get('register_attachment'))} | " + f"{100.0 * sum(1 for r in rows if r.get('register_attachment')) / max(n, 1):.1f} % |", + f"| register cahier extracts | {with_cahier} | " + f"{100.0 * with_cahier / max(n, 1):.1f} % |", + "", + "## Verdicts", + "", + "| verdict | count |", + "|---|---:|", + ] + for k, v in sorted(verdicts.items(), key=lambda kv: -kv[1]): + lines.append(f"| {k} | {v} |") + lines += ["", "## Fetch status", "", "| status | count |", "|---|---:|"] + for k, v in sorted(fetch_status.items(), key=lambda kv: -kv[1]): + lines.append(f"| {k} | {v} |") + + compared = [r for r in rows if r.get("lien_similarity") is not None] + cosmetic = [r for r in compared + if r["verdict"].startswith("differs") + and r["lien_similarity"] >= COSMETIC_SIMILARITY] + substantive = [r for r in compared + if r["verdict"].startswith("differs") + and r["lien_similarity"] < COSMETIC_SIMILARITY] + lines += [ + "", + "## How far apart are the two documents?", + "", + "`lien_similarity` is the word-multiset Dice coefficient between the two " + f"`lien_au_terroir` blocks. At or above {COSMETIC_SIMILARITY} the delta is " + "typographic — a guillemet space, `mélant` → `mêlant`, a different column " + "width — rather than a different text.", + "", + "| bucket | count |", + "|---|---:|", + f"| byte-identical | {sum(1 for r in compared if r['verdict'] == 'identical')} |", + f"| differs, cosmetically (≥ {COSMETIC_SIMILARITY}) | {len(cosmetic)} |", + f"| differs, substantively (< {COSMETIC_SIMILARITY}) | {len(substantive)} |", + "", + "### Vintage, where both sides carry a homologation date", + "", + "| relation | count |", + "|---|---:|", + ] + vintages = Counter(_vintage(r) for r in compared) + for k, v in sorted(vintages.items(), key=lambda kv: -kv[1]): + lines.append(f"| {k} | {v} |") + if substantive: + lines += [ + "", + f"### Substantively different ({len(substantive)}) — the ones to read", + "", + "| appellation | file number | similarity | build lien | register lien | vintage |", + "|---|---|---:|---:|---:|---|", + ] + for r in sorted(substantive, key=lambda r: r["lien_similarity"]): + lines.append( + f"| {r['name']} | {r['file_number']} | {r['lien_similarity']:.3f} | " + f"{r['build']['lien_chars']} | {r['register']['lien_chars']} | " + f"{_vintage(r)} |" + ) + + interesting = [r for r in rows if r["verdict"] != "identical"] + lines += [ + "", + f"## Non-identical ({len(interesting)})", + "", + "| appellation | file number | verdict | sim. | build lien | register lien | " + "build homol. | register homol. | vintage |", + "|---|---|---|---:|---:|---:|---|---|---|", + ] + for r in sorted(interesting, key=lambda r: (r["verdict"], r["name"].lower())): + sim = r.get("lien_similarity") + lines.append( + f"| {r['name']} | {r['file_number'] or '—'} | {r['verdict']} | " + f"{'—' if sim is None else format(sim, '.3f')} | " + f"{r['build'].get('lien_chars', '—')} | " + f"{r['register'].get('lien_chars', '—')} | " + f"{r['build_source'].get('homologated_at') or '—'} | " + f"{r['register'].get('homologated_at') or '—'} | " + f"{_vintage(r)} |" + ) + return "\n".join(lines) + "\n" + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/deploy.sh b/scripts/deploy.sh index d321ac5..35199f5 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -3,6 +3,11 @@ set -euo pipefail # Trampoline to scripts/deploy.py — publishes wiki/ to Bunny Storage via # the HTTP API and purges the CDN cache. See deploy.py for env vars. +# +# The .env sourced below also carries CARTO_BASEMAP_KEY, which is a *build*-time +# variable: stage 04 bakes it into the emitted app.js tile URLs. It is listed +# here because .env is the one place credentials live; deploy.py itself never +# reads it. Rebuild (04_build_maps.py) after rotating it, then deploy. cd "$(dirname "$0")/.." diff --git a/tests/test_fr_register_match.py b/tests/test_fr_register_match.py new file mode 100644 index 0000000..4e30bf2 --- /dev/null +++ b/tests/test_fr_register_match.py @@ -0,0 +1,226 @@ +"""Guards for the FR SIQO → eAmbrosia register name resolver. + +The resolver's failure mode is a *wrong* bind — attaching another +appellation's cahier des charges — which is worse than a gap, so the tests +below pin the three refusals (cross-product-type, ambiguity, collision) +alongside the synonym folds the register actually needs. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) + +from _lib.fr import register_match as rm # noqa: E402 + + +def _row(file_number, name, product_type="Wine", status="Registered", country="fr"): + return { + "id": abs(hash(file_number)) % 100000, + "fileName": file_number, + "protectedName": name, + "countryId": country, + "qualityProductType": product_type, + "geographicalIndicatorTypeCode": file_number.split("-")[0], + "status": status, + "showInRegister": True, + } + + +def _app(id_app, name, categorie="Vin tranquille"): + return {"id_appellation": id_app, "name": name, "categorie": categorie} + + +def test_register_keys_split_on_slash_synonyms() -> None: + keys = rm.register_keys("Bourg / Côtes de Bourg / Bourgeais") + assert "cotesdebourg" in keys + assert "bourgeais" in keys + + +def test_siqo_alias_matches_register_slash_alias() -> None: + rows = [_row("PDO-FR-A0828", "Bourg / Côtes de Bourg / Bourgeais")] + resolved, unresolved = rm.resolve_all( + [_app("1", "Côtes de Bourg, Bourg et Bourgeais")], rows, overrides={} + ) + assert not unresolved + assert resolved["1"].file_number == "PDO-FR-A0828" + assert resolved["1"].matched_via == "alias" + + +def test_product_type_partition_keeps_wine_and_spirit_apart() -> None: + """'Calvados' is both a Normandy IGP wine and an eau-de-vie PDO.""" + rows = [ + _row("PGI-FR-A1222", "Calvados", "Wine"), + _row("PGI-FR-02033", "Calvados", "Spirit drink"), + ] + resolved, unresolved = rm.resolve_all( + [ + _app("1", "Calvados", "Vin tranquille"), + _app("2", "Calvados", "Eaux-de-vie de cidre et de poiré"), + ], + rows, + overrides={}, + ) + assert not unresolved + assert resolved["1"].file_number == "PGI-FR-A1222" + assert resolved["2"].file_number == "PGI-FR-02033" + + +def test_unmapped_categorie_refuses_when_partitions_disagree() -> None: + """With no categorie the resolver searches every partition, so a name + claimed by two product types has to fall to the curator.""" + rows = [ + _row("PGI-FR-A1222", "Calvados", "Wine"), + _row("PGI-FR-02033", "Calvados", "Spirit drink"), + ] + resolved, unresolved = rm.resolve_all( + [_app("1", "Calvados", categorie="")], rows, overrides={} + ) + assert not resolved + assert unresolved[0]["reason"] == "ambiguous" + + +def test_unmapped_categorie_resolves_when_unique_across_partitions() -> None: + rows = [_row("PDO-FR-A0201", "Côte Roannaise", "Wine")] + resolved, unresolved = rm.resolve_all( + [_app("1", "Côte roannaise", categorie="")], rows, overrides={} + ) + assert not unresolved + assert resolved["1"].file_number == "PDO-FR-A0201" + assert resolved["1"].partition == "any" + + +def test_cancelled_row_loses_to_the_registered_one() -> None: + rows = [ + _row("PDO-FR-0001", "Bandol", status="Cancelled"), + _row("PDO-FR-A0002", "Bandol", status="Registered"), + ] + resolved, _ = rm.resolve_all([_app("1", "Bandol")], rows, overrides={}) + assert resolved["1"].file_number == "PDO-FR-A0002" + + +def test_two_appellations_claiming_one_gi_are_both_refused() -> None: + rows = [_row("PDO-FR-A0100", "Bourgogne")] + resolved, unresolved = rm.resolve_all( + [_app("1", "Bourgogne"), _app("2", "Bourgogne ou Bourgogne")], rows, overrides={} + ) + assert not resolved + assert {u["reason"] for u in unresolved} == {"collision"} + + +def test_short_alias_fragment_cannot_bind() -> None: + """A 3-letter alias remnant is too generic to attach a cahier on.""" + rows = [_row("PDO-FR-A0300", "Vin")] + _, unresolved = rm.resolve_all([_app("1", "Quelque chose ou Vin")], rows, overrides={}) + assert unresolved[0]["reason"] == "no-name-match" + + +def test_no_name_match_goes_to_the_queue_not_a_fuzzy_bind() -> None: + rows = [_row("PDO-FR-A0400", "Chablis Grand Cru")] + resolved, unresolved = rm.resolve_all([_app("1", "Chablis")], rows, overrides={}) + assert not resolved + assert unresolved[0]["reason"] == "no-name-match" + + +def test_override_pins_a_file_number_the_matcher_cannot_reach() -> None: + rows = [_row("PGI-FR-01837", "Calvados Domfrontais", "Spirit drink")] + resolved, unresolved = rm.resolve_all( + [_app("1", "Calvados Domfontais", "Eaux-de-vie de cidre et de poiré")], + rows, + overrides={"1": {"file_number": "PGI-FR-01837"}}, + ) + assert not unresolved + assert resolved["1"].matched_via == "override" + + +def test_empty_override_records_a_verified_absence() -> None: + resolved, unresolved = rm.resolve_all( + [_app("1", "Nowhere")], [], overrides={"1": {"file_number": ""}} + ) + assert not resolved and not unresolved + + +def test_override_pointing_at_an_unknown_gi_is_reported() -> None: + _, unresolved = rm.resolve_all( + [_app("1", "Nowhere")], [], overrides={"1": {"file_number": "PDO-FR-XXXX"}} + ) + assert unresolved[0]["reason"] == "override-file-number-not-in-register" + + +def test_cross_border_rows_are_french_rows() -> None: + rows = rm.fr_rows([ + _row("PDO-ES-FR-02309", "Euskal Sagardoa", "Food", country="es,fr"), + _row("PDO-ES-0001", "Rioja", "Wine", country="es"), + ]) + assert [r["fileName"] for r in rows] == ["PDO-ES-FR-02309"] + + +def test_shipped_overrides_are_well_formed() -> None: + for id_app, pin in rm.load_overrides().items(): + assert id_app.isdigit(), id_app + assert "file_number" in pin and "note" in pin, id_app + if pin["file_number"]: + assert pin["file_number"].startswith(("PDO-", "PGI-")), id_app + + +def test_cancelled_gi_is_refused_not_bound() -> None: + """A GI the Commission struck off cannot be an appellation's current + specification; binding to one has to be a deliberate curator pin.""" + rows = [_row("PGI-FR-A1203", "Cité de Carcassonne", status="Cancelled")] + resolved, unresolved = rm.resolve_all( + [_app("1", "Cité de Carcassonne")], rows, overrides={} + ) + assert not resolved + assert unresolved[0]["reason"] == "withdrawn-registration" + + +def test_pending_registration_still_binds() -> None: + rows = [_row("PDO-FR-03288", "Grés de Montpellier", status="Applied")] + resolved, unresolved = rm.resolve_all( + [_app("1", "Grés de Montpellier")], rows, overrides={} + ) + assert not unresolved + assert resolved["1"].status == "Applied" + + +def test_populated_but_unknown_categorie_refuses_rather_than_widening() -> None: + rows = [_row("PGI-FR-99999", "Somewhere", "Spirit drink")] + resolved, unresolved = rm.resolve_all( + [_app("1", "Somewhere", categorie="Boisson inconnue")], rows, overrides={} + ) + assert not resolved + assert unresolved[0]["reason"] == "unmapped-categorie" + + +def test_every_siqo_categorie_in_the_corpus_is_mapped() -> None: + """A new SIQO categorie must be added to the table, not silently widened + to every product-type partition.""" + import csv + from pathlib import Path + + siqo = Path(__file__).resolve().parents[1] / "raw" / "inao" / "siqo-referentiel.csv" + if not siqo.exists(): + return + with open(siqo, encoding="utf-8-sig", newline="") as f: + seen = { + row["categorie"].strip() + for row in csv.DictReader(f) + if row["secteur"].strip() == "VITICOLE" + and row["lib_etat"].strip() == "Publié" + and (row["signe_fr"].strip() or row["signe_ue"].strip()) in {"AOC", "AOP", "IGP"} + } + missing = sorted(c for c in seen if c and c not in rm.CATEGORIE_PRODUCT_TYPE) + assert not missing, f"unmapped SIQO categorie values: {missing}" + + +def test_a_pin_survives_a_collision_and_the_auto_match_loses() -> None: + rows = [_row("PDO-FR-A0100", "Bourgogne")] + resolved, unresolved = rm.resolve_all( + [_app("1", "Bourgogne"), _app("2", "Bourgogne")], + rows, + overrides={"2": {"file_number": "PDO-FR-A0100"}}, + ) + assert list(resolved) == ["2"] + assert unresolved[0]["reason"] == "collision-with-pin" diff --git a/tests/test_fr_register_tier.py b/tests/test_fr_register_tier.py new file mode 100644 index 0000000..a795588 --- /dev/null +++ b/tests/test_fr_register_tier.py @@ -0,0 +1,233 @@ +"""Guards for stage 01's eAmbrosia register fallback tier. + +Offline: the register client is never called. What is pinned here is the +*shape* of a register-sourced manifest entry — the provenance keys, the +promotion of the cached attachment into the content-addressed cahiers dir, +and the fact that `fetched_at` comes from the cache rather than the clock, so +a re-run cannot churn the manifest. +""" + +from __future__ import annotations + +import hashlib +import importlib +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) + +from _lib.fr.register_cahier import RegisterCahier # noqa: E402 + +m01 = importlib.import_module("01_scrape_cahiers") + +PDF_BYTES = b"%PDF-1.4\nfake cahier\n" +SHA = hashlib.sha256(PDF_BYTES).hexdigest() +FETCHED_AT = "2026-08-29T09:00:00+00:00" + + +def _cahier(tmp_path: Path) -> tuple[RegisterCahier, Path]: + src = tmp_path / "register-cache" + src.mkdir() + (src / f"{SHA}.pdf").write_bytes(PDF_BYTES) + cahier = RegisterCahier( + file_number="PDO-FR-A0571", + attachment_uri="4685", + attachment_url="https://example.invalid/attachments/4685", + attachment_name="CDC_Batard-Montrachet.pdf", + single_doc_ref="Ares(2011)1350579", + filename=f"{SHA}.pdf", + sha256=SHA, + bytes=len(PDF_BYTES), + fetched_at=FETCHED_AT, + ) + return cahier, src + + +class _Tier: + """Minimal stand-in for RegisterTier — no network, no resolved.json.""" + + def __init__(self, cahier, binding): + self._cahier = cahier + self.resolved = {"130": binding} + self.enabled = True + + def cahier_for(self, id_appellation): + return self._cahier if id_appellation in self.resolved else None + + +def _patch_dirs(monkeypatch, tmp_path, cache_dir): + """Point the register cache at `cache_dir` and the cahiers dir at a temp + one, so nothing touches raw/.""" + out = tmp_path / "cahiers" + monkeypatch.setattr(m01, "OUT_DIR", out) + monkeypatch.setattr(m01.rc, "PDF_DIR", cache_dir) + return out + + +def test_register_tier_writes_provenance_and_promotes_the_pdf(monkeypatch, tmp_path): + cahier, cache = _cahier(tmp_path) + out = _patch_dirs(monkeypatch, tmp_path, cache) + app = m01.Appellation(id_appellation="130", name="Bâtard-Montrachet", products=[]) + manifest: dict = {} + + status, extra = m01._register_tier( + app, manifest, m01._stub_meta(app), + _Tier(cahier, {"register_name": "Bâtard-Montrachet", "matched_via": "full"}), + {}, fallback=("missed", 0), + ) + + assert (status, extra) == ("register", 0) + entry = manifest["130"] + assert entry["source_kind"] == "eambrosia-register" + assert entry["register_file_number"] == "PDO-FR-A0571" + assert entry["register_attachment_uri"] == "4685" + assert entry["register_attachment_name"] == "CDC_Batard-Montrachet.pdf" + assert entry["register_protected_name"] == "Bâtard-Montrachet" + assert entry["register_matched_via"] == "full" + assert entry["boagri_url"] == "", "a register attachment did not come from BO Agri" + assert entry["register_attachment_url"].endswith("/4685") + assert entry["sha256"] == SHA + assert entry["filename"] == f"{SHA}.pdf" + assert entry["fetched_at"] == FETCHED_AT, "must come from the cache, not the clock" + assert (out / f"{SHA}.pdf").read_bytes() == PDF_BYTES + + +def test_register_tier_is_inert_without_a_binding(monkeypatch, tmp_path): + cahier, cache = _cahier(tmp_path) + _patch_dirs(monkeypatch, tmp_path, cache) + app = m01.Appellation(id_appellation="999", name="Nowhere", products=[]) + manifest: dict = {} + + assert m01._register_tier( + app, manifest, m01._stub_meta(app), _Tier(cahier, {}), {}, + fallback=("missed", 0), + ) == ("missed", 0) + assert manifest == {} + + +def test_register_tier_is_inert_when_disabled(monkeypatch, tmp_path): + app = m01.Appellation(id_appellation="130", name="Bâtard-Montrachet", products=[]) + manifest: dict = {} + assert m01._register_tier( + app, manifest, m01._stub_meta(app), None, {}, fallback=None + ) == (None, 0) + assert manifest == {} + + +def test_promotion_is_idempotent(monkeypatch, tmp_path): + cahier, cache = _cahier(tmp_path) + out = _patch_dirs(monkeypatch, tmp_path, cache) + out.mkdir() + (out / f"{SHA}.pdf").write_bytes(PDF_BYTES) + mtime = (out / f"{SHA}.pdf").stat().st_mtime_ns + + m01._promote_register_pdf(cahier) + + assert (out / f"{SHA}.pdf").stat().st_mtime_ns == mtime + + +def test_stub_meta_carries_no_register_keys(): + """A miss that the register cannot fill must leave a plain entry, so an + appellation still on BO Agri keeps a byte-identical manifest entry.""" + app = m01.Appellation(id_appellation="1", name="X", products=[]) + assert not any(k.startswith("register_") for k in m01._stub_meta(app)) + assert "source_kind" not in m01._stub_meta(app) + + +def test_cached_rejects_a_pdf_that_no_longer_hashes_to_its_name(monkeypatch, tmp_path): + """The cached file is fed to the stage-02 extractor as if it were the + regulator's document, so corruption must fail loudly.""" + from _lib.fr import register_cahier as regc + + cahier, cache = _cahier(tmp_path) + monkeypatch.setattr(regc, "PDF_DIR", cache) + manifest = {cahier.file_number: {"status": "ok", **_asdict(cahier)}} + + assert regc.cached(cahier.file_number, manifest) is not None + (cache / cahier.filename).write_bytes(PDF_BYTES[:5]) + assert regc.cached(cahier.file_number, manifest) is None + + +def test_sticky_miss_is_remembered_and_transient_failure_is_not(tmp_path): + from _lib.fr import register_cahier as regc + + assert regc.recorded_miss("X", {"X": {"status": "no-cahier"}}) == "no-cahier" + assert regc.recorded_miss("X", {"X": {"status": "unresolved"}}) == "unresolved" + assert regc.recorded_miss("X", {"X": {"status": "fetch-failed"}}) is None + assert regc.recorded_miss("X", {}) is None + + +def test_legacy_manifest_entries_without_status_still_resolve(monkeypatch, tmp_path): + from _lib.fr import register_cahier as regc + + cahier, cache = _cahier(tmp_path) + monkeypatch.setattr(regc, "PDF_DIR", cache) + legacy = {cahier.file_number: _asdict(cahier)} # written before `status` existed + assert regc.cached(cahier.file_number, legacy) is not None + + +def _asdict(cahier): + from dataclasses import asdict + + return asdict(cahier) + + +def test_register_tier_never_re_sources_an_appellation_that_has_a_cahier( + monkeypatch, tmp_path, +): + """A transient INAO outage is not an upstream change: `resolve_cahier` and + `_download_first_pdf` both return None for a 5xx or a timeout, so the tier + must leave a working entry exactly as it found it.""" + cahier, cache = _cahier(tmp_path) + out = _patch_dirs(monkeypatch, tmp_path, cache) + out.mkdir() + (out / "aaaa.pdf").write_bytes(b"%PDF-1.4\nthe canonical BO Agri cahier\n") + prior = { + "name": "Bâtard-Montrachet", + "boagri_url": "https://info.agriculture.gouv.fr/.../telechargement", + "filename": "aaaa.pdf", + "sha256": "aaaa", + "fetched_at": "2026-05-01T00:00:00+00:00", + } + manifest = {"130": dict(prior)} + app = m01.Appellation(id_appellation="130", name="Bâtard-Montrachet", products=[]) + + assert m01._register_tier( + app, manifest, m01._stub_meta(app, prior), + _Tier(cahier, {"register_name": "Bâtard-Montrachet", "matched_via": "full"}), + prior, fallback=("missed", 0), + ) == ("missed", 0) + assert manifest["130"] == prior + + +def test_stub_meta_carries_prior_inao_metadata_forward(): + """Failing to reach www2.inao.gouv.fr is not evidence that the product + page or the Légifrance trail ceased to exist.""" + app = m01.Appellation(id_appellation="1", name="X", products=[]) + prior = { + "product_url": "https://www2.inao.gouv.fr/produit/42", + "show_texte_url": "https://www2.inao.gouv.fr/show_texte/9", + "show_texte_paths": ["/show_texte/9"], + "boagri_url_candidates": ["https://info.agriculture.gouv.fr/a"], + "legifrance_jorftext_ids": ["JORFTEXT000000000001"], + "canonical_idproduit": "42", + "canonical_produit": "X rouge", + } + meta = m01._stub_meta(app, prior) + for key, value in prior.items(): + assert meta[key] == value + assert meta["boagri_url"] == "" + + +def test_absent_id_map_does_not_record_a_sticky_miss(monkeypatch, tmp_path): + """A listing that failed to load says nothing about any individual GI, so + one bad network moment must not blank the tier permanently.""" + from _lib.fr import register_cahier as regc + + manifest: dict = {} + monkeypatch.setattr(regc, "MANIFEST_PATH", tmp_path / "manifest.json") + assert regc.fetch("PDO-FR-A0571", {}, manifest) == (None, "unresolved") + assert manifest == {} + + assert regc.fetch("PDO-FR-A0571", {"PDO-FR-OTHER": 1}, manifest) == (None, "unresolved") + assert manifest["PDO-FR-A0571"]["status"] == "unresolved"