From a5c84bf247d710abb4c3602e036c172e4fdd2ab4 Mon Sep 17 00:00:00 2001 From: Alexandre Catarino Date: Fri, 28 Aug 2026 19:21:26 +0100 Subject: [PATCH] Sync supported assets to dataset pages and refresh docs from the API Resources/datasets/supported-securities is regenerated weekly from the symbol-properties database, but the same tables are the "Supported Assets" section of a dozen dataset pages, where they were pasted by hand. The pages drifted behind the repo silently: Binance listed 3,320 pairs against 3,651 generated ones. sync_supported_assets.py renders each page's section from the repo files and writes it through the market/sections endpoints, keeping the Introduction, Data Summary and sidebar coverage counts in step by substituting the number rather than the sentence. It only overwrites the sections it maps, never creates one, and re-reads to confirm every write. Its workflow reports on a pull request and pushes on a merge to master, so the generator PR shows the diff before it reaches the live website. The alternative-dataset generators had the opposite problem: they were built entirely from alternative-data-dump-v2024-01-02.json, a snapshot that only changes when someone re-uploads it, so edits to a listing never reached the docs. get_dataset_listings() splits the two things the dump was doing. The dump still says which datasets are public, because nothing in the API records that. Page content comes from market/sections/read, so a listing edit shows up on the next run. Drift between the two sources is printed rather than acted on, since either direction needs a person to decide. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/code_generators.yml | 1 + .github/workflows/sync_supported_assets.yml | 53 +++ .../Alternative-Datasets-Code-Generator.py | 7 +- .../Alternative-Datasets-Skill-Generator.py | 17 +- code-generators/_code_generation_helpers.py | 96 ++++- code-generators/sync_supported_assets.py | 359 ++++++++++++++++++ 6 files changed, 518 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/sync_supported_assets.yml create mode 100644 code-generators/sync_supported_assets.py diff --git a/.github/workflows/code_generators.yml b/.github/workflows/code_generators.yml index 9ca9fd8866..96d1ff65c5 100644 --- a/.github/workflows/code_generators.yml +++ b/.github/workflows/code_generators.yml @@ -26,6 +26,7 @@ jobs: env: QUANTCONNECT_USER_ID: ${{ secrets.QUANTCONNECT_USER_ID }} QUANTCONNECT_API_TOKEN: ${{ secrets.QUANTCONNECT_API_TOKEN }} + QUANTCONNECT_ORGANIZATION_ID: ${{ secrets.QUANTCONNECT_ORGANIZATION_ID }} run: | failed="" for script in \ diff --git a/.github/workflows/sync_supported_assets.yml b/.github/workflows/sync_supported_assets.yml new file mode 100644 index 0000000000..20c86c58e4 --- /dev/null +++ b/.github/workflows/sync_supported_assets.yml @@ -0,0 +1,53 @@ +name: Sync Dataset Page Supported Assets + +on: + pull_request: + branches: [master] + paths: + - 'Resources/datasets/supported-securities/**' + - 'code-generators/sync_supported_assets.py' + - '.github/workflows/sync_supported_assets.yml' + push: + branches: [master] + paths: + - 'Resources/datasets/supported-securities/**' + - 'code-generators/sync_supported_assets.py' + - '.github/workflows/sync_supported_assets.yml' + workflow_dispatch: + inputs: + dry_run: + description: 'Report what would change without writing' + type: boolean + default: true + +jobs: + sync: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: Sync the dataset pages + env: + QUANTCONNECT_USER_ID: ${{ secrets.QUANTCONNECT_USER_ID }} + QUANTCONNECT_API_TOKEN: ${{ secrets.QUANTCONNECT_API_TOKEN }} + QUANTCONNECT_ORGANIZATION_ID: ${{ secrets.QUANTCONNECT_ORGANIZATION_ID }} + run: | + if [[ -z "$QUANTCONNECT_API_TOKEN" ]]; then + echo "::notice::No QuantConnect credentials available; skipping the sync." + exit 0 + fi + if [[ "${{ github.event_name }}" == "push" || "${{ inputs.dry_run }}" == "false" ]]; then + python code-generators/sync_supported_assets.py --snapshot snapshot.json + else + python code-generators/sync_supported_assets.py --dry-run --snapshot snapshot.json + fi + + - name: Upload the pre-change sections + if: always() + uses: actions/upload-artifact@v4 + with: + name: dataset-page-snapshot + path: snapshot.json + if-no-files-found: ignore + retention-days: 90 diff --git a/code-generators/Alternative-Datasets-Code-Generator.py b/code-generators/Alternative-Datasets-Code-Generator.py index 4a870e0b64..1660f186a1 100644 --- a/code-generators/Alternative-Datasets-Code-Generator.py +++ b/code-generators/Alternative-Datasets-Code-Generator.py @@ -4,10 +4,8 @@ from itertools import groupby from shutil import move, rmtree from bs4 import BeautifulSoup -from urllib.request import urlopen -import json -from _code_generation_helpers import generate_landing_page +from _code_generation_helpers import generate_landing_page, get_dataset_listings DATASET = "03 Writing Algorithms/14 Datasets" @@ -115,8 +113,7 @@ def _parse_content(content): return content if __name__ == '__main__': - url = "https://s3.amazonaws.com/cdn.quantconnect.com/web/docs/alternative-data-dump-v2024-01-02.json" - docs = json.loads(urlopen(url).read().decode('utf-8')) + docs = get_dataset_listings() for d in docs: if d["vendorName"].strip() == "CoinAPI": d["vendorName"] = "QuantConnect" diff --git a/code-generators/Alternative-Datasets-Skill-Generator.py b/code-generators/Alternative-Datasets-Skill-Generator.py index 0b267eba7b..8278f854a0 100644 --- a/code-generators/Alternative-Datasets-Skill-Generator.py +++ b/code-generators/Alternative-Datasets-Skill-Generator.py @@ -1,9 +1,9 @@ """Generate the alternative-data and alternative-data-universes SKILL.md files. -Walks the QC alternative-data dump JSON, scrapes `add_data` / `add_universe` -class references and accessing-data list markers from each dataset's section -content, then enriches the alt-data skill with C#/Python property names -fetched from the QC inspector in parallel. +Walks the dataset listings read live from the QuantConnect API, scrapes +`add_data` / `add_universe` class references and accessing-data list markers +from each dataset's section content, then enriches the alt-data skill with +C#/Python property names fetched from the QC inspector in parallel. Page generation lives in `Alternative-Datasets-Code-Generator.py`; this file only writes the two SKILL.md files. @@ -15,8 +15,9 @@ class references and accessing-data list markers from each dataset's section from pathlib import Path from urllib.request import urlopen +from _code_generation_helpers import get_dataset_listings + -DUMP_URL = "https://s3.amazonaws.com/cdn.quantconnect.com/web/docs/alternative-data-dump-v2024-01-02.json" INSPECTOR_URL = ( "https://www.quantconnect.com/services/inspector" "?language={lang}&type=T:QuantConnect.DataSource.{cls}" @@ -286,7 +287,7 @@ def _build_alt_data_skill(skill_data, props_lookup=None): return "\n".join(parts) + "\n" -# === Dump traversal === +# === Listing traversal === def _ordered_sections(dataset): """Flatten the dataset's about + documentation entries into a {title: content} dict.""" @@ -340,8 +341,8 @@ def _scrape_skill_data(docs): def main(): - print(f"Downloading alt-data dump from {DUMP_URL}") - docs = json.loads(urlopen(DUMP_URL).read().decode("utf-8")) + print("Reading the dataset listings from the QuantConnect API...") + docs = get_dataset_listings() alt_universe_skill_data, alt_data_skill_data = _scrape_skill_data(docs) diff --git a/code-generators/_code_generation_helpers.py b/code-generators/_code_generation_helpers.py index 6775561e77..bd038d495b 100644 --- a/code-generators/_code_generation_helpers.py +++ b/code-generators/_code_generation_helpers.py @@ -1,7 +1,12 @@ -from json import dumps +from base64 import b64encode +from concurrent.futures import ThreadPoolExecutor +from hashlib import sha256 +from json import dumps, loads +from os import environ from re import findall, finditer +from time import time from typing import List -from urllib.request import urlopen +from urllib.request import Request, urlopen WRITING_ALGORITHMS = '03 Writing Algorithms' INDICATORS = f'{WRITING_ALGORITHMS}/28 Indicators/01 Supported Indicators' @@ -48,6 +53,93 @@ def get_lean_cli_commands() -> dict: commands[name] = desc.rstrip('.') return commands +QC_API = 'https://www.quantconnect.com/api/v2' +SECTION_GROUPS = ('about', 'documentation', 'examples') + +def get_api_credentials() -> tuple: + """User id and API token from the environment.""" + user_id, api_token = environ.get('QUANTCONNECT_USER_ID'), environ.get('QUANTCONNECT_API_TOKEN') + if not user_id or not api_token: + raise RuntimeError('Set QUANTCONNECT_USER_ID and QUANTCONNECT_API_TOKEN. In Actions ' + 'they are empty on a pull request from a fork.') + return str(user_id), api_token + +def api_post(endpoint: str, payload: dict = None) -> dict: + """POST to the QuantConnect API v2 and return the envelope. Raises on failure.""" + user_id, api_token = get_api_credentials() + timestamp = str(int(time())) + hashed_token = sha256(f'{api_token}:{timestamp}'.encode('utf-8')).hexdigest() + auth = b64encode(f'{user_id}:{hashed_token}'.encode('utf-8')).decode('ascii') + request = Request(f'{QC_API}{endpoint}', data=dumps(payload or {}).encode('utf-8'), + headers={'Authorization': f'Basic {auth}', 'Timestamp': timestamp, + 'Content-Type': 'application/json'}) + body = loads(urlopen(request, timeout=120).read()) + if not body.get('success'): + raise RuntimeError(f'{endpoint} failed: {body.get("errors")}') + return body + +def get_organization_id() -> str: + """The organization id that market/sections/read requires on every call. + + Any organization the account belongs to works; it need not own the dataset. Omitting + it fails with the unhelpful 'Organization not found. '. + """ + if environ.get('QUANTCONNECT_ORGANIZATION_ID'): + return environ['QUANTCONNECT_ORGANIZATION_ID'] + organizations = api_post('/organizations/list').get('organizations', []) + if not organizations: + raise RuntimeError('The account belongs to no organizations.') + print(f'note: QUANTCONNECT_ORGANIZATION_ID unset, using {organizations[0]["id"]} ' + f'({organizations[0].get("name")})') + return organizations[0]['id'] + +DATASET_DUMP = 'https://s3.amazonaws.com/cdn.quantconnect.com/web/docs/alternative-data-dump-v2024-01-02.json' + +def get_public_dataset_slugs() -> set: + """Which datasets are public, per the dump. Nothing in the API records this.""" + return {d['url'].rsplit('/', 1)[-1] for d in loads(get_text_content(DATASET_DUMP))} + +def get_dataset_listings() -> List[dict]: + """Every public dataset listing, with its page sections read live from the API. + + The dump says which datasets are public; the API says what each page holds, so edits + to a listing reach the docs without waiting for a dump re-upload. Each record carries + what the dataset generators use: + + {"name": ..., "vendorName": ..., "url": "/datasets/", + "about": [{"title": ..., "content": ...}, ...], "documentation": [...], + "examples": [...]} + + Drift between the two sources is reported, not applied -- either direction needs a + person to decide. + """ + organization_id = get_organization_id() + public = get_public_dataset_slugs() + masters = {m['url']: m for m in api_post('/market/data/list')['list'] if not m.get('pending')} + + for slug in sorted(set(masters) - public): + print(f'note: {slug} is a live listing but is not in the dump, so it gets no docs ' + f'page. Refresh the dump if it should be public.') + for slug in sorted(public - set(masters)): + print(f'note: {slug} is in the dump but no longer a listing; it has been retired.') + + def read(master): + sections = api_post('/market/sections/read', + {'id': master['id'], 'organizationId': organization_id})['sections'] + listing = {'name': master.get('name'), 'vendorName': master.get('vendorName'), + 'url': f'/datasets/{master.get("url")}'} + for group in SECTION_GROUPS: + # Sort by position: the generators index into `about` for the vendor landing + # page, so reading order is load-bearing, not cosmetic. + listing[group] = [{'title': s.get('title'), 'content': s.get('content')} + for s in sorted(sections.get(group) or [], + key=lambda s: s.get('position') or 0)] + return listing + + masters = [m for slug, m in masters.items() if slug in public] + with ThreadPoolExecutor(8) as executor: + return list(executor.map(read, masters)) + def get_json_content(url: str) -> List: content = get_text_content(url) \ .replace("null", "None").replace("true", "True").replace("false", "False") diff --git a/code-generators/sync_supported_assets.py b/code-generators/sync_supported_assets.py new file mode 100644 index 0000000000..19f237f774 --- /dev/null +++ b/code-generators/sync_supported_assets.py @@ -0,0 +1,359 @@ +"""Push the generated supported-securities tables onto the QuantConnect dataset pages. + +`Supported-Assets-Table-Code-Generator.py` and `future-table-code-generator.py` regenerate +`Resources/datasets/supported-securities/` from the symbol-properties database. The same +tables are the body of the "Supported Assets" section on a handful of dataset pages, where +they used to be pasted by hand -- so every regeneration left the website behind, silently. +This closes that gap: it renders each page's section from the repo files and writes it +through the API v2 `market/sections` endpoints. + + python code-generators/sync_supported_assets.py --dry-run + python code-generators/sync_supported_assets.py --snapshot snapshot.json + +`--dry-run` reads the live pages, reports what it would change, and writes nothing. Without +it the script writes, then re-reads and diffs to confirm -- `success: true` from these +endpoints does not mean the write happened, so the envelope is never taken as evidence. +Exit status is non-zero if any page ends up different from what was sent. + +Credentials come from `QUANTCONNECT_USER_ID` / `QUANTCONNECT_API_TOKEN`, or from +`~/.lean/credentials`. `QUANTCONNECT_ORGANIZATION_ID` is required on every read; any +organization the account belongs to works, and the first one is used if the variable is +unset. + +The asset count +--------------- +A page states its size in four places, and pushing only the table leaves three of them +contradicting it: the table header, the Introduction sentence, the Data Summary "Asset +Coverage" row, and the sidebar Coverage field (`reach`). The other three are rewritten by +substituting the *number*, never the sentence, so reviewed wording survives. + +Guard-rails +----------- +This only ever overwrites the sections named in `MAPPING`; it never creates one, and a +missing section is an error rather than something to fill in. A page can therefore not be +restructured by an automated run, only refreshed. +""" + +import argparse +import json +import re +import sys +from pathlib import Path + +from _code_generation_helpers import api_post, get_organization_id + +RESOURCES = Path("Resources/datasets/supported-securities") +SECTION_TITLE = "Supported Assets" + +CRYPTO_LEAD = "

The following table shows the available Cryptocurrency pairs:

" +CRYPTO_FUTURE_LEAD = "

The following table shows the available Crypto Future pairs:

" + +# One entry per dataset page whose "Supported Assets" section is a copy of these files. +# `lead` is the sentence already on the live page -- keep it verbatim rather than rewriting +# prose reviewed elsewhere. The Futures files open with their own "(N) Futures" sentence, +# so they take an empty lead and carry their count with them (`counts: False`). +# +# Deliberately absent: +# * cryptofuture/dydx.html -- no dYdX dataset listing exists +# * cfd/interactivebrokers.html, forex/interactivebrokers.html +# -- datasets 25 and 26 are QuantConnect's OANDA-sourced +# CFD/Forex data; the IB tables belong to the Writing +# Algorithms docs, not to these listings +# * 118 tickdata-international-futures +# -- its table is hand-written, not generated +MAPPING = [ + {"product": 58, "slug": "binance-crypto-price-data", + "files": ["crypto/binance.html"], "lead": CRYPTO_LEAD}, + {"product": 76, "slug": "binance-us-crypto-price-data", + "files": ["crypto/binanceus.html"], "lead": CRYPTO_LEAD}, + {"product": 57, "slug": "bitfinex-crypto-price-data", + "files": ["crypto/bitfinex.html"], "lead": CRYPTO_LEAD}, + {"product": 96, "slug": "bybit-crypto-price-data", + "files": ["crypto/bybit.html"], "lead": CRYPTO_LEAD}, + {"product": 27, "slug": "coinbase-crypto-price-data", + "files": ["crypto/coinbase.html"], "lead": CRYPTO_LEAD}, + {"product": 60, "slug": "kraken-crypto-price-data", + "files": ["crypto/kraken.html"], "lead": CRYPTO_LEAD}, + + {"product": 85, "slug": "binance-cryptofuture-price-data", + "files": ["cryptofuture/binance.html"], "lead": CRYPTO_FUTURE_LEAD}, + {"product": 87, "slug": "binance-cryptofuture-margin-rate-data", + "files": ["cryptofuture/binance.html"], "lead": CRYPTO_FUTURE_LEAD}, + {"product": 97, "slug": "bybit-cryptofuture-price-data", + "files": ["cryptofuture/bybit.html"], "lead": CRYPTO_FUTURE_LEAD}, + {"product": 98, "slug": "bybit-cryptofuture-margin-rate-data", + "files": ["cryptofuture/bybit.html"], "lead": CRYPTO_FUTURE_LEAD}, + + {"product": 25, "slug": "quantconnect-forex", + "files": ["forex/oanda.html"], + "lead": "

The following table shows the available Forex pairs:

"}, + {"product": 26, "slug": "quantconnect-cfd", + "files": ["cfd/oanda.html"], + "lead": "

The following table shows the available contracts:

"}, + + {"product": 30, "slug": "algoseek-us-futures", + "files": ["future/supported-contracts.html"], "lead": "", "counts": False}, + {"product": 31, "slug": "algoseek-us-future-options", + "files": ["futureoption/supported-contracts.html"], "lead": "", "counts": False, + "hold": "the generated file says '(16) Futures Options' above 15 items -- " + "future-table-code-generator.py counts len(FUTURE_OPTIONS) while DC is " + "dropped, because 'Dairy' is missing from its category list. The live " + "'(15)' is the correct number; fix the generator before syncing."}, +] + +for _entry in MAPPING: + _entry.setdefault("counts", True) + +_TICKER = re.compile(r"([^<]+)") +_ITEM = re.compile(r"
  • ]*>([^<]+)") +_ASSET_COVERAGE = re.compile(r"Asset Coverage.*?\s*([\d,]+)\b", re.S) + + +# --------------------------------------------------------------------------- transport + +def call(endpoint, payload=None): + """POST to the API, turning a failure into a clean exit rather than a traceback.""" + try: + return api_post(endpoint, payload) + except Exception as error: + sys.exit(f"{endpoint}: {error}") + + +def read_sections(product, org): + return call("/market/sections/read", + {"id": int(product), "organizationId": org})["sections"] + + +def find_section(sections, group, title): + """Match on case and whitespace, because live titles carry typos and padding.""" + target = " ".join((title or "").split()).lower() + for section in sections.get(group, []): + if " ".join((section.get("title") or "").split()).lower() == target: + return section + return None + + +def masters(): + return call("/market/data/list")["list"] + + +# ---------------------------------------------------------------------------- rendering + +def render(entry, docs): + """The section body this page should hold: the live lead sentence, then the tables.""" + parts = [entry["lead"]] if entry["lead"] else [] + for name in entry["files"]: + path = docs / RESOURCES / name + if not path.exists(): + sys.exit(f"{path} does not exist. Run the table generators first.") + parts.append(path.read_text(encoding="utf-8").strip()) + return "\n".join(parts) + + +def symbols(html): + """The tickers a table lists, or the enum members a Futures list does. + + Byte comparison says *whether* a page is stale; this says *what* changed, which is the + part worth reading in a log. + """ + return set(_TICKER.findall(html)) or set(_ITEM.findall(html)) + + +def substitute_count(text, old, new): + """Swap the asset count in `text`, returning (text, hits). + + Both spellings are matched, because a page mixes them -- the Introduction says '3,320' + while a sidebar may say '3320'. `\\b` keeps 71 from matching inside 1971 or a ticker. + """ + hits = 0 + for spelling in dict.fromkeys([old, old.replace(",", "")]): + text, count = re.subn(rf"\b{re.escape(spelling)}\b", new, text) + hits += count + return text, hits + + +def count_updates(entry, sections, wanted, master): + """Sections and the `reach` field that still quote the old asset count. + + Returns (list of (title, new_content), (old_reach, new_reach) or None, note). + """ + if not entry["counts"]: + return [], None, None + + summary = find_section(sections, "about", "Data Summary") + match = _ASSET_COVERAGE.search((summary or {}).get("content") or "") + if not match: + return [], None, "no Data Summary asset count found; counts left alone" + + # Take the old count as the page writes it -- '3,320', not 3320 -- because that is the + # token to substitute. The surrounding sentence differs per page, and rewriting it + # would replace prose that was reviewed elsewhere. + old = match.group(1) + total = len(symbols(wanted)) + new = f"{total:,}" if "," in old or total >= 1000 else str(total) + if old == new: + return [], None, None + + updates = [] + for title in ("Introduction", "Data Summary"): + section = find_section(sections, "about", title) + if section is None: + continue + content, hits = substitute_count(section.get("content") or "", old, new) + if hits: + updates.append((section["title"].strip(), content)) + + reach = None + if master.get("reach"): + updated, hits = substitute_count(master["reach"], old, new) + if hits: + reach = (master["reach"], updated) + + return updates, reach, f"asset count {old} -> {new}" + + +# ------------------------------------------------------------------------------ the run + +def plan(docs, org): + """Everything each page needs, with nothing written yet.""" + by_id = {m["id"]: m for m in masters()} + rows = [] + for entry in MAPPING: + print(f"{entry['product']:>4} {entry['slug']} <- {', '.join(entry['files'])}") + master = by_id.get(entry["product"]) + if master is None: + sys.exit(f" product {entry['product']} is not in market/data/list") + if not master.get("editable"): + sys.exit(f" product {entry['product']} is not editable by this account") + + wanted = render(entry, docs) + sections = read_sections(entry["product"], org) + section = find_section(sections, "about", SECTION_TITLE) + if section is None: + # Creating it would append at the end of the group rather than in reading + # order, so this is a person's job, not an automated run's. + sys.exit(f" no {SECTION_TITLE!r} section on this page; add it by hand first") + + current = section.get("content") or "" + if current.strip() == wanted.strip(): + state = "in sync" + else: + have, want = symbols(current), symbols(wanted) + added, removed = sorted(want - have), sorted(have - want) + if added or removed: + state = f"stale, {len(have)} -> {len(want)} entries" + if added: + print(f" +{len(added)}: {', '.join(added[:12])}" + f"{' ...' if len(added) > 12 else ''}") + if removed: + print(f" -{len(removed)}: {', '.join(removed[:12])}" + f"{' ...' if len(removed) > 12 else ''}") + else: + state = f"markup differs, same {len(want)} entries" + + if entry.get("hold"): + print(f" {state}\n ON HOLD: {entry['hold']}") + continue + + updates, reach, note = count_updates(entry, sections, wanted, master) + if note: + print(f" {note}") + + planned = list(updates) + if current.strip() != wanted.strip(): + # Reading order: Introduction, Data Summary, then Supported Assets last. + planned.append((SECTION_TITLE, wanted)) + if not planned and not reach: + print(f" {state}") + continue + + print(f" {state}; writing {', '.join(t for t, _ in planned) or 'nothing'}" + + (f"; reach -> {reach[1]!r}" if reach else "")) + rows.append({"entry": entry, "sections": sections, "planned": planned, + "reach": reach}) + return rows + + +def write(rows, org): + """Write every planned section, then re-read and confirm. Returns the problems found.""" + problems = [] + for row in rows: + entry = row["entry"] + for title, content in row["planned"]: + section = find_section(row["sections"], "about", title) + call("/market/sections/update/", + {"sectionId": int(section["id"]), "title": section["title"], + "content": content}) + print(f" wrote {entry['slug']}/{title} ({len(content)}b)") + if row["reach"]: + call("/market/update-master/", + {"productId": int(entry["product"]), "reach": row["reach"][1]}) + print(f" wrote {entry['slug']}/reach = {row['reach'][1]!r}") + + # The envelope is not evidence: at least two of these endpoints report success while + # changing nothing, so everything is confirmed by reading it back. + by_id = {m["id"]: m for m in masters()} + for row in rows: + entry = row["entry"] + after = read_sections(entry["product"], org) + for title, content in row["planned"]: + section = find_section(after, "about", title) + if section is None: + problems.append(f"{entry['slug']}/{title}: missing after the write") + elif (section.get("content") or "") != content.strip(): + problems.append(f"{entry['slug']}/{title}: does not match what was sent") + for section_id, before in _flatten(row["sections"]).items(): + now = _flatten(after).get(section_id) + if now is None: + problems.append(f"{entry['slug']}: section {section_id} disappeared") + elif now.get("content") != before.get("content") and \ + before.get("title", "").strip() not in [t for t, _ in row["planned"]]: + problems.append(f"{entry['slug']}: section {section_id} " + f"({before.get('title')}) changed but was not planned") + if row["reach"] and by_id[entry["product"]].get("reach") != row["reach"][1]: + problems.append(f"{entry['slug']}: reach did not take") + return problems + + +def _flatten(sections): + return {s["id"]: s + for group in ("about", "documentation", "examples") + for s in sections.get(group, []) if s.get("id") is not None} + + +def main(): + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--docs", default=".", help="the Documentation repo root") + parser.add_argument("--dry-run", action="store_true", help="report, write nothing") + parser.add_argument("--snapshot", help="save the pre-change sections to this file") + args = parser.parse_args() + + org = get_organization_id() + rows = plan(Path(args.docs), org) + + if args.snapshot: + with open(args.snapshot, "w", encoding="utf-8") as fh: + json.dump({str(r["entry"]["product"]): r["sections"] for r in rows}, fh, indent=2) + print(f"\nsnapshot of {len(rows)} page(s) -> {args.snapshot}") + + if not rows: + print("\nnothing to do: every page already matches the repo") + return + if args.dry_run: + print(f"\ndry run: {len(rows)} page(s) would be written") + return + + print() + problems = write(rows, org) + print() + if problems: + print("VERIFICATION FAILED:") + for problem in problems: + print(f" - {problem}") + sys.exit(1) + print(f"verified: {len(rows)} page(s) match what was sent, nothing else changed") + + +if __name__ == "__main__": + main()