diff --git a/.github/workflows/code_generators.yml b/.github/workflows/code_generators.yml index 9ca9fd8866..96d1ff65c5 100644 --- a/.github/workflows/code_generators.yml +++ b/.github/workflows/code_generators.yml @@ -26,6 +26,7 @@ jobs: env: QUANTCONNECT_USER_ID: ${{ secrets.QUANTCONNECT_USER_ID }} QUANTCONNECT_API_TOKEN: ${{ secrets.QUANTCONNECT_API_TOKEN }} + QUANTCONNECT_ORGANIZATION_ID: ${{ secrets.QUANTCONNECT_ORGANIZATION_ID }} run: | failed="" for script in \ diff --git a/.github/workflows/sync_supported_assets.yml b/.github/workflows/sync_supported_assets.yml new file mode 100644 index 0000000000..20c86c58e4 --- /dev/null +++ b/.github/workflows/sync_supported_assets.yml @@ -0,0 +1,53 @@ +name: Sync Dataset Page Supported Assets + +on: + pull_request: + branches: [master] + paths: + - 'Resources/datasets/supported-securities/**' + - 'code-generators/sync_supported_assets.py' + - '.github/workflows/sync_supported_assets.yml' + push: + branches: [master] + paths: + - 'Resources/datasets/supported-securities/**' + - 'code-generators/sync_supported_assets.py' + - '.github/workflows/sync_supported_assets.yml' + workflow_dispatch: + inputs: + dry_run: + description: 'Report what would change without writing' + type: boolean + default: true + +jobs: + sync: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: Sync the dataset pages + env: + QUANTCONNECT_USER_ID: ${{ secrets.QUANTCONNECT_USER_ID }} + QUANTCONNECT_API_TOKEN: ${{ secrets.QUANTCONNECT_API_TOKEN }} + QUANTCONNECT_ORGANIZATION_ID: ${{ secrets.QUANTCONNECT_ORGANIZATION_ID }} + run: | + if [[ -z "$QUANTCONNECT_API_TOKEN" ]]; then + echo "::notice::No QuantConnect credentials available; skipping the sync." + exit 0 + fi + if [[ "${{ github.event_name }}" == "push" || "${{ inputs.dry_run }}" == "false" ]]; then + python code-generators/sync_supported_assets.py --snapshot snapshot.json + else + python code-generators/sync_supported_assets.py --dry-run --snapshot snapshot.json + fi + + - name: Upload the pre-change sections + if: always() + uses: actions/upload-artifact@v4 + with: + name: dataset-page-snapshot + path: snapshot.json + if-no-files-found: ignore + retention-days: 90 diff --git a/code-generators/Alternative-Datasets-Code-Generator.py b/code-generators/Alternative-Datasets-Code-Generator.py index 4a870e0b64..1660f186a1 100644 --- a/code-generators/Alternative-Datasets-Code-Generator.py +++ b/code-generators/Alternative-Datasets-Code-Generator.py @@ -4,10 +4,8 @@ from itertools import groupby from shutil import move, rmtree from bs4 import BeautifulSoup -from urllib.request import urlopen -import json -from _code_generation_helpers import generate_landing_page +from _code_generation_helpers import generate_landing_page, get_dataset_listings DATASET = "03 Writing Algorithms/14 Datasets" @@ -115,8 +113,7 @@ def _parse_content(content): return content if __name__ == '__main__': - url = "https://s3.amazonaws.com/cdn.quantconnect.com/web/docs/alternative-data-dump-v2024-01-02.json" - docs = json.loads(urlopen(url).read().decode('utf-8')) + docs = get_dataset_listings() for d in docs: if d["vendorName"].strip() == "CoinAPI": d["vendorName"] = "QuantConnect" diff --git a/code-generators/Alternative-Datasets-Skill-Generator.py b/code-generators/Alternative-Datasets-Skill-Generator.py index 0b267eba7b..8278f854a0 100644 --- a/code-generators/Alternative-Datasets-Skill-Generator.py +++ b/code-generators/Alternative-Datasets-Skill-Generator.py @@ -1,9 +1,9 @@ """Generate the alternative-data and alternative-data-universes SKILL.md files. -Walks the QC alternative-data dump JSON, scrapes `add_data` / `add_universe` -class references and accessing-data list markers from each dataset's section -content, then enriches the alt-data skill with C#/Python property names -fetched from the QC inspector in parallel. +Walks the dataset listings read live from the QuantConnect API, scrapes +`add_data` / `add_universe` class references and accessing-data list markers +from each dataset's section content, then enriches the alt-data skill with +C#/Python property names fetched from the QC inspector in parallel. Page generation lives in `Alternative-Datasets-Code-Generator.py`; this file only writes the two SKILL.md files. @@ -15,8 +15,9 @@ class references and accessing-data list markers from each dataset's section from pathlib import Path from urllib.request import urlopen +from _code_generation_helpers import get_dataset_listings + -DUMP_URL = "https://s3.amazonaws.com/cdn.quantconnect.com/web/docs/alternative-data-dump-v2024-01-02.json" INSPECTOR_URL = ( "https://www.quantconnect.com/services/inspector" "?language={lang}&type=T:QuantConnect.DataSource.{cls}" @@ -286,7 +287,7 @@ def _build_alt_data_skill(skill_data, props_lookup=None): return "\n".join(parts) + "\n" -# === Dump traversal === +# === Listing traversal === def _ordered_sections(dataset): """Flatten the dataset's about + documentation entries into a {title: content} dict.""" @@ -340,8 +341,8 @@ def _scrape_skill_data(docs): def main(): - print(f"Downloading alt-data dump from {DUMP_URL}") - docs = json.loads(urlopen(DUMP_URL).read().decode("utf-8")) + print("Reading the dataset listings from the QuantConnect API...") + docs = get_dataset_listings() alt_universe_skill_data, alt_data_skill_data = _scrape_skill_data(docs) diff --git a/code-generators/_code_generation_helpers.py b/code-generators/_code_generation_helpers.py index 6775561e77..bd038d495b 100644 --- a/code-generators/_code_generation_helpers.py +++ b/code-generators/_code_generation_helpers.py @@ -1,7 +1,12 @@ -from json import dumps +from base64 import b64encode +from concurrent.futures import ThreadPoolExecutor +from hashlib import sha256 +from json import dumps, loads +from os import environ from re import findall, finditer +from time import time from typing import List -from urllib.request import urlopen +from urllib.request import Request, urlopen WRITING_ALGORITHMS = '03 Writing Algorithms' INDICATORS = f'{WRITING_ALGORITHMS}/28 Indicators/01 Supported Indicators' @@ -48,6 +53,93 @@ def get_lean_cli_commands() -> dict: commands[name] = desc.rstrip('.') return commands +QC_API = 'https://www.quantconnect.com/api/v2' +SECTION_GROUPS = ('about', 'documentation', 'examples') + +def get_api_credentials() -> tuple: + """User id and API token from the environment.""" + user_id, api_token = environ.get('QUANTCONNECT_USER_ID'), environ.get('QUANTCONNECT_API_TOKEN') + if not user_id or not api_token: + raise RuntimeError('Set QUANTCONNECT_USER_ID and QUANTCONNECT_API_TOKEN. In Actions ' + 'they are empty on a pull request from a fork.') + return str(user_id), api_token + +def api_post(endpoint: str, payload: dict = None) -> dict: + """POST to the QuantConnect API v2 and return the envelope. Raises on failure.""" + user_id, api_token = get_api_credentials() + timestamp = str(int(time())) + hashed_token = sha256(f'{api_token}:{timestamp}'.encode('utf-8')).hexdigest() + auth = b64encode(f'{user_id}:{hashed_token}'.encode('utf-8')).decode('ascii') + request = Request(f'{QC_API}{endpoint}', data=dumps(payload or {}).encode('utf-8'), + headers={'Authorization': f'Basic {auth}', 'Timestamp': timestamp, + 'Content-Type': 'application/json'}) + body = loads(urlopen(request, timeout=120).read()) + if not body.get('success'): + raise RuntimeError(f'{endpoint} failed: {body.get("errors")}') + return body + +def get_organization_id() -> str: + """The organization id that market/sections/read requires on every call. + + Any organization the account belongs to works; it need not own the dataset. Omitting + it fails with the unhelpful 'Organization not found. '. + """ + if environ.get('QUANTCONNECT_ORGANIZATION_ID'): + return environ['QUANTCONNECT_ORGANIZATION_ID'] + organizations = api_post('/organizations/list').get('organizations', []) + if not organizations: + raise RuntimeError('The account belongs to no organizations.') + print(f'note: QUANTCONNECT_ORGANIZATION_ID unset, using {organizations[0]["id"]} ' + f'({organizations[0].get("name")})') + return organizations[0]['id'] + +DATASET_DUMP = 'https://s3.amazonaws.com/cdn.quantconnect.com/web/docs/alternative-data-dump-v2024-01-02.json' + +def get_public_dataset_slugs() -> set: + """Which datasets are public, per the dump. Nothing in the API records this.""" + return {d['url'].rsplit('/', 1)[-1] for d in loads(get_text_content(DATASET_DUMP))} + +def get_dataset_listings() -> List[dict]: + """Every public dataset listing, with its page sections read live from the API. + + The dump says which datasets are public; the API says what each page holds, so edits + to a listing reach the docs without waiting for a dump re-upload. Each record carries + what the dataset generators use: + + {"name": ..., "vendorName": ..., "url": "/datasets/", + "about": [{"title": ..., "content": ...}, ...], "documentation": [...], + "examples": [...]} + + Drift between the two sources is reported, not applied -- either direction needs a + person to decide. + """ + organization_id = get_organization_id() + public = get_public_dataset_slugs() + masters = {m['url']: m for m in api_post('/market/data/list')['list'] if not m.get('pending')} + + for slug in sorted(set(masters) - public): + print(f'note: {slug} is a live listing but is not in the dump, so it gets no docs ' + f'page. Refresh the dump if it should be public.') + for slug in sorted(public - set(masters)): + print(f'note: {slug} is in the dump but no longer a listing; it has been retired.') + + def read(master): + sections = api_post('/market/sections/read', + {'id': master['id'], 'organizationId': organization_id})['sections'] + listing = {'name': master.get('name'), 'vendorName': master.get('vendorName'), + 'url': f'/datasets/{master.get("url")}'} + for group in SECTION_GROUPS: + # Sort by position: the generators index into `about` for the vendor landing + # page, so reading order is load-bearing, not cosmetic. + listing[group] = [{'title': s.get('title'), 'content': s.get('content')} + for s in sorted(sections.get(group) or [], + key=lambda s: s.get('position') or 0)] + return listing + + masters = [m for slug, m in masters.items() if slug in public] + with ThreadPoolExecutor(8) as executor: + return list(executor.map(read, masters)) + def get_json_content(url: str) -> List: content = get_text_content(url) \ .replace("null", "None").replace("true", "True").replace("false", "False") diff --git a/code-generators/sync_supported_assets.py b/code-generators/sync_supported_assets.py new file mode 100644 index 0000000000..19f237f774 --- /dev/null +++ b/code-generators/sync_supported_assets.py @@ -0,0 +1,359 @@ +"""Push the generated supported-securities tables onto the QuantConnect dataset pages. + +`Supported-Assets-Table-Code-Generator.py` and `future-table-code-generator.py` regenerate +`Resources/datasets/supported-securities/` from the symbol-properties database. The same +tables are the body of the "Supported Assets" section on a handful of dataset pages, where +they used to be pasted by hand -- so every regeneration left the website behind, silently. +This closes that gap: it renders each page's section from the repo files and writes it +through the API v2 `market/sections` endpoints. + + python code-generators/sync_supported_assets.py --dry-run + python code-generators/sync_supported_assets.py --snapshot snapshot.json + +`--dry-run` reads the live pages, reports what it would change, and writes nothing. Without +it the script writes, then re-reads and diffs to confirm -- `success: true` from these +endpoints does not mean the write happened, so the envelope is never taken as evidence. +Exit status is non-zero if any page ends up different from what was sent. + +Credentials come from `QUANTCONNECT_USER_ID` / `QUANTCONNECT_API_TOKEN`, or from +`~/.lean/credentials`. `QUANTCONNECT_ORGANIZATION_ID` is required on every read; any +organization the account belongs to works, and the first one is used if the variable is +unset. + +The asset count +--------------- +A page states its size in four places, and pushing only the table leaves three of them +contradicting it: the table header, the Introduction sentence, the Data Summary "Asset +Coverage" row, and the sidebar Coverage field (`reach`). The other three are rewritten by +substituting the *number*, never the sentence, so reviewed wording survives. + +Guard-rails +----------- +This only ever overwrites the sections named in `MAPPING`; it never creates one, and a +missing section is an error rather than something to fill in. A page can therefore not be +restructured by an automated run, only refreshed. +""" + +import argparse +import json +import re +import sys +from pathlib import Path + +from _code_generation_helpers import api_post, get_organization_id + +RESOURCES = Path("Resources/datasets/supported-securities") +SECTION_TITLE = "Supported Assets" + +CRYPTO_LEAD = "

The following table shows the available Cryptocurrency pairs:

" +CRYPTO_FUTURE_LEAD = "

The following table shows the available Crypto Future pairs:

" + +# One entry per dataset page whose "Supported Assets" section is a copy of these files. +# `lead` is the sentence already on the live page -- keep it verbatim rather than rewriting +# prose reviewed elsewhere. The Futures files open with their own "(N) Futures" sentence, +# so they take an empty lead and carry their count with them (`counts: False`). +# +# Deliberately absent: +# * cryptofuture/dydx.html -- no dYdX dataset listing exists +# * cfd/interactivebrokers.html, forex/interactivebrokers.html +# -- datasets 25 and 26 are QuantConnect's OANDA-sourced +# CFD/Forex data; the IB tables belong to the Writing +# Algorithms docs, not to these listings +# * 118 tickdata-international-futures +# -- its table is hand-written, not generated +MAPPING = [ + {"product": 58, "slug": "binance-crypto-price-data", + "files": ["crypto/binance.html"], "lead": CRYPTO_LEAD}, + {"product": 76, "slug": "binance-us-crypto-price-data", + "files": ["crypto/binanceus.html"], "lead": CRYPTO_LEAD}, + {"product": 57, "slug": "bitfinex-crypto-price-data", + "files": ["crypto/bitfinex.html"], "lead": CRYPTO_LEAD}, + {"product": 96, "slug": "bybit-crypto-price-data", + "files": ["crypto/bybit.html"], "lead": CRYPTO_LEAD}, + {"product": 27, "slug": "coinbase-crypto-price-data", + "files": ["crypto/coinbase.html"], "lead": CRYPTO_LEAD}, + {"product": 60, "slug": "kraken-crypto-price-data", + "files": ["crypto/kraken.html"], "lead": CRYPTO_LEAD}, + + {"product": 85, "slug": "binance-cryptofuture-price-data", + "files": ["cryptofuture/binance.html"], "lead": CRYPTO_FUTURE_LEAD}, + {"product": 87, "slug": "binance-cryptofuture-margin-rate-data", + "files": ["cryptofuture/binance.html"], "lead": CRYPTO_FUTURE_LEAD}, + {"product": 97, "slug": "bybit-cryptofuture-price-data", + "files": ["cryptofuture/bybit.html"], "lead": CRYPTO_FUTURE_LEAD}, + {"product": 98, "slug": "bybit-cryptofuture-margin-rate-data", + "files": ["cryptofuture/bybit.html"], "lead": CRYPTO_FUTURE_LEAD}, + + {"product": 25, "slug": "quantconnect-forex", + "files": ["forex/oanda.html"], + "lead": "

The following table shows the available Forex pairs:

"}, + {"product": 26, "slug": "quantconnect-cfd", + "files": ["cfd/oanda.html"], + "lead": "

The following table shows the available contracts:

"}, + + {"product": 30, "slug": "algoseek-us-futures", + "files": ["future/supported-contracts.html"], "lead": "", "counts": False}, + {"product": 31, "slug": "algoseek-us-future-options", + "files": ["futureoption/supported-contracts.html"], "lead": "", "counts": False, + "hold": "the generated file says '(16) Futures Options' above 15 items -- " + "future-table-code-generator.py counts len(FUTURE_OPTIONS) while DC is " + "dropped, because 'Dairy' is missing from its category list. The live " + "'(15)' is the correct number; fix the generator before syncing."}, +] + +for _entry in MAPPING: + _entry.setdefault("counts", True) + +_TICKER = re.compile(r"([^<]+)") +_ITEM = re.compile(r"
  • ]*>([^<]+)") +_ASSET_COVERAGE = re.compile(r"Asset Coverage.*?\s*([\d,]+)\b", re.S) + + +# --------------------------------------------------------------------------- transport + +def call(endpoint, payload=None): + """POST to the API, turning a failure into a clean exit rather than a traceback.""" + try: + return api_post(endpoint, payload) + except Exception as error: + sys.exit(f"{endpoint}: {error}") + + +def read_sections(product, org): + return call("/market/sections/read", + {"id": int(product), "organizationId": org})["sections"] + + +def find_section(sections, group, title): + """Match on case and whitespace, because live titles carry typos and padding.""" + target = " ".join((title or "").split()).lower() + for section in sections.get(group, []): + if " ".join((section.get("title") or "").split()).lower() == target: + return section + return None + + +def masters(): + return call("/market/data/list")["list"] + + +# ---------------------------------------------------------------------------- rendering + +def render(entry, docs): + """The section body this page should hold: the live lead sentence, then the tables.""" + parts = [entry["lead"]] if entry["lead"] else [] + for name in entry["files"]: + path = docs / RESOURCES / name + if not path.exists(): + sys.exit(f"{path} does not exist. Run the table generators first.") + parts.append(path.read_text(encoding="utf-8").strip()) + return "\n".join(parts) + + +def symbols(html): + """The tickers a table lists, or the enum members a Futures list does. + + Byte comparison says *whether* a page is stale; this says *what* changed, which is the + part worth reading in a log. + """ + return set(_TICKER.findall(html)) or set(_ITEM.findall(html)) + + +def substitute_count(text, old, new): + """Swap the asset count in `text`, returning (text, hits). + + Both spellings are matched, because a page mixes them -- the Introduction says '3,320' + while a sidebar may say '3320'. `\\b` keeps 71 from matching inside 1971 or a ticker. + """ + hits = 0 + for spelling in dict.fromkeys([old, old.replace(",", "")]): + text, count = re.subn(rf"\b{re.escape(spelling)}\b", new, text) + hits += count + return text, hits + + +def count_updates(entry, sections, wanted, master): + """Sections and the `reach` field that still quote the old asset count. + + Returns (list of (title, new_content), (old_reach, new_reach) or None, note). + """ + if not entry["counts"]: + return [], None, None + + summary = find_section(sections, "about", "Data Summary") + match = _ASSET_COVERAGE.search((summary or {}).get("content") or "") + if not match: + return [], None, "no Data Summary asset count found; counts left alone" + + # Take the old count as the page writes it -- '3,320', not 3320 -- because that is the + # token to substitute. The surrounding sentence differs per page, and rewriting it + # would replace prose that was reviewed elsewhere. + old = match.group(1) + total = len(symbols(wanted)) + new = f"{total:,}" if "," in old or total >= 1000 else str(total) + if old == new: + return [], None, None + + updates = [] + for title in ("Introduction", "Data Summary"): + section = find_section(sections, "about", title) + if section is None: + continue + content, hits = substitute_count(section.get("content") or "", old, new) + if hits: + updates.append((section["title"].strip(), content)) + + reach = None + if master.get("reach"): + updated, hits = substitute_count(master["reach"], old, new) + if hits: + reach = (master["reach"], updated) + + return updates, reach, f"asset count {old} -> {new}" + + +# ------------------------------------------------------------------------------ the run + +def plan(docs, org): + """Everything each page needs, with nothing written yet.""" + by_id = {m["id"]: m for m in masters()} + rows = [] + for entry in MAPPING: + print(f"{entry['product']:>4} {entry['slug']} <- {', '.join(entry['files'])}") + master = by_id.get(entry["product"]) + if master is None: + sys.exit(f" product {entry['product']} is not in market/data/list") + if not master.get("editable"): + sys.exit(f" product {entry['product']} is not editable by this account") + + wanted = render(entry, docs) + sections = read_sections(entry["product"], org) + section = find_section(sections, "about", SECTION_TITLE) + if section is None: + # Creating it would append at the end of the group rather than in reading + # order, so this is a person's job, not an automated run's. + sys.exit(f" no {SECTION_TITLE!r} section on this page; add it by hand first") + + current = section.get("content") or "" + if current.strip() == wanted.strip(): + state = "in sync" + else: + have, want = symbols(current), symbols(wanted) + added, removed = sorted(want - have), sorted(have - want) + if added or removed: + state = f"stale, {len(have)} -> {len(want)} entries" + if added: + print(f" +{len(added)}: {', '.join(added[:12])}" + f"{' ...' if len(added) > 12 else ''}") + if removed: + print(f" -{len(removed)}: {', '.join(removed[:12])}" + f"{' ...' if len(removed) > 12 else ''}") + else: + state = f"markup differs, same {len(want)} entries" + + if entry.get("hold"): + print(f" {state}\n ON HOLD: {entry['hold']}") + continue + + updates, reach, note = count_updates(entry, sections, wanted, master) + if note: + print(f" {note}") + + planned = list(updates) + if current.strip() != wanted.strip(): + # Reading order: Introduction, Data Summary, then Supported Assets last. + planned.append((SECTION_TITLE, wanted)) + if not planned and not reach: + print(f" {state}") + continue + + print(f" {state}; writing {', '.join(t for t, _ in planned) or 'nothing'}" + + (f"; reach -> {reach[1]!r}" if reach else "")) + rows.append({"entry": entry, "sections": sections, "planned": planned, + "reach": reach}) + return rows + + +def write(rows, org): + """Write every planned section, then re-read and confirm. Returns the problems found.""" + problems = [] + for row in rows: + entry = row["entry"] + for title, content in row["planned"]: + section = find_section(row["sections"], "about", title) + call("/market/sections/update/", + {"sectionId": int(section["id"]), "title": section["title"], + "content": content}) + print(f" wrote {entry['slug']}/{title} ({len(content)}b)") + if row["reach"]: + call("/market/update-master/", + {"productId": int(entry["product"]), "reach": row["reach"][1]}) + print(f" wrote {entry['slug']}/reach = {row['reach'][1]!r}") + + # The envelope is not evidence: at least two of these endpoints report success while + # changing nothing, so everything is confirmed by reading it back. + by_id = {m["id"]: m for m in masters()} + for row in rows: + entry = row["entry"] + after = read_sections(entry["product"], org) + for title, content in row["planned"]: + section = find_section(after, "about", title) + if section is None: + problems.append(f"{entry['slug']}/{title}: missing after the write") + elif (section.get("content") or "") != content.strip(): + problems.append(f"{entry['slug']}/{title}: does not match what was sent") + for section_id, before in _flatten(row["sections"]).items(): + now = _flatten(after).get(section_id) + if now is None: + problems.append(f"{entry['slug']}: section {section_id} disappeared") + elif now.get("content") != before.get("content") and \ + before.get("title", "").strip() not in [t for t, _ in row["planned"]]: + problems.append(f"{entry['slug']}: section {section_id} " + f"({before.get('title')}) changed but was not planned") + if row["reach"] and by_id[entry["product"]].get("reach") != row["reach"][1]: + problems.append(f"{entry['slug']}: reach did not take") + return problems + + +def _flatten(sections): + return {s["id"]: s + for group in ("about", "documentation", "examples") + for s in sections.get(group, []) if s.get("id") is not None} + + +def main(): + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--docs", default=".", help="the Documentation repo root") + parser.add_argument("--dry-run", action="store_true", help="report, write nothing") + parser.add_argument("--snapshot", help="save the pre-change sections to this file") + args = parser.parse_args() + + org = get_organization_id() + rows = plan(Path(args.docs), org) + + if args.snapshot: + with open(args.snapshot, "w", encoding="utf-8") as fh: + json.dump({str(r["entry"]["product"]): r["sections"] for r in rows}, fh, indent=2) + print(f"\nsnapshot of {len(rows)} page(s) -> {args.snapshot}") + + if not rows: + print("\nnothing to do: every page already matches the repo") + return + if args.dry_run: + print(f"\ndry run: {len(rows)} page(s) would be written") + return + + print() + problems = write(rows, org) + print() + if problems: + print("VERIFICATION FAILED:") + for problem in problems: + print(f" - {problem}") + sys.exit(1) + print(f"verified: {len(rows)} page(s) match what was sent, nothing else changed") + + +if __name__ == "__main__": + main()