diff --git a/deploy/tests/test_caddy_page_scripts.py b/deploy/tests/test_caddy_page_scripts.py index c4a831b0..2aae8b17 100644 --- a/deploy/tests/test_caddy_page_scripts.py +++ b/deploy/tests/test_caddy_page_scripts.py @@ -1,5 +1,6 @@ """The generated survey pages load portal scripts (/src/fetchcmd.js, /src/fetchdialog.js, -/src/page-fetch.js) beside the SPA. Two Caddyfile facts make that safe, and both are pinned here +/src/page-fetch.js) beside the SPA, and the surveys hub loads /src/surveys-hub.js the same way. +Two Caddyfile facts make that safe, and both are pinned here POSITIVELY: not that some exclusion is absent, but that the rule which reaches /src/*.js and the rule which serves /surveys/ under the strict CSP are the rules the config actually declares. @@ -28,7 +29,7 @@ _REPO = Path(__file__).resolve().parents[2] _BOX_CADDY = _REPO / "deploy" / "docker" / "caddy" / "Caddyfile" -_SCRIPTS = ("/src/fetchcmd.js", "/src/fetchdialog.js", "/src/page-fetch.js") +_SCRIPTS = ("/src/fetchcmd.js", "/src/fetchdialog.js", "/src/page-fetch.js", "/src/surveys-hub.js") _PAGE = "/surveys/example-survey" diff --git a/docs/docs/developer/discoverability.md b/docs/docs/developer/discoverability.md index bc63b1fc..dda79e78 100644 --- a/docs/docs/developer/discoverability.md +++ b/docs/docs/developer/discoverability.md @@ -52,6 +52,12 @@ or curl command with the portal's own modules (`/src/fetchcmd.js`, `/src/fetchdi href is the SPA deep link `#/survey/?fetch=`, which opens the same dialog, and the card also links the document itself as "Pointers file (JSON)". +The surveys hub can be searched, filtered and sorted in the browser. The page renders its controls +hidden, with option lists drawn from the rows it lists and each card's facts as data attributes, +and loads one script, `/src/surveys-hub.js`, at the end of the body to reveal and drive them; the +view rides the URL query (`/surveys?type=GDS&sort=newest`), and without script the page is the +whole list in title order. + Station pages are deliberately unadvertised but served. Thousands of templated documents would read as thin content at scale and dilute the survey and collection pages that carry the ranking, so they declare `noindex` and stay out of the sitemap; they keep working for anyone following a published diff --git a/docs/docs/reference/portal-internals.md b/docs/docs/reference/portal-internals.md index eb48bf53..b90535d6 100644 --- a/docs/docs/reference/portal-internals.md +++ b/docs/docs/reference/portal-internals.md @@ -5099,6 +5099,35 @@ navigation and the page test replaces it to observe the hand-off. The card also itself as "Pointers file (JSON)", the no-script way to the urls. ``` +## portal/src/surveys-hub.js + +#### The surveys hub's search, filter and sort + +```text +The /surveys hub (engine/extract/_pages.py surveys_index_page) renders its controls as one hidden +GET form, #idxctl, on its own path: a search box (q) and selects for data type, organisation, +region, licence and sort, whose option lists are drawn from the rows themselves, so the page can +offer nothing the catalogue does not carry. Every card carries its facts as data attributes (slug, +title, org, region, stations, y0, y1, types, lic, doi), and the page loads this file at the end of +its body under the same strict CSP as the survey pages, script-src 'self' alone, so nothing is +inline. Without script the page is the whole list in title order and the form never shows. + +The driver reveals the form, reads the URL query into the controls (a value no option offers reads +as any), and on every input, change or submit keeps the cards whose title, organisation, region or +slug carry every search word and whose facts match each set select, sorts them (title; most +stations; newest or oldest by the stated years, a survey with no year last; organisation), hides +the rest, moves the kept cards to the front of the list, and writes the shown count into +#idxShown. The view is written back to the URL with history.replaceState, only the values that +differ from the page's defaults, so a bare /surveys stays canonical and a shared or reloaded URL +opens as it was left. Submit is answered in the page; a submit that reaches the server is a plain +reload. Clear (#idxReset, a link to /surveys hidden until something is set) empties the controls +in the page. There is no button anywhere: the hierarchy is catalogue, survey, data, and the only +way off the hub is a card's title. + +The engine's page test pins the ids, names, options and attributes this file reads, and +tests/surveys_hub.test.js drives the driver in jsdom against a fixture that mirrors them. +``` + ## portal/src/plots.js #### Pure SVG transfer-function plotters (no data/DOM dependency): ρ, φ ... diff --git a/engine/extract/_pages.py b/engine/extract/_pages.py index a01af1e6..c4acbdf0 100644 --- a/engine/extract/_pages.py +++ b/engine/extract/_pages.py @@ -709,6 +709,13 @@ def _inside(lon, lat): .idxlede{max-width:62ch;margin:.2rem 0 .1rem} .idxsum{color:#8FA3B0;font-size:.92rem;font-variant-numeric:tabular-nums;margin:.2rem 0 .2rem} .idxact{font-size:.9rem;margin:.2rem 0 1.1rem} + .idxctl{display:flex;flex-wrap:wrap;gap:.5rem .8rem;align-items:end;margin:.9rem 0 .2rem} + .idxctl label{display:flex;flex-direction:column;gap:.15rem;font-size:.78rem;color:#8FA3B0} + .idxctl .idxq{flex:1 1 16rem} + .idxctl input,.idxctl select{font:inherit;font-size:.9rem;color:#C9D4E8;background:#18213D;border:1px solid #2B3557;border-radius:6px;padding:.35rem .55rem;min-width:0} + .idxctl input:focus-visible,.idxctl select:focus-visible{outline:2px solid #EF7256;outline-offset:1px} + .idxctl .idxshown{flex:1 0 100%;margin:.2rem 0 .4rem;font-size:.85rem;color:#8FA3B0;font-variant-numeric:tabular-nums} + .idxctl .idxshown a{margin-left:.6rem} .idxlist{display:flex;flex-direction:column;gap:.7rem;margin:0 0 1rem} .idxcard{position:relative;display:grid;grid-template-columns:115px 1fr;gap:.9rem;align-items:start;background:#18213D;border:1px solid #2B3557;border-radius:8px;padding:.7rem .9rem} .idxcard svg{width:100%;height:auto;display:block} @@ -926,10 +933,17 @@ def _shell(*, title, description, canonical, body, jsonld=None, noindex=False, ) -def _survey_years(sm_doc, smeta): +def _survey_year_bounds(sm_doc, smeta): + """(year_start, year_end) as the survey record states them, the coverage block first and the + catalogue row second; None where neither states a year.""" cov = ((sm_doc or {}).get("dates") or {}).get("coverage") or {} y0 = cov.get("year_start") or (smeta or {}).get("year_start") y1 = cov.get("year_end") or (smeta or {}).get("year_end") + return (y0 or None, y1 or None) + + +def _survey_years(sm_doc, smeta): + y0, y1 = _survey_year_bounds(sm_doc, smeta) if y0 and y1: return f"{y0}" if y0 == y1 else _range(y0, y1) return str(y0 or y1 or "") @@ -2252,20 +2266,72 @@ def _hub_catalogue(*, name, description, url, base) -> dict: "keywords": _keywords()} +# The one script the surveys hub loads, same-origin under /src like the survey page's dialog +# modules: the pages are served under a CSP whose script-src is 'self' alone. +_HUB_SCRIPTS = ("/src/surveys-hub.js",) + +# The controls' sort orders, value then label, the first being the static order of the page. +_HUB_SORTS = (("title", "Title, A to Z"), ("stations", "Most stations"), + ("newest", "Newest first"), ("oldest", "Oldest first"), + ("org", "Organisation")) + + +def _card_data(fields) -> str: + """The data attributes a hub card carries, in a fixed order so the page bytes are stable. A None + value is written as an empty attribute: the script reads absence, never a placeholder.""" + return "".join(f' data-{k}="{_e("" if v is None else str(v))}"' for k, v in fields.items()) + + +def _hub_controls(vocab, *, n_rows) -> str: + """The search, filter and sort controls of the surveys hub, rendered HIDDEN. Without script the + page is the whole list and the form never shows; the script reveals it. The form is a GET to the + hub's own path, so a submit that reaches the server is a plain reload, and it carries no button: + the hierarchy is catalogue, survey, data, and the only way off this page is a card's title. + + Every option is a value some row carries, so the controls can never offer a filter the + catalogue cannot satisfy; the empty option is "any".""" + def select(name, label, values): + opts = "".join(f'' for v in values) + return (f'') + sorts = "".join(f'' for v, t in _HUB_SORTS) + return ( + '') + + def surveys_index_page(*, rows, base, build=None, og_image=None) -> str: """The /surveys hub: every published survey as one linked row with the facts a reader chooses on. Rendered from the catalogue rollups alone (mtcat.json / surveys.json), so it states nothing the served documents do not already publish and needs no survey-metadata read. - Rows carry: slug, title, org, region, n_stations, years, types {type: n}, period_min_s, - period_max_s, lic, doi, points [(lon, lat, type)]. The card is a DISCOVERY SUMMARY, not a - miniature survey record: no abstract, and exactly one action (the title link).""" + Rows carry: slug, title, org, region, n_stations, years, year_start, year_end, types {type: n}, + period_min_s, period_max_s, lic, doi, points [(lon, lat, type)]. The card is a DISCOVERY + SUMMARY, not a miniature survey record: no abstract, and exactly one action (the title link). + + The hub is searchable, filterable and sortable in the browser, and the page stays the whole + list without script: every card carries its facts as data attributes, the controls are rendered + HIDDEN with option lists drawn from the rows themselves (so the script can offer nothing the + catalogue does not carry), and one same-origin file, /src/surveys-hub.js, reveals and drives + them. The static order is by title, which is also the controls' default sort.""" base = (base or "").rstrip("/") url = f"{base}/surveys" rows = sorted(rows or [], key=lambda r: (str(r.get("title") or ""), str(r.get("slug") or ""))) n_stations = sum(int(r.get("n_stations") or 0) for r in rows) defs, ref = au_outline_defs(_INDEX_MAP_WIDTH) cards = [] + vocab = {"type": set(), "org": set(), "region": set(), "lic": set()} for r in rows: slug = str(r.get("slug") or "") title = str(r.get("title") or slug) @@ -2274,9 +2340,10 @@ def surveys_index_page(*, rows, base, build=None, og_image=None) -> str: pmin, pmax = r.get("period_min_s"), r.get("period_max_s") period = (f'{_range(_fmt_period(pmin), _fmt_period(pmax))} s' if pmin is not None and pmax is not None else "") + licence = _fmt_licence(r.get("lic")) facts = _card_facts_line([ _e(_plural(int(r.get("n_stations") or 0), "station")), - _e(type_bit), _e(str(r.get("years") or "")), _e(period), _e(_fmt_licence(r.get("lic"))), + _e(type_bit), _e(str(r.get("years") or "")), _e(period), _e(licence), 'DOI' if r.get("doi") else ""]) svg = _minimap_svg(r.get("points") or [], width=_INDEX_MAP_WIDTH, outline_ref=ref, label=f"{title} location in Australia") @@ -2287,12 +2354,30 @@ def surveys_index_page(*, rows, base, build=None, og_image=None) -> str: f'{_e(str(r.get("org") or ""))}' if r.get("org") else "", f'{_e(str(r.get("region") or ""))}' if r.get("region") else ""]) + org = str(r.get("org") or "") + region = str(r.get("region") or "") + vocab["type"].update(str(t) for t in types) + if org: + vocab["org"].add(org) + if region: + vocab["region"].add(region) + if licence: + vocab["lic"].add(licence) + # The facts the controls read, one attribute each, in the card's own values: the script + # never parses the printed line. A year the record does not state is an empty attribute. + data = _card_data({ + "slug": slug, "title": title, "org": org, "region": region, + "stations": int(r.get("n_stations") or 0), + "y0": r.get("year_start"), "y1": r.get("year_end"), + "types": " ".join(str(t) for t in types), "lic": licence, + "doi": "1" if r.get("doi") else ""}) cards.append( - f'
{svg}
' + f'
{svg}
' f'

{_e(title)}

' f'

{org_line}

' f'

{facts}

' f'{_CARD_ARROW}
') + controls = _hub_controls(vocab, n_rows=len(rows)) # The page-level counts go through _plural like the card counts do: a corpus of one is a real # state (it is where every new deployment starts), and the summary line and the description are # the two strings a reader and a search result actually read. @@ -2310,6 +2395,7 @@ def surveys_index_page(*, rows, base, build=None, og_image=None) -> str: "

Surveys

\n" f'

{summary}

\n' f'

{_SURVEYS_LEDE}

\n' + f"{controls}\n" f"{defs}\n" f'
{"".join(cards)}
\n') return _shell(title="Surveys - magnetotelluric survey data - AusMT", @@ -2318,7 +2404,7 @@ def surveys_index_page(*, rows, base, build=None, og_image=None) -> str: base=base), _breadcrumb(base, [(_SITE_NAME, "/"), ("surveys", "/surveys")])], extra_css=_INDEX_CSS, nav="navSurveys", build=build, status=counts, - og_image=og_image) + og_image=og_image, scripts=_HUB_SCRIPTS) def collections_index_page(*, rows, base, build=None, og_image=None) -> str: @@ -3193,6 +3279,8 @@ def emit_pages(out, base, *, surveys_meta, survey_docs, station_docs, collection "n_stations": _drow.get("n_stations") if _drow.get("n_stations") is not None else len(docs), "years": _survey_years(survey_docs.get(slug), smeta), + "year_start": _survey_year_bounds(survey_docs.get(slug), smeta)[0], + "year_end": _survey_year_bounds(survey_docs.get(slug), smeta)[1], "types": _drow.get("data_types"), "period_min_s": _drow.get("period_min_s"), "period_max_s": _drow.get("period_max_s"), diff --git a/engine/tests/test_index_pages.py b/engine/tests/test_index_pages.py index 8aea7c41..0cd02430 100644 --- a/engine/tests/test_index_pages.py +++ b/engine/tests/test_index_pages.py @@ -7,8 +7,10 @@ These pages close that. They are emitted by the same emitter and under the same flag as the entity pages, render ONLY from the catalogue rollups (mtcat.json / surveys.json / the collections rollup), -and carry no script and no external asset, so the build stays offline and deterministic and the -served document needs nothing but itself. +and carry no inline script and no external asset, so the build stays offline and deterministic and +the served document needs nothing but itself. The surveys hub loads ONE same-origin file at the end +of its body, /src/surveys-hub.js, which reveals and drives the search, filter and sort controls the +page renders hidden; without it the page is the whole list, in title order. The surveys index carries 27 minimaps today and grows with the corpus. Emitting the Australian outline once as a and referencing it from every card is what keeps the document small @@ -128,7 +130,7 @@ def test_surveys_index_marks_a_doi_only_where_the_rollup_carries_one(built): """FAILS IF the DOI marker is invented for a survey without one (or dropped for one with). The rollup is the only source; nothing here derives or reserves an identifier.""" page = (built / "pages" / "surveys" / "index.html").read_text(encoding="utf-8") - rows = re.findall(r'
.*?
', page, re.S) + rows = re.findall(r'
]*>.*?
', page, re.S) assert len(rows) == 2, f"one card per survey, got {len(rows)}" by_slug = {("idx-a" if "/surveys/idx-a" in r else "idx-b"): r for r in rows} assert "DOI" in by_slug["idx-a"], "the survey declaring a DOI must carry the marker" @@ -189,14 +191,23 @@ def test_index_pages_ride_the_sitemap_flag(tmp_path): # pages are served under a CSP whose script-src is 'self' alone. FETCH_SCRIPT_SRCS = ["/src/fetchcmd.js", "/src/fetchdialog.js", "/src/page-fetch.js"] +# The surveys hub's one script, on the same terms: same-origin under /src, at the end of the body, +# and only on the page that renders the controls it drives. +HUB_SCRIPT_SRCS = ["/src/surveys-hub.js"] + + +def _page_scripts(page): + """The script files a page may load, from what its own body carries.""" + scripts = FETCH_SCRIPT_SRCS if 'id="wgetModal"' in page else [] + return scripts + (HUB_SCRIPT_SRCS if 'id="idxctl"' in page else []) + def _allowed_srcs(rel, page=""): """The exact, ordered src list this page kind may carry, in DOCUMENT order: the two header marks, - then whatever body mark the kind is named for, then the footer's lockup, then the dialog's three - scripts on a page that carries the dialog. The order is the assertion, so the body mark has to be - spliced between the two chrome groups rather than appended after them.""" - scripts = FETCH_SCRIPT_SRCS if 'id="wgetModal"' in page else [] - return ALLOWED_HEADER_SRCS + ALLOWED_BODY_SRCS.get(rel, []) + ALLOWED_FOOTER_SRCS + scripts + then whatever body mark the kind is named for, then the footer's lockup, then the scripts the + page's body earns (the dialog's three, the hub's one). The order is the assertion, so the body + mark has to be spliced between the two chrome groups rather than appended after them.""" + return ALLOWED_HEADER_SRCS + ALLOWED_BODY_SRCS.get(rel, []) + ALLOWED_FOOTER_SRCS + _page_scripts(page) def _page_srcs(page, rel): @@ -210,18 +221,21 @@ def _page_srcs(page, rel): def test_index_pages_carry_no_script_and_only_the_identity_mark(built): - """FAILS IF an index page grows a script, a stylesheet link, or any fetched asset beyond the one - allow-listed identity mark. The entity pages' determinism posture (stdlib-only build, everything - else inline, no network at build) is inherited, not re-litigated; the mark is the single named - exception and it is same-origin. + """FAILS IF an index page grows an inline script, a stylesheet link, or any fetched asset beyond + the allow-listed identity mark and, on the surveys hub alone, its one same-origin controls script. + The entity pages' determinism posture (stdlib-only build, everything else inline, no network at + build) is inherited, not re-litigated; the mark is the single named exception and it is + same-origin, and so is the hub's script. A ld+json block is data, not code: nothing in it executes, and the hub pages carry one so a search result can show the trail back to the root. It is stripped before the guard rather than - excused, so an EXECUTABLE script on a hub still fails on the same line.""" + excused, so an EXECUTABLE inline script on a hub still fails on the same line.""" for rel in ("surveys/index.html", "collections/index.html"): page = (built / "pages" / rel).read_text(encoding="utf-8") - assert "', ""), \ - f"{rel}: no executable script may appear on an index page" + for tag in re.findall(r"]*>", page): + assert tag == '\n\n') or \ + page.count('') == 1, "the hub loads its one script" + assert page.index('