Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
98 changes: 71 additions & 27 deletions scripts/build_frontend_data.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,26 @@
"""Build landing/data-exports.js from data/exports/*.json.
"""Build the frontend's copy of the exported books from data/exports/*.json.

Converts each exported book (the real, pipeline-produced knowledge graph) into
the record shape the landing page's window.LG_DATA expects, computes a generic
the record shape the landing page's window.LG_DATA expects and computes a generic
graph layout (phyllotaxis by degree, so any book lays out without curated
coordinates), and emits a single JS file that merges everything into LG_DATA
after data.js has loaded.
coordinates).

Two outputs, because the graphs are large and the app does not need them to draw
its own first screen:

landing/data-exports.js shelf metadata + the manifest
landing/assets/exports/<id>.json one book's entities, edges, glucose,
chunks and localized entity names

The JS file is small and loads with the app. It merges each book's metadata over
the manifest placeholder in data.js — which is all the library grid needs — and
publishes window.LG_BOOK_EXPORTS so the app knows which books have a graph to
fetch. The per-book JSON is fetched only when that book is opened, by
window.LG_LOAD_BOOK in app.jsx.

It used to be one JS file of everything, and it had grown to 11 MB: 90% of a
12.5 MB app bundle, parsed before the first paint, for two books out of 85 — and
the app opens on the library grid, which shows neither of them expanded.

uv run python scripts/build_frontend_data.py
"""
Expand All @@ -20,7 +36,12 @@

ROOT = Path(__file__).resolve().parent.parent
EXPORTS_DIR = ROOT / "data" / "exports"
OUT = ROOT / "src" / "loregraph" / "web" / "landing" / "data-exports.js"
LANDING = ROOT / "src" / "loregraph" / "web" / "landing"
OUT = LANDING / "data-exports.js"
# Under assets/ because build-landing.cjs copies that directory into dist/
# verbatim, and serve-landing.cjs serves the source tree at the same path — so the
# fetch URL is /assets/exports/<id>.json in a local preview and in production.
OUT_BOOKS = LANDING / "assets" / "exports"

# Per-book display metadata the export JSON doesn't carry (localized title, year,
# spine colour). Falls back to sensible defaults for unknown books.
Expand Down Expand Up @@ -601,30 +622,54 @@ def main() -> int:
if not files:
raise SystemExit(f"no *.json in {EXPORTS_DIR}")

books, entities, edges, glucose, chunks = [], [], [], [], []
entity_locale: dict[str, dict] = {}
def j(x: object) -> str:
return json.dumps(x, ensure_ascii=False, separators=(",", ":"))

OUT_BOOKS.mkdir(parents=True, exist_ok=True)
for stale in OUT_BOOKS.glob("*.json"):
stale.unlink()

books: list[dict] = []
for f in files:
payload = json.loads(f.read_text(encoding="utf-8"))
conv = convert(payload)
books.append(conv["book"])
entities += conv["entities"]
edges += conv["edges"]
glucose += conv["glucose"]
chunks += conv["chunks"]
book = conv["book"]
bid = book["id"]

i18n_path = f.with_name(f.stem + ".i18n.json")
if i18n_path.exists():
entity_locale.update(json.loads(i18n_path.read_text(encoding="utf-8")))
print(
f" {f.name}: {len(conv['entities'])} entities, {len(conv['edges'])} edges, i18n={i18n_path.exists()}"
entity_locale = (
json.loads(i18n_path.read_text(encoding="utf-8")) if i18n_path.exists() else {}
)

def j(x: object) -> str:
return json.dumps(x, ensure_ascii=False, separators=(",", ":"))
# The graph itself, and the two per-entity maps that are only read once a
# graph is on screen. socialPos and themesPos travel with the graph rather
# than with the shelf metadata for the same reason the edges do: the grid
# never plots a node.
detail = {
"id": bid,
"entities": conv["entities"],
"edges": conv["edges"],
"glucose": conv["glucose"],
"chunks": conv["chunks"],
"entityLocale": entity_locale,
"socialPos": book.pop("socialPos", {}),
"themesPos": book.pop("themesPos", {}),
}
detail_path = OUT_BOOKS / f"{bid}.json"
detail_path.write_text(j(detail), encoding="utf-8")
books.append(book)
print(
f" {f.name}: {len(conv['entities'])} entities, {len(conv['edges'])} edges, "
f"i18n={i18n_path.exists()} -> assets/exports/{bid}.json "
f"({detail_path.stat().st_size / 1024:.0f} KB)"
)

out = f"""/* AUTO-GENERATED by scripts/build_frontend_data.py — do not edit by hand.
Merges real pipeline-exported books (data/exports/*.json) into window.LG_DATA
and their per-locale entity names/glosses into window.LG_ENTITY_LOCALE.
Loads AFTER i18n.js so LG_ENTITY_LOCALE already exists. */
Merges the exported books' SHELF METADATA (data/exports/*.json) over the
manifest placeholders in window.LG_DATA, which is everything the library grid
draws. Their graphs — entities, edges, glucose, chunks, localized names — are
one file each under assets/exports/, fetched by window.LG_LOAD_BOOK when a book
is opened. Loads AFTER i18n.js so LG_ENTITY_LOCALE already exists. */
(function () {{
if (!window.LG_DATA || window.LG_DATA.__exportsMerged) return;
var D = window.LG_DATA;
Expand All @@ -636,17 +681,16 @@ def j(x: object) -> str:
if (idx >= 0) D.books[idx] = Object.assign({{}}, D.books[idx], b); // merge real data over the manifest placeholder
else D.books.push(b);
}});
Array.prototype.push.apply(D.entities, {j(entities)});
Array.prototype.push.apply(D.edges, {j(edges)});
Array.prototype.push.apply(D.glucose, {j(glucose)});
Array.prototype.push.apply(D.chunks, {j(chunks)});
if (window.LG_ENTITY_LOCALE) Object.assign(window.LG_ENTITY_LOCALE, {j(entity_locale)});
/* Which books have a graph to fetch, and where from. */
window.LG_BOOK_EXPORTS = {j([b["id"] for b in books])};
}})();
"""
OUT.write_text(out, encoding="utf-8")
kb = len(out.encode("utf-8")) / 1024
detail_kb = sum(p.stat().st_size for p in OUT_BOOKS.glob("*.json")) / 1024
print(
f"wrote {OUT.relative_to(ROOT)} ({kb:.0f} KB, {len(books)} book(s), {len(entity_locale)} localized entities)"
f"wrote {OUT.relative_to(ROOT)} ({kb:.0f} KB, {len(books)} book(s)) "
f"+ {OUT_BOOKS.relative_to(ROOT)}/*.json ({detail_kb / 1024:.1f} MB, fetched on demand)"
)
return 0

Expand Down
13 changes: 11 additions & 2 deletions src/loregraph/web/build-landing.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ const ORDER = [
"data-exports.js",
"avatars.jsx",
"covers.jsx",
"book-empty.jsx",
"view-library.jsx",
"view-shelf3d.jsx",
"graph-physics.jsx",
Expand All @@ -49,8 +50,15 @@ const ORDER = [
"app.jsx",
];

// Three families, not seven. This also asked for Noto Serif SC, TC, JP and KR at
// four weights each: 1828 @font-face rules, a 1.79 MB stylesheet (466 KB over the
// wire, 1.4 s) blocking render in the <head> — and not one element on the page
// named any of them. The app sets CJK in the platform serif, which is what
// .serif's own fallback has always done; the only place the Noto names appeared
// was the 3-D shelf's canvas font string, and it now names platform serifs too.
// The request is 37 KB.
const FONTS =
"https://fonts.googleapis.com/css2?family=Spectral:ital,wght@0,300;0,400;0,500;0,600;1,300;1,400;1,500;1,600&family=Inter:wght@300;400;500;600;700&family=JetBrains+Mono:wght@300;400;500;600&family=Noto+Serif+SC:wght@300;400;500;600&family=Noto+Serif+TC:wght@300;400;500;600&family=Noto+Serif+JP:wght@300;400;500;600&family=Noto+Serif+KR:wght@300;400;500;600&display=swap";
"https://fonts.googleapis.com/css2?family=Spectral:ital,wght@0,300;0,400;0,500;0,600;1,300;1,400;1,500;1,600&family=Inter:wght@300;400;500;600;700&family=JetBrains+Mono:wght@300;400;500;600&display=swap";

// Asset URLs are content-hashed (cssName / bundleName), so a new build always
// has a new URL — caches (browser + GitHub Pages CDN) can never serve a stale
Expand All @@ -69,7 +77,8 @@ const htmlDoc = (cssName, bundleName) => `<!DOCTYPE html>
<link rel="stylesheet" href="${cssName}" />
<script crossorigin src="https://unpkg.com/react@18.3.1/umd/react.production.min.js"></script>
<script crossorigin src="https://unpkg.com/react-dom@18.3.1/umd/react-dom.production.min.js"></script>
<script crossorigin src="https://unpkg.com/three@0.160.1/build/three.min.js"></script>
<!-- three.js is fetched on demand by view-shelf3d.jsx, not here: 670 KB for one
mode of one view, and the library opens in grid mode. -->
</head>
<body>
<div id="root"></div>
Expand Down
76 changes: 75 additions & 1 deletion src/loregraph/web/landing/app.jsx
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,57 @@

const { useState, useEffect, useMemo, useRef } = React;

// ---- a book's graph, on demand ----
//
// The exported books' graphs are big — 1.8 MB for Alice, 9.4 MB for 西游记 — and
// they used to be merged into window.LG_DATA by a generated 11 MB script that was
// 90% of the app's bundle and parsed before the first paint. Two books out of 85,
// and the app opens on the library grid, which draws neither of their graphs.
//
// So the grid gets the shelf metadata (data-exports.js, 11 KB) and a book's graph
// arrives when that book is opened. Nothing needs a loading state: 82 of the 85
// books have no graph at all and every view already renders them, so a book whose
// file is still in flight looks exactly like one of those, for as long as it takes.
const bookLoads = new Map();
const bookLoaded = new Set();

// Whether a graph is still on its way, answered synchronously so a view can tell
// "loading" from "this work has never been through the pipeline" — which is what
// 82 of the 85 books on the shelf are, and a different thing to say.
window.LG_BOOK_PENDING = (bookId) =>
(window.LG_BOOK_EXPORTS || []).includes(bookId) && !bookLoaded.has(bookId);

window.LG_LOAD_BOOK = function (bookId) {
if (!bookId || !(window.LG_BOOK_EXPORTS || []).includes(bookId)) return Promise.resolve(false);
if (bookLoads.has(bookId)) return bookLoads.get(bookId);
// Relative, so it resolves under /app, /app.html and a project-path deploy alike.
const load = fetch(`assets/exports/${encodeURIComponent(bookId)}.json`)
.then((r) => {
if (!r.ok) throw new Error(`assets/exports/${bookId}.json: HTTP ${r.status}`);
return r.json();
})
.then((detail) => {
const D = window.LG_DATA;
for (const key of ["entities", "edges", "glucose", "chunks"]) {
Array.prototype.push.apply(D[key], detail[key] || []);
}
if (window.LG_ENTITY_LOCALE) Object.assign(window.LG_ENTITY_LOCALE, detail.entityLocale || {});
// The two per-entity layouts travel with the graph, not the shelf metadata.
const book = D.books.find((b) => b.id === bookId);
if (book) {
book.socialPos = detail.socialPos || {};
book.themesPos = detail.themesPos || {};
}
bookLoaded.add(bookId);
return true;
});
// A failed fetch must not be cached as "loading forever": the reader can switch
// away and back, and the retry should be allowed to succeed.
load.catch(() => bookLoads.delete(bookId));
bookLoads.set(bookId, load);
return load;
};

// Most of this interface's controls are <div onClick> — rows, cards, chips,
// tabs, TOC entries. They work with a mouse and are unreachable by keyboard,
// which made the whole app unusable without one: even the sidebar that switches
Expand Down Expand Up @@ -136,6 +187,11 @@ function App() {
// Phone-width navigation drawer. Not the same thing as sbCollapsed, which
// narrows the rail in place; here it is off-canvas entirely.
const [navOpen, setNavOpen] = useState(false);
// Bumped when a book's graph finishes loading — see the effect below and
// LG_LOAD_BOOK above. Its job is to re-render the tree after LG_DATA has been
// mutated in place, which React cannot see on its own; it goes into ctx so a
// view that memoizes an expensive layout has something to key on.
const [bookDataVersion, setBookDataVersion] = useState(0);

// Mirror the locale onto <html lang>. Besides being correct for screen readers
// and line-breaking, the stylesheet keys its letter-spacing tokens off this:
Expand All @@ -157,6 +213,24 @@ function App() {
return () => document.removeEventListener("keydown", onKey);
}, [navOpen]);
useEffect(() => { setNavOpen(false); }, [activeView]);
// Fetch the active book's graph if it has one and we have not already. LG_DATA
// is mutated in place, which React cannot see, so the counter is what re-renders
// the tree once the file lands.
useEffect(() => {
let live = true;
window.LG_LOAD_BOOK(activeBookId).then(
(merged) => {
if (live && merged) setBookDataVersion((v) => v + 1);
},
// Nothing to report: the book renders as one without a graph, and selecting
// it again retries.
() => {},
);
return () => {
live = false;
};
}, [activeBookId]);

// Promote the active book to the head of the MRU list whenever it changes.
useEffect(() => {
if (!activeBookId) return;
Expand Down Expand Up @@ -189,7 +263,7 @@ function App() {
if (view) setActiveView(view);
};

const ctx = { data, locale, setLocale, tt, activeView, setActiveView, activeBook, setActiveBookId, bookMru, entities, edges, chunks, glucose, conversations, selectedEntityId, setSelectedEntityId, gotoEntity, selectedConvId, setSelectedConvId, settingsSection, setSettingsSection, coverStyle, setCoverStyle, graphViewMode, setGraphViewMode, graphLeftHidden, setGraphLeftHidden, graphRightHidden, setGraphRightHidden, tlMode, setTlMode };
const ctx = { data, locale, setLocale, tt, activeView, setActiveView, activeBook, setActiveBookId, bookMru, bookDataVersion, entities, edges, chunks, glucose, conversations, selectedEntityId, setSelectedEntityId, gotoEntity, selectedConvId, setSelectedConvId, settingsSection, setSettingsSection, coverStyle, setCoverStyle, graphViewMode, setGraphViewMode, graphLeftHidden, setGraphLeftHidden, graphRightHidden, setGraphRightHidden, tlMode, setTlMode };

return (
<div className={"app" + (sbCollapsed ? " sb-collapsed" : "") + (navOpen ? " nav-open" : "")}>
Expand Down
1 change: 1 addition & 0 deletions src/loregraph/web/landing/assets/exports/alice.json

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions src/loregraph/web/landing/assets/exports/xyj.json

Large diffs are not rendered by default.

25 changes: 25 additions & 0 deletions src/loregraph/web/landing/book-empty.jsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
// LoreGraph — the empty state for a work with no extracted graph.
//
// A work on the shelf with no graph to draw. Two views used to read straight into
// the first passage and the first entity, so this state took them down with the
// error boundary — for the 82 books that have never been through the pipeline as
// much as for one whose file is still arriving. Those are different facts, and it
// says which.
window.LGBookEmpty = function LGBookEmpty({ ctx }) {
const { tt, activeBook } = ctx;
if (window.LG_BOOK_PENDING(activeBook && activeBook.id)) {
return (
<div className="empty book-empty">
<div className="glyph">◌</div>
<div>{tt("empty.bookLoading")}</div>
</div>
);
}
return (
<div className="empty book-empty">
<div className="glyph">○</div>
<h2 className="book-empty-title">{tt("empty.bookNone.title")}</h2>
<p className="book-empty-body">{tt("empty.bookNone.body")}</p>
</div>
);
};
6 changes: 5 additions & 1 deletion src/loregraph/web/landing/covers.jsx
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,11 @@ function CoverImage({ src, alt, fallback }) {
background: "#1a1714",
}}>
{!loaded && <div className="cover-skeleton" aria-hidden="true" />}
<img src={src} alt={alt}
{/* The library grid is 85 of these, and four are on screen. Eager, they were
85 cross-origin requests to Wikimedia before the first paint, for 81
scans nobody had scrolled to. The skeleton above is what a not-yet-loaded
board already looks like, so laziness costs nothing visually. */}
<img src={src} alt={alt} loading="lazy" decoding="async"
onError={() => setErrored(true)}
onLoad={() => setLoaded(true)}
style={{
Expand Down
17 changes: 8 additions & 9 deletions src/loregraph/web/landing/data-exports.js

Large diffs are not rendered by default.

6 changes: 6 additions & 0 deletions src/loregraph/web/landing/i18n.js

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading
Loading