diff --git a/.github/scripts/site-check.py b/.github/scripts/site-check.py
old mode 100755
new mode 100644
index 03b449f4..b25c1bf5
--- a/.github/scripts/site-check.py
+++ b/.github/scripts/site-check.py
@@ -30,12 +30,16 @@
- an awesome list's count token left in a page (``, `n:categorias`,
`n:retirados`): the Markdown extension that fills them in did not run. Only those three
names: CashPilot writes other `` comments of its own in its pages;
- - a raw , or an unclosed <textarea>, <xmp>, <noembed>, <noframes>, or any
+ - a raw <title>, or an unclosed <textarea>, <xmp>, <noembed>, <noframes> (a self-closed
+ `<textarea/>` counts as unclosed: HTML ignores that slash), or any
<plaintext>, in a page's article outside inline SVG and MathML: a browser's HTML parser
treats everything after it as text, so the rest of the page disappears (Markdown passes
`<title>` written in prose through as a real tag; Telegram-Archive shipped that way);
- a "Copy as Markdown" button (data-md-copy-url) whose Markdown copy does not exist in the
site: the llmstxt plugin writes the copies, and only for pages its `sections` globs match.
+ - a link that cannot be resolved to a file (a %00 in its path): an error, not a traceback;
+ - an index.html that is not a content page (no md-content block, or a redirect stub);
+ - a site_url, or the canonical link standing in for it, that is not an absolute http(s) URL.
And, from zensical.toml (Zensical drops what it does not know without a word; under MkDocs
strict an unknown plugin aborted the build):
- a plugin name Zensical 0.1.0 does not know (`redirect` for `redirects`, `llmstxtt`): the
@@ -43,11 +47,13 @@
- a theme feature name the Zensical UI 0.0.41 does not know (the list below is what its
templates and bundle test for; `search.suggest` and `search.share` are not in it);
- `theme.variant` other than "modern" or "classic" (anything else is classic, silently),
- and a "modern" config whose index.html does not load the modern stylesheet;
- - a `redirect_maps` entry whose stub was not written;
+ and a modern config (the default) whose index.html does not load the modern stylesheet;
+ - a `redirect_maps` entry whose stub was not written, a `redirect_maps` that is not a
+ table, and `theme.features` that is not a list of strings;
- an enabled llmstxt plugin with no site/llms.txt, or `content.action.copy` in the
features with a page that carries no copy button.
-Plugin keys are read the way Zensical reads them (`material/redirects` is `redirects`), and
+Plugins are read in every form Zensical accepts (a table, or a list of names and one-entry
+tables; `material/redirects` is `redirects`), and
a plugin with `enabled = false` is treated as off. With directory URLs off (the offline
plugin turns them off) every .html file is a page.
Stdlib only (tomllib needs Python 3.11+), so it runs under `python -I` on any runner.
@@ -60,11 +66,11 @@
from pathlib import Path
from urllib.parse import unquote, urlparse
-SWALLOW = re.compile(r"<(title|textarea|xmp|plaintext|noembed|noframes)\b", re.IGNORECASE)
-FOREIGN = re.compile(r"<(svg|math)\b.*?</\1\s*>", re.IGNORECASE | re.DOTALL)
+SWALLOW = {"title", "textarea", "xmp", "plaintext", "noembed", "noframes"}
+# HTML ignores the slash in <x/> except on these and in SVG/MathML: <textarea/> opens a textarea.
+VOID = {"area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source", "track", "wbr"}
TOKEN = re.compile(r"<!--\s*n:(?:proyectos|categorias|retirados)\s*-->")
-ARTICLE = re.compile(r"<article\b.*?</article>", re.DOTALL)
-STUB = "<title>Redirecting..."
+STUB = "Redirecting..." # the of a Zensical redirect stub
# Keys of _PLUGIN_UNSUPPORTED_OPTIONS in zensical/config.py (0.1.0, same set as 0.0.74): every other name is
# dropped by `_convert_plugins` without a message. `material/` prefixes and `social/<x>`,
@@ -96,7 +102,8 @@
class Page(HTMLParser):
"""The parts of one built HTML file the checks read, from a real HTML parser: any
- quoting, entities decoded, nothing taken from <script>/<style> text or escaped code."""
+ quoting, entities decoded, nothing taken from <script>/<style> text, comments or
+ escaped code."""
def __init__(self, text: str):
"""Parse one file's text completely."""
@@ -105,33 +112,108 @@ def __init__(self, text: str):
self.ids: set[str] = set()
self.copies: list[str] = []
self.nav: list[str] = []
+ self.styles: list[str] = []
self.canonical: str | None = None
+ self.title: str | None = None # the <head> title text, once one opens
+ self.in_title = False
+ self.content = False # carries the theme's md-content block: a page, not a copied asset
+ self.swallowed: list[str] = [] # raw tags that hide the rest of the page
+ self.tokens: list[str] = []
self.in_head = True
+ self.article = 0
+ self.template = 0
+ self.context = ["html"] # "foreign" inside <svg>/<math>, back to "html" in <foreignObject>
+ self.open: list[str] = [] # swallowing tags waiting for their end tag
self.feed(text)
self.close()
+ self.swallowed += self.open
+ self.stub = (self.title or "").strip() == STUB # a redirect stub: links checked, not a page
def handle_starttag(self, tag, attrs):
"""Record ids, URLs, copy targets, nav links and the canonical link of one tag."""
- a = {k: v or "" for k, v in attrs}
+ self.tag(tag, attrs, empty=False)
+
+ def handle_startendtag(self, tag, attrs):
+ """A self-closed tag: empty only for a void element or in SVG/MathML; HTML ignores
+ the slash elsewhere, so <textarea/> opens a textarea."""
+ foreign = self.context[-1] == "foreign" or tag in ("svg", "math")
+ self.tag(tag, attrs, empty=tag in VOID or foreign)
+
+ def tag(self, tag, attrs, empty):
+ """Shared work of the two start-tag callbacks."""
+ a = {k: (v or "").strip() for k, v in attrs}
if tag == "body":
self.in_head = False
- for key in ("id", "name") if tag == "a" else ("id",):
- if a.get(key):
- self.ids.add(a[key])
+ if tag == "title" and self.in_head and self.title is None and not empty:
+ self.title, self.in_title = "", True
+ if not empty:
+ if tag == "article":
+ self.article += 1
+ elif tag == "template":
+ self.template += 1
+ elif tag in ("svg", "math"):
+ self.context.append("foreign")
+ elif tag == "foreignobject" and self.context[-1] == "foreign":
+ self.context.append("html")
+ if not self.template: # ids inside <template> are not in the document
+ for key in ("id", "name") if tag == "a" else ("id",):
+ if a.get(key):
+ self.ids.add(a[key])
for key in ("href", "src"):
if key in a:
self.urls.append((tag, a[key], self.in_head))
if "data-md-copy-url" in a:
self.copies.append(a["data-md-copy-url"])
- if tag == "link" and "canonical" in a.get("rel", "").split():
+ rel = a.get("rel", "").split()
+ if tag == "link" and "canonical" in rel:
self.canonical = a.get("href")
+ if tag == "link" and "stylesheet" in rel:
+ self.styles.append(a.get("href", ""))
classes = a.get("class", "").split()
+ if "md-content" in classes:
+ self.content = True
# The toc's #heading entries carry md-nav__link too: they point at the page itself,
# so only links with a path count toward reachability.
- if tag == "a" and ("md-nav__link" in classes or "md-tabs__link" in classes) and urlparse(a.get("href", "")).path:
+ if tag == "a" and ("md-nav__link" in classes or "md-tabs__link" in classes) and url_path(a.get("href", "")):
self.nav.append(a["href"])
-
- handle_startendtag = handle_starttag
+ # In an article and in HTML context (an SVG <title> is the drawing's name), a raw
+ # <title> or <plaintext> always hides text; the others only when never closed.
+ if self.article and self.context[-1] == "html" and tag in SWALLOW:
+ if tag in ("title", "plaintext"):
+ self.swallowed.append(tag)
+ elif not empty:
+ self.open.append(tag)
+
+ def handle_data(self, data):
+ """Collect the <head> title text."""
+ if self.in_title:
+ self.title += data
+
+ def handle_endtag(self, tag):
+ """Close what the matching start tag opened."""
+ if tag == "title":
+ self.in_title = False
+ if tag == "article" and self.article:
+ self.article -= 1
+ elif tag == "template" and self.template:
+ self.template -= 1
+ elif len(self.context) > 1 and self.context[-1] == {"svg": "foreign", "math": "foreign", "foreignobject": "html"}.get(tag):
+ self.context.pop()
+ if tag in self.open:
+ self.open.remove(tag)
+
+ def handle_comment(self, data):
+ """An awesome list's count token is a comment: report one left in an article."""
+ if self.article and TOKEN.fullmatch(f"<!--{data}-->"):
+ self.tokens.append(f"<!--{data}-->")
+
+
+def url_path(url: str) -> str:
+ """The path of a URL, or "" when there is none or the URL cannot be parsed."""
+ try:
+ return urlparse(url.strip()).path
+ except ValueError:
+ return ""
def canonical_plugin(name: str) -> str:
@@ -140,10 +222,31 @@ def canonical_plugin(name: str) -> str:
return name.split("/", 1)[0] if name.startswith(("social/", "privacy/", "optimize/")) else name
+def plugin_table(raw, errors: set[str]) -> dict:
+ """`plugins` in any form Zensical accepts (a table, or a list of names and one-entry
+ tables) as one table keyed by the names as written."""
+ if raw is None:
+ return {}
+ if isinstance(raw, dict):
+ return raw
+ table: dict = {}
+ if isinstance(raw, list):
+ for item in raw:
+ if isinstance(item, str):
+ table[item] = {}
+ elif isinstance(item, dict) and len(item) == 1:
+ table.update(item)
+ else:
+ errors.add(f"zensical.toml: plugins entry {item!r} is neither a name nor a one-entry table")
+ return table
+ errors.add(f"zensical.toml: plugins is a {type(raw).__name__}, not a table or a list")
+ return {}
+
+
def plugin_on(plugins: dict, name: str) -> bool:
"""True when the plugin is configured under any of its key spellings and not `enabled = false`."""
for key, opts in plugins.items():
- if canonical_plugin(key) == name:
+ if isinstance(key, str) and canonical_plugin(key) == name:
return not (isinstance(opts, dict) and opts.get("enabled", True) is False)
return False
@@ -151,25 +254,11 @@ def plugin_on(plugins: dict, name: str) -> bool:
def plugin_opts(plugins: dict, name: str) -> dict:
"""The plugin's option table, under whichever key spelling the config uses."""
for key, opts in plugins.items():
- if canonical_plugin(key) == name and isinstance(opts, dict):
+ if isinstance(key, str) and canonical_plugin(key) == name and isinstance(opts, dict):
return opts
return {}
-def swallowing_tags(article: str) -> list[str]:
- """Raw tags in an article that make a browser treat the rest of the page as text."""
- # <title> inside inline SVG is the SVG's accessible name, and MathML has its own
- # content model: neither swallows the page, so both are cut out first.
- art = FOREIGN.sub("", article)
- found = []
- for m in SWALLOW.finditer(art):
- tag = m.group(1).lower()
- closed = re.search(rf"</{tag}\s*>", art[m.end():], re.IGNORECASE)
- if tag in ("title", "plaintext") or not closed:
- found.append(tag)
- return found
-
-
def read_config(path: Path, errors: set[str]) -> dict:
"""The [project] table of zensical.toml; an unreadable or empty config is an error."""
if not path.is_file():
@@ -196,19 +285,12 @@ def main(site_dir: str, config_path: str) -> int:
return 1
errors: set[str] = set()
cfg = read_config(Path(config_path), errors)
- plugins = cfg.get("plugins") or {}
+ plugins = plugin_table(cfg.get("plugins"), errors)
+ theme = cfg.get("theme")
+ theme = {"name": theme} if isinstance(theme, str) else theme if isinstance(theme, dict) else {}
# The offline plugin turns directory URLs off (zensical/config.py, _apply_offline_plugin).
directory_urls = cfg.get("use_directory_urls", True) is not False and not plugin_on(plugins, "offline")
- site_url = cfg.get("site_url") or Page(index.read_text(encoding="utf-8", errors="replace")).canonical or ""
- origin = urlparse(site_url)
- prefix = origin.path or ""
- if not site_url:
- errors.add("no site_url in zensical.toml and no canonical link in index.html: the "
- "root-absolute link check has no prefix to check against")
- prefix = "/"
- prefix = prefix if prefix.endswith("/") else prefix + "/"
-
parsed: dict[Path, Page] = {}
def parse(path: Path) -> Page:
@@ -217,27 +299,63 @@ def parse(path: Path) -> Page:
parsed[path] = Page(path.read_text(encoding="utf-8", errors="replace"))
return parsed[path]
+ from_canonical = not cfg.get("site_url")
+ site_url = str(cfg.get("site_url") or parse(index).canonical or "")
+ try:
+ origin = urlparse(site_url)
+ origin_port = origin.port
+ except ValueError:
+ errors.add(f"site_url {site_url!r} is not a valid URL")
+ origin, origin_port, site_url = urlparse(""), None, ""
+ if site_url and (origin.scheme not in ("http", "https") or not origin.hostname):
+ errors.add(f"site_url {site_url!r} is not an absolute http(s) URL")
+ host = (origin.hostname or "").lower()
+ prefix = unquote(origin.path or "")
+ if from_canonical and prefix.endswith(".html"): # canonical .../index.html: its directory
+ prefix = prefix.rsplit("/", 1)[0]
+ if not site_url:
+ errors.add("no site_url in zensical.toml and no canonical link in index.html: the "
+ "root-absolute link check has no prefix to check against")
+ prefix = "/"
+ prefix = prefix if prefix.endswith("/") else prefix + "/"
+
def resolve(page: Path, url: str):
"""None for a URL outside this site, "ABS" for a rooted path outside the prefix,
- else the file the URL lands on (not necessarily existing)."""
- u = urlparse(url)
- if u.scheme in ("http", "https") and u.netloc and u.netloc == origin.netloc:
- if not u.path.startswith(prefix) and u.path + "/" != prefix:
+ "BAD" for a URL that cannot be parsed, else the file the URL lands on."""
+ url = url.strip()
+ try:
+ u = urlparse(url)
+ port = u.port
+ except ValueError:
+ return "BAD"
+ if u.scheme not in ("", "http", "https"):
+ return None # mailto:, data:, javascript:
+ if u.netloc: # https://host/... or //host/...: in this site only on its own host and prefix
+ if (u.hostname or "").lower() != host or port not in (origin_port, None, 80, 443):
+ return None
+ path = unquote(u.path) or "/"
+ if not path.startswith(prefix) and path + "/" != prefix:
return None # another site on the same host (geiserx.github.io/<other repo>)
- elif u.scheme or u.netloc:
+ elif u.scheme:
return None
- if url == "" or url.startswith("#"):
- return page
- path = unquote(u.path)
+ else:
+ path = unquote(u.path)
+ if not path:
+ return page # "", "#x", "?q#x": the page itself
+ if "\x00" in path:
+ return "BAD"
if path.startswith("/"):
if not path.startswith(prefix) and path + "/" != prefix:
return "ABS"
target = site / path[len(prefix):]
else:
target = page.parent / path
- if path.endswith("/") or target.is_dir():
- target = target / "index.html"
- return target.resolve()
+ try:
+ if path.endswith("/") or target.is_dir():
+ target = target / "index.html"
+ return target.resolve()
+ except (OSError, ValueError):
+ return "BAD"
for p in site.rglob("*"):
if p.suffix in (".py", ".pyc") or "__pycache__" in p.relative_to(site).parts:
@@ -245,25 +363,29 @@ def resolve(page: Path, url: str):
nav_targets: set[Path] = set()
pages: list[Path] = []
- copy_on = "content.action.copy" in ((cfg.get("theme") or {}).get("features") or [])
+ features = theme.get("features") or []
+ if not isinstance(features, list) or not all(isinstance(f, str) for f in features):
+ errors.add(f"zensical.toml: theme.features must be a list of strings, got {features!r}")
+ features = []
+ copy_on = "content.action.copy" in features
for page in sorted(site.rglob("*.html")):
- text = page.read_text(encoding="utf-8", errors="replace")
rel = page.relative_to(site)
- stub = STUB in text # a redirect stub: its links are checked, it is not a page
- is_page = not stub and (rel.name == "index.html" if directory_urls else rel.as_posix() != "404.html")
+ doc = parse(page)
+ named = rel.name == "index.html" if directory_urls else rel.as_posix() != "404.html"
+ is_page = named and doc.content and not doc.stub
if is_page:
pages.append(page)
- for art in ARTICLE.findall(text):
- for tag in swallowing_tags(art):
- errors.add(f"{rel}: raw <{tag}> in the page body hides everything after it in a browser")
- for tok in TOKEN.findall(art):
- errors.add(f"{rel}: build token {tok} was not filled in (extension not loaded?)")
- doc = parse(page)
+ for tag in doc.swallowed:
+ errors.add(f"{rel}: raw <{tag}> in the page body hides everything after it in a browser")
+ for tok in doc.tokens:
+ errors.add(f"{rel}: build token {tok} was not filled in (extension not loaded?)")
for tag, url, in_head in doc.urls:
t = resolve(page, url)
if t is None:
continue
- if t == "ABS":
+ if t == "BAD":
+ errors.add(f"{rel}: {url!r} is not a valid URL")
+ elif t == "ABS":
errors.add(f"{rel}: absolute link {url} (outside {prefix})")
elif not t.is_relative_to(site):
# ../LICENSE from the top page: the file exists in the checkout, not on Pages.
@@ -273,17 +395,18 @@ def resolve(page: Path, url: str):
else:
# <head> holds <link rel="alternate" type="text/markdown" href="./index.md">
# from the llmstxt plugin, a legitimate .md href; this rule reads the body only.
- if not in_head and unquote(urlparse(url).path).endswith(".md"):
+ if not in_head and unquote(url_path(url)).endswith(".md"):
errors.add(f"{rel}: {url} links a Markdown source, not a page URL (raw HTML link?)")
- frag = unquote(urlparse(url).fragment)
- # #top and text fragments (#:~:text=) are valid in a browser with no matching id.
- special = frag.lower() == "top" or frag.startswith(":~:")
- if tag == "a" and frag and not special and t.suffix == ".html" and frag not in parse(t).ids:
+ # A text directive (#id:~:text=...) is not part of the id; #top needs no id. A
+ # redirect stub's ids are not checked: the browser lands on the stub's target.
+ frag = unquote(urlparse(url.strip()).fragment).split(":~:", 1)[0]
+ if tag in ("a", "area") and frag and frag.lower() != "top" and t.suffix == ".html" \
+ and not parse(t).stub and frag not in parse(t).ids:
errors.add(f"{rel}: {url} -> no id {frag!r} on {t.relative_to(site)}")
for url in doc.copies:
t = resolve(page, url)
- if t is None or t == "ABS" or not t.is_relative_to(site) or not t.is_file():
- errors.add(f"{rel}: Copy as Markdown points at {url}, which is not in the site (llmstxt sections?)")
+ if not isinstance(t, Path) or not t.is_relative_to(site) or t.suffix != ".md" or not t.is_file():
+ errors.add(f"{rel}: Copy as Markdown points at {url!r}, which is not a Markdown copy in the site (llmstxt sections?)")
if is_page and copy_on and not doc.copies:
errors.add(f"{rel}: content.action.copy is on but the page has no Copy as Markdown button")
for url in doc.nav:
@@ -291,35 +414,41 @@ def resolve(page: Path, url: str):
if isinstance(t, Path):
nav_targets.add(t)
+ if index not in pages:
+ errors.add("index.html is not a content page (no md-content block, or a redirect stub)")
for page in pages:
if page.resolve() not in nav_targets:
errors.add(f"{page.relative_to(site)}: page is in no navigation entry (stray page)")
if cfg:
for name in plugins:
- if canonical_plugin(name) not in KNOWN_PLUGINS:
+ if not isinstance(name, str) or canonical_plugin(name) not in KNOWN_PLUGINS:
errors.add(f"zensical.toml: plugin {name!r} is unknown to Zensical and silently ignored")
- theme = cfg.get("theme") or {}
- for f in theme.get("features") or []:
+ for f in features:
if f not in KNOWN_FEATURES:
errors.add(f"zensical.toml: feature {f!r} is unknown to the Zensical UI and silently ignored")
- variant = theme.get("variant", "classic")
+ variant = theme.get("variant", "modern") # Zensical's default (config.py, set_default)
if variant not in ("modern", "classic"):
errors.add(f"zensical.toml: theme.variant {variant!r} is not modern or classic (Zensical falls back to classic silently)")
- elif variant == "modern" and "assets/stylesheets/modern/" not in index.read_text(encoding="utf-8", errors="replace"):
+ elif variant == "modern" and not any("assets/stylesheets/modern/" in h for h in parse(index).styles):
errors.add("index.html does not load the modern stylesheet although theme.variant is modern")
redirects = plugin_opts(plugins, "redirects") if plugin_on(plugins, "redirects") else {}
- for src in redirects.get("redirect_maps") or {}:
- s = src.removesuffix(".md")
+ maps = redirects.get("redirect_maps") or {}
+ if not isinstance(maps, dict):
+ errors.add(f"zensical.toml: redirects redirect_maps must be a table, got {maps!r}")
+ maps = {}
+ for src in maps:
+ s = str(src).lstrip("/").removesuffix(".md")
# A README page is its folder's index: sub/README.md -> sub/index.html in every mode.
if s == "README" or s.endswith("/README"):
s = s[: -len("README")] + "index"
if not directory_urls or s == "index" or s.endswith("/index"):
- stub = site / (s + ".html")
+ stub = (site / (s + ".html")).resolve()
else:
- stub = site / s / "index.html"
- if not stub.is_file() or STUB not in stub.read_text(encoding="utf-8", errors="replace"):
- errors.add(f"redirect_maps {src!r}: no redirect stub at {stub.relative_to(site)}")
+ stub = (site / s / "index.html").resolve()
+ shown = stub.relative_to(site) if stub.is_relative_to(site) else stub
+ if not stub.is_file() or not parse(stub).stub:
+ errors.add(f"redirect_maps {src!r}: no redirect stub at {shown}")
if plugin_on(plugins, "llmstxt") and not (site / "llms.txt").is_file():
errors.add("llmstxt plugin is configured but site/llms.txt was not written")