diff --git a/README.md b/README.md index b1516402..8c02158e 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,9 @@ Episode notes and media links live in [logseq-encode-garden](https://github.com/codekiln/logseq-encode-garden). Gitpa mirrors each session under its exact page name, together with its embedded audio asset page. The website and RSS read those checked-in pages. -To refresh an episode, run `mise run episode:sync -- --source /path/to/logseq-encode-garden 'GitP/A/Session/26/09/24-Thu'`. Sync replaces the page body and preserves Gitpa's page properties. A destination page without a matching proxy URL stops the whole batch before writing. Relative assets are copied; missing source assets are reported. +To mirror or refresh an episode, run the garden's sync task, imported into `gitp-garden/mise-tasks/`: `mise run logseq:entity:proxy:page:sync --source /path/to/logseq-encode-garden --destination gitp-garden --page 'GitP/A/Session/26/09/24-Thu' --follow-embeds --apply`. `--follow-embeds` brings the recording, artwork, preset and MIDI pages the session embeds in the same batch; leave out `--apply` to preview. The page bodies are replaced with the garden's. Properties follow the garden too, except `tags::`, `public::`, `podcast-guid::` and `podcast-published-at::`, which stay as Gitpa set them; the preview lists each property it adds, updates or removes. A destination page without a matching proxy URL stops the batch before writing. The task also brings each page's entity definitions and records what it imported in `gitp-garden/.logseq-proxy/manifest.json`. Re-running it against unchanged garden pages changes nothing except `logseq-proxy-last-sync-date::`. -To re-sync a page already mirrored here, run the garden's own task, imported into `gitp-garden/mise-tasks/`: `mise run logseq:entity:proxy:page:sync --source /path/to/logseq-encode-garden --destination gitp-garden --page 'GitP/A/Session/26/09/24-Thu' --apply`. It also brings the page's entity definitions and records what it imported in `gitp-garden/.logseq-proxy/manifest.json`. Re-running it against unchanged garden pages changes nothing except `logseq-proxy-last-sync-date::`. +To update the imported sync task itself, sync its entity definition: `mise run logseq:entity:proxy:page:sync --source /path/to/logseq-encode-garden --destination gitp-garden --page 'Logseq/Entity/Proxy/Page' --apply`. Set `public:: true` on the episode page to release the episode on the website and RSS. The website build turns embedded recording and artwork pages into direct audio and image links, and labels preset and MIDI downloads. Proxy and storage properties remain in the checked-in garden; the exported listener view hides page properties. A `Notes` or `Production notes` heading publishes its children only when the heading carries `public:: true`. The website build derives asset-page visibility from public episodes; shared assets stay visible while any public episode links to them. The episode's `podcast-guid::` identifies its RSS item permanently; `podcast-published-at::` holds its publication time with a timezone offset. The RSS builder reads the title and description from the session's opening heading and paragraph, follows its embedded MP3 asset page, and gets the enclosure size from the public MP3 server. `public:: false` keeps an episode out of the website and RSS. diff --git a/gitp-garden/.logseq-proxy/manifest.json b/gitp-garden/.logseq-proxy/manifest.json index 975b057e..8a0c1595 100644 --- a/gitp-garden/.logseq-proxy/manifest.json +++ b/gitp-garden/.logseq-proxy/manifest.json @@ -92,7 +92,7 @@ }, "mise-tasks/logseq/entity/proxy/page/lib/core.py": { "mode": 420, - "sha256": "3b861f705281fa0cddbd847065d76776354a108aacf7b9f143bf28b3a473267f", + "sha256": "b29946f9062c099cc0c7c3a8077b5300b8582796674b919b4e273323b781b3df", "source": [ "github.com/codekiln/logseq-encode-garden", ".", @@ -101,7 +101,7 @@ }, "mise-tasks/logseq/entity/proxy/page/sync": { "mode": 493, - "sha256": "9c8a8eb5fd9f10c3972b427210eac1843ad024a547fea8c22453ae9e7b9178ab", + "sha256": "4dc4adbcd690cb3d37fac58a7471454dc019d57160e8afc609d2508b1632a5ff", "source": [ "github.com/codekiln/logseq-encode-garden", ".", diff --git a/gitp-garden/mise-tasks/logseq/entity/proxy/page/lib/core.py b/gitp-garden/mise-tasks/logseq/entity/proxy/page/lib/core.py index 6c2c0bee..534410d9 100644 --- a/gitp-garden/mise-tasks/logseq/entity/proxy/page/lib/core.py +++ b/gitp-garden/mise-tasks/logseq/entity/proxy/page/lib/core.py @@ -16,6 +16,10 @@ from zoneinfo import ZoneInfo PROXY_KEYS = ('logseq-proxy-url', 'logseq-proxy-codeforge-url', 'logseq-proxy-last-sync-date') +PROXY_ENTITY = 'Logseq/Entity/Proxy/Page' +# An entity definition lists the properties a proxy's destination garden owns. +DESTINATION_PROPERTIES = 'entity-proxy-destination-properties' +EMBED = re.compile(r'\{\{embed\s+\[\[([^\]]+)\]\]\s*\}\}') TRANSACTION = '.logseq-proxy-transaction' PROPERTY = re.compile(r'^([^\s:]+):: (.*?)(?:\r?\n)?$') @@ -39,6 +43,7 @@ class Plan: expected_modes: dict[str, int | None] = field(default_factory=dict) source_expected: dict[Path, bytes] = field(default_factory=dict) source_modes: dict[Path, int] = field(default_factory=dict) + follow_embeds: bool = False def safe_path(root: Path, relative: str) -> Path: @@ -92,6 +97,60 @@ def parse_properties(text: str) -> tuple[dict[str, str], str]: return props, body +def property_key(line: str) -> str: + return PROPERTY.match(line).group(1) + + +def with_proxy_membership(lines: list[str]) -> list[str]: + """Add the Proxy Page entity to logseq-entity, creating the property if needed.""" + marker = f'[[{PROXY_ENTITY}]]' + if not any(property_key(line) == 'logseq-entity' for line in lines): + return [f'logseq-entity:: {marker}\n'] + lines + return [line.rstrip('\r\n') + f', {marker}\n' + if property_key(line) == 'logseq-entity' and marker not in line else line + for line in lines] + + +def destination_owned(plan: Plan, source_props: dict[str, str]) -> set[str]: + """Properties a re-sync keeps from the destination: tags plus each definition's declared keys.""" + keys = {'tags'} + entities = set(re.findall(r'\[\[([^\]]+)\]\]', source_props.get('logseq-entity', ''))) | {PROXY_ENTITY} + for entity in sorted(entities): + definition = resolve_page(plan.source_root, entity) + if not definition.is_file(): + continue + data = definition.read_bytes() + plan.source_expected[definition] = data + props, _ = parse_properties(data.decode('utf-8')) + keys.update(key.strip() for key in props.get(DESTINATION_PROPERTIES, '').split(',') if key.strip()) + return keys + + +def merge_properties(destination: list[str], source: list[str], owned: set[str]) -> tuple[list[str], list[str]]: + """Take source properties on re-sync, keeping destination-owned lines and the destination's order.""" + incoming = {property_key(line): line for line in source if property_key(line) not in PROXY_KEYS} + merged, changes, seen = [], [], set() + for line in destination: + key = property_key(line) + if key in PROXY_KEYS: + continue + seen.add(key) + if key in owned: + merged.append(line) + elif key not in incoming: + changes.append(f'removed {key}') + elif incoming[key].rstrip('\r\n') == line.rstrip('\r\n'): + merged.append(line) + else: + merged.append(incoming[key]) + changes.append(f'updated {key}') + for key, line in incoming.items(): + if key not in seen and key not in owned: + merged.append(line) + changes.append(f'added {key}') + return merged, changes + + def page_filename(page: str) -> str: if not page or page.startswith('/') or any(x in ('', '.', '..') for x in page.split('/')) or '\\' in page: raise SyncError(f'Unsafe page name: {page}') @@ -268,14 +327,11 @@ def add_page(plan: Plan, page: str) -> None: raise SyncError(f'Proxy source conflict: {target}') if props.get('logseq-proxy-codeforge-url'): validate_codeforge(props['logseq-proxy-codeforge-url'], plan.source_root, source) - lines, _ = property_lines(old_text) + old_lines, _ = property_lines(old_text) + lines, changes = merge_properties(old_lines, with_proxy_membership(source_lines), destination_owned(plan, source_props)) else: - lines = source_lines - if 'logseq-entity' in source_props: - lines = [line.rstrip('\r\n') + ', [[Logseq/Entity/Proxy/Page]]\n' if PROPERTY.match(line).group(1) == 'logseq-entity' and '[[Logseq/Entity/Proxy/Page]]' not in source_props['logseq-entity'] else line for line in lines] - else: - lines = ['logseq-entity:: [[Logseq/Entity/Proxy/Page]]\n'] + lines - retained = [line for line in lines if PROPERTY.match(line).group(1) not in PROXY_KEYS] + lines, changes = with_proxy_membership(source_lines), [] + retained = [line for line in lines if property_key(line) not in PROXY_KEYS] # A final property without a newline still needs to remain a separate property. retained = [line if line.endswith('\n') else line + '\n' for line in retained] metadata = {'logseq-proxy-url': graph_url(plan.source_root, page)} @@ -289,6 +345,7 @@ def add_page(plan: Plan, page: str) -> None: plan.pages.add(page) plan.details.append(f'{"Create" if first else "Re-sync"} {source} -> {target}') plan.details.extend(f' {key}:: {value}' for key, value in metadata.items()) + plan.details.extend(f' property {change}' for change in changes) # Capture Markdown link targets, allowing <...> around paths with literal spaces. assets = re.findall(r'\]\(\s*(?:<([^>]+)>|([^\s)]+))', body) for a, b in assets: @@ -305,14 +362,21 @@ def add_page(plan: Plan, page: str) -> None: plan.source_expected[asset] = asset_data add_write(plan, relative, asset_data) plan.details.append(f'Asset {asset} -> {plan.destination_root / relative}') + if plan.follow_embeds: + for embedded in EMBED.findall(body): + if resolve_page(plan.source_root, embedded).exists(): + add_page(plan, embedded) + else: + plan.warnings.append(f'Missing embedded page: {embedded}') -def build_page_plan(source: Path | None, destination: Path, page: str, codeforge_url: str | None = None) -> Plan: +def build_page_plan(source: Path | None, destination: Path, page: str, codeforge_url: str | None = None, + follow_embeds: bool = False) -> Plan: destination = validate_graph(destination) source = validate_graph(source if source is not None else resolve_source(destination, page)) if source == destination: raise SyncError('Source and destination must be different graphs') - plan = Plan(source, destination, page, codeforge_url=codeforge_url) + plan = Plan(source, destination, page, codeforge_url=codeforge_url, follow_embeds=follow_embeds) add_page(plan, page) return plan diff --git a/gitp-garden/mise-tasks/logseq/entity/proxy/page/sync b/gitp-garden/mise-tasks/logseq/entity/proxy/page/sync index 0e341fc1..0a9de95f 100755 --- a/gitp-garden/mise-tasks/logseq/entity/proxy/page/sync +++ b/gitp-garden/mise-tasks/logseq/entity/proxy/page/sync @@ -5,6 +5,7 @@ #USAGE flag "--destination " required=#true help="Destination graph root" #USAGE flag "--page " help="Exact logical page name" #USAGE flag "--codeforge-url " help="Source page URL on a code forge" +#USAGE flag "--follow-embeds" help="Also sync pages the page embeds, recursively" #USAGE flag "--apply" help="Apply the previewed import" #USAGE flag "--recover" help="Roll back an interrupted import, then exit" """Task reference: [[Logseq/Entity/Proxy/Page/mise/Task/sync]].""" @@ -23,6 +24,7 @@ def main(): parser.add_argument('--destination', type=Path, required=True) parser.add_argument('--page') parser.add_argument('--codeforge-url') + parser.add_argument('--follow-embeds', action='store_true') parser.add_argument('--apply', action='store_true') parser.add_argument('--recover', action='store_true') args = list(sys.argv[1:]) @@ -31,9 +33,9 @@ def main(): for name in ('source', 'destination', 'page', 'codeforge_url'): if os.environ.get('usage_' + name): args.extend(['--' + name.replace('_', '-'), os.environ['usage_' + name]]) - for name in ('apply', 'recover'): + for name in ('follow_embeds', 'apply', 'recover'): if os.environ.get('usage_' + name) == 'true': - args.append('--' + name) + args.append('--' + name.replace('_', '-')) options = parser.parse_args(args) try: if options.recover: @@ -42,7 +44,8 @@ def main(): return 0 if not options.page: parser.error('--page is required unless --recover is used') - plan = build_page_plan(options.source, options.destination, options.page, options.codeforge_url) + plan = build_page_plan(options.source, options.destination, options.page, options.codeforge_url, + options.follow_embeds) try: import companions except ModuleNotFoundError as error: diff --git a/gitp-garden/pages/Logseq___Entity___Definition.md b/gitp-garden/pages/Logseq___Entity___Definition.md index 1d1ebf95..4e168dfa 100644 --- a/gitp-garden/pages/Logseq___Entity___Definition.md +++ b/gitp-garden/pages/Logseq___Entity___Definition.md @@ -31,5 +31,8 @@ logseq-proxy-last-sync-date:: [[2026-10-06]] - [[Logseq/Entity/Proxy/Page/mise/Task/sync]] follows an instance's `logseq-entity::` links, imports its entity definitions, and follows each definition's `entity-tasks::` links. Task references declare their executable files and required task dependencies through the companion-file contract on [[Logseq/Entity/Mise/Task]]. - Shared entity definitions and task dependencies are visited once per sync. Cyclic relationships are allowed and terminate after their pages have been visited. - A referenced entity definition with no source file is reported as a logical page. A declared companion task or task dependency requires a source reference page and its mapped implementation files; missing task content stops application. + - ## Properties a proxy's destination owns + - `entity-proxy-destination-properties::` names, as a comma-separated list of property keys, the properties on an instance that belong to the garden holding a [[Logseq/Entity/Proxy/Page]] of it rather than to the source page. Re-syncing that proxy keeps the destination's values for them. An absent property names none. + - For example, [[Logseq/Entity/Podcast/Episode]] names `podcast-guid` and `podcast-published-at`, which the publishing garden records. - ## Relationship to [[Logseq/Entity]] - [[Logseq/Entity]] is the general conceptual model — what an entity is and how any page is marked as one. This page is the entity type whose instances are the definition pages themselves, so the garden's set of entity types is queryable through its backlinks rather than kept as a hand-maintained list. \ No newline at end of file diff --git a/gitp-garden/pages/Logseq___Entity___Mise___Task.md b/gitp-garden/pages/Logseq___Entity___Mise___Task.md index 78811721..73b098f5 100644 --- a/gitp-garden/pages/Logseq___Entity___Mise___Task.md +++ b/gitp-garden/pages/Logseq___Entity___Mise___Task.md @@ -11,7 +11,7 @@ logseq-proxy-last-sync-date:: [[2026-10-06]] - Built-in mise subcommands such as `mise run` are [[Logseq/Entity/CLI/Command]] instances. Jobs invoked through them are Mise Task instances. - ## Naming and links - Task reference pages belong to the topic served by the job: `Topic/mise/Task/`. A namespace prefix already expressed by the topic can be omitted from the page name while remaining in `task-name::`. - - For example, `episode:sync` lives at [[GitP/mise/Task/episode/sync]]. Repository ownership remains explicit in frontmatter. + - For example, `logseq:entity:proxy:page:sync` lives at [[Logseq/Entity/Proxy/Page/mise/Task/sync]]: the topic already expresses `logseq:entity:proxy:page`. Repository ownership remains explicit in frontmatter. - If the topic contains distinct implementations of the same task name, add a repository or config-scope segment before the task path. Displayed invocation text retains the exact runnable name. - ## Frontmatter - `logseq-entity::` marks the page as a Mise Task instance. diff --git a/gitp-garden/pages/Logseq___Entity___Podcast___Episode.md b/gitp-garden/pages/Logseq___Entity___Podcast___Episode.md index 435187ca..f736a7ba 100644 --- a/gitp-garden/pages/Logseq___Entity___Podcast___Episode.md +++ b/gitp-garden/pages/Logseq___Entity___Podcast___Episode.md @@ -1,4 +1,5 @@ logseq-entity:: [[Logseq/Entity/Definition]], [[Logseq/Entity/Proxy/Page]] +entity-proxy-destination-properties:: podcast-guid, podcast-published-at logseq-proxy-url:: logseq://graph/logseq-encode-garden?page=Logseq%2FEntity%2FPodcast%2FEpisode logseq-proxy-codeforge-url:: https://github.com/codekiln/logseq-encode-garden/blob/main/pages/Logseq___Entity___Podcast___Episode.md logseq-proxy-last-sync-date:: [[2026-10-06]] @@ -28,7 +29,7 @@ logseq-proxy-last-sync-date:: [[2026-10-06]] - Mark instances with **`logseq-entity:: [[Logseq/Entity/Podcast/Episode]]`** so this definition page collects backlinks to every episode. - Set **`created-by::`** to the host. - Set **`date-created::`** to the air date for listening notes, or the recording date for a producer-owned session. - - The publishing garden records feed identity with [[Logseq/Entity/Podcast/Episode/Frontmatter/podcast-guid]] and publication time with [[Logseq/Entity/Podcast/Episode/Frontmatter/podcast-published-at]]. + - The publishing garden records feed identity with [[Logseq/Entity/Podcast/Episode/Frontmatter/podcast-guid]] and publication time with [[Logseq/Entity/Podcast/Episode/Frontmatter/podcast-published-at]]. When the episode page there is a [[Logseq/Entity/Proxy/Page]], re-syncing it keeps both. - Optional: **`logseq-created-time-year::`** linking to the matching [[Logseq/Entity/Time/Year]] instance; see that page for how it differs from `date-created::`. - **Never add, remove, or edit a `tags::` line on an existing page.** Older episode pages carry assorted `tags::` values; those are the author's and stay as they are. - ## Page shape diff --git a/gitp-garden/pages/Logseq___Entity___Proxy___Page.md b/gitp-garden/pages/Logseq___Entity___Proxy___Page.md index a9a1e8a5..b05be0d2 100644 --- a/gitp-garden/pages/Logseq___Entity___Proxy___Page.md +++ b/gitp-garden/pages/Logseq___Entity___Proxy___Page.md @@ -1,6 +1,7 @@ alias:: [[Proxy Page]], [[Logseq Proxy Page]] entity-tasks:: [[Logseq/Entity/Proxy/Page/mise/Task/sync]] logseq-entity:: [[Logseq/Entity/Definition]], [[Logseq/Entity/Proxy/Page]] +entity-proxy-destination-properties:: public logseq-proxy-url:: logseq://graph/logseq-encode-garden?page=Logseq%2FEntity%2FProxy%2FPage logseq-proxy-codeforge-url:: https://github.com/codekiln/logseq-encode-garden/blob/main/pages/Logseq___Entity___Proxy___Page.md logseq-proxy-last-sync-date:: [[2026-10-06]] @@ -25,7 +26,9 @@ logseq-proxy-last-sync-date:: [[2026-10-06]] - No shape of its own. The body is the source page's body, so the source's shape — and the shape of whatever entity the page is primarily — governs. - ## Invariants - **`tags::` on an existing page is never touched.** On re-sync the destination's `tags::` win outright and the source's are discarded. On first create, the source's `tags::` come across with the copy unless the user says otherwise. - - **Destination properties survive a re-sync.** Only the proxy keys are rewritten; every other property line already on the page stays. The **body**, by contrast, is replaced wholesale by the source's. + - **The source owns the properties, except those the destination owns.** A re-sync adds properties the source gained since the last sync, updates those whose value changed, and removes those the source no longer has. The **body** is replaced wholesale by the source's. + - **The destination owns** `tags::`, the proxy keys, and every property an entity definition names in `entity-proxy-destination-properties::`. The definitions consulted are the ones in the source page's `logseq-entity::`, plus this one, which names `public::`: whether a page is published is the destination garden's decision. A re-sync keeps the destination's value of an owned property and never takes the source's, including when the destination has none. + - **A property only the destination has goes on re-sync** unless the destination owns it. A garden that wants to keep a local property declares it on the entity definition. The preview names every property it adds, updates or removes. - **A name collision is never resolved silently.** If the destination page exists and has no `logseq-proxy-url::`, it is a real page that happens to share the name — stop and ask before writing anything. - ## Finding and deduplicating - Proxies scatter across `pages/` wherever their mirrored names fall, so there is no namespace to narrow a search to. Always search repo-wide from the garden root. @@ -44,17 +47,18 @@ logseq-proxy-last-sync-date:: [[2026-10-06]] - ## Syncing an instance - **Source file:** the file the code forge URL names, or else `pages/.md` under the graph root, with `page=` decoded (`%2F` is `/`) and each `/` written as `___`. - **First copy:** the whole source file, with the proxy keys set in its property block. - - **Re-sync:** the source's body under the destination's property block, with the proxy keys rewritten, per **Invariants** above. + - **Re-sync:** the source's body under a property block merged per **Invariants** above. Properties already on the destination keep their order; properties new from the source follow them, and the proxy keys come last. - **Property block:** the run of `key:: value` lines from the top of the file, ending at the first line of any other kind — a blank line, a `- # Heading`, or plain text. A destination with no property block gains one above its first line, holding the proxy keys. - **Assets:** every `../assets/` the source body links to is copied to the same `assets/` here, replacing what is there. A missing source asset is reported and the sync goes on. - - **Report:** the source path, the destination path, the proxy key values written, whether it was a first copy or a re-sync, and the assets copied. + - **Embeds:** with `--follow-embeds`, each page embedded with the `embed` macro is synced as a proxy too, recursively, so a page and the pages it embeds sync in one batch. A missing embedded page is reported and the sync goes on. + - **Report:** the source path, the destination path, the proxy key values written, whether it was a first copy or a re-sync, the properties added, updated or removed, and the assets copied. - ## Executable sync - [[Logseq/Entity/Proxy/Page/mise/Task/sync]] previews the page, entity definitions, companion task references, implementation files, and assets imported by a sync. `--apply` applies the previewed changes. - An instance's `logseq-entity::` links lead to its source entity definitions. Each definition's `entity-tasks::` declares companion task references, whose file mappings and task dependencies follow [[Logseq/Entity/Mise/Task]]. The sync visits shared and cyclic dependencies once. - `.logseq-proxy/manifest.json` in the destination graph records source repository and graph root, logical page identity, source-to-destination file mappings, ownership, and last imported content hashes. - - The manifest retains the latest source companion declarations. When a proxy becomes the source for another garden, task discovery uses those declarations and resolves imported implementation paths through the source manifest. Destination page properties remain preserved during re-sync. + - The manifest retains the latest source companion declarations. When a proxy becomes the source for another garden, task discovery uses those declarations and resolves imported implementation paths through the source manifest. - Re-sync refreshes source changes where imported implementation files still match their recorded hashes. Local implementation edits, unowned destination files, and competing source claims produce conflicts before application. - - Destination page properties survive re-sync under the existing frontmatter rules; source page bodies refresh. Files removed from upstream task mappings are reported as cleanup candidates and retained locally. + - Page properties and bodies refresh from the source per **Invariants** above. Files removed from upstream task mappings are reported as cleanup candidates and retained locally. - A preview reports logical entity definitions without files separately from missing declared tasks or implementation files. Missing source assets remain warnings; missing task requirements stop application. - ## Examples in this garden - [[Book/ML with PyTorch and Scikit-Learn]] diff --git a/gitp-garden/pages/Logseq___Entity___Proxy___Page___mise___Task___sync.md b/gitp-garden/pages/Logseq___Entity___Proxy___Page___mise___Task___sync.md index e0e3bf86..45830527 100644 --- a/gitp-garden/pages/Logseq___Entity___Proxy___Page___mise___Task___sync.md +++ b/gitp-garden/pages/Logseq___Entity___Proxy___Page___mise___Task___sync.md @@ -21,14 +21,19 @@ logseq-proxy-last-sync-date:: [[2026-10-06]] mise run logseq:entity:proxy:page:sync --source /path/to/logseq-encode-garden --destination /path/to/other-garden --page 'Logseq/Entity/Proxy/Page' --apply ~~~ - `--page` is the source page's logical name, with slash namespaces. On re-sync, `--source` can be omitted when the destination page's proxy metadata resolves to a local source graph. + - `--follow-embeds` also syncs every page the page embeds with the `embed` macro, and the pages those embed, in the same batch. A podcast session and its embedded recording, artwork and preset pages sync together this way: + - ~~~sh + mise run logseq:entity:proxy:page:sync --source /path/to/logseq-encode-garden --destination /path/to/other-garden --page 'GitP/A/Session/26/09/24-Thu' --follow-embeds --apply + ~~~ - ## Inputs and outputs - Reads the source page and the entity definitions named by `logseq-entity::`. Definitions declare companion tasks with `entity-tasks::`; task references declare implementation paths and required task dependencies through [[Logseq/Entity/Mise/Task]]. - - Copies pages under their source logical names, adds [[Logseq/Entity/Proxy/Page]] membership, and sets source URL, available code forge URL, and sync date. Existing destination page properties, including `tags::`, survive re-sync; source bodies replace destination bodies. + - Copies pages under their source logical names, adds [[Logseq/Entity/Proxy/Page]] membership, and sets source URL, available code forge URL, and sync date. Source bodies replace destination bodies. + - On re-sync, page properties follow the source: new ones are added, changed ones updated, and ones the source dropped are removed. `tags::`, the proxy keys, and the properties named by an entity definition's `entity-proxy-destination-properties::` keep their destination values. [[Logseq/Entity/Proxy/Page]] states the rule; the preview lists each property added, updated or removed. - Copies linked `../assets/` files and explicitly mapped task files. Source implementation paths start at the repository root, even when the source graph occupies a subdirectory; destination paths start at the destination graph root. - Records imported ownership, source paths, destination paths, and last imported hashes in `.logseq-proxy/manifest.json`. A repeat run previews only the changes still needed. - - The manifest keeps current source task declarations and path mappings for onward imports from a proxy garden. These declarations track upstream task changes while existing destination frontmatter remains intact. + - The manifest keeps current source task declarations and path mappings for onward imports from a proxy garden. - ## Side effects - - Preview reads local files and reports planned changes, conflicts, missing assets, and upstream removal candidates. `--apply` creates or refreshes the previewed imports and manifest after validation. + - Preview reads local files and reports planned changes, property changes, conflicts, missing assets and embedded pages, and upstream removal candidates. `--apply` creates or refreshes the previewed imports and manifest after validation. - An existing page without proxy metadata is a name collision. A task file with local edits, an unowned file at the target path, or competing source ownership stops application. Destination edits remain available for review. - Files removed from upstream task mappings remain in the destination and appear as cleanup candidates. Missing linked source assets are warnings; missing declared task requirements stop application. - Application stages writes and retains recovery information or restores previous files if a write fails. diff --git a/mise-tasks/episode/sync b/mise-tasks/episode/sync deleted file mode 100755 index 93bfc8bf..00000000 --- a/mise-tasks/episode/sync +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env python3 -#MISE description="Mirror episode and asset pages from the knowledge garden" -#MISE tools={python="3.13.16"} -#USAGE flag "--source " required=#true help="Source Logseq garden root" -#USAGE flag "--destination " help="Destination graph root (defaults to gitp-garden)" -#USAGE arg "..." help="Exact logical episode page names" -from pathlib import Path -import runpy - -runpy.run_path(str(Path(__file__).resolve().parents[2] / "scripts/sync_episode.py"), run_name="__main__") diff --git a/mise.toml b/mise.toml index 6d3f4a66..6c6c776e 100644 --- a/mise.toml +++ b/mise.toml @@ -12,7 +12,7 @@ description = "Check that rss.xml matches public episode pages" run = "uv run --no-project python scripts/build_rss.py --check" [tasks."rss:test"] -description = "Test episode proxy sync and podcast feed generation" +description = "Test podcast feed generation and website preparation" run = "uv run --no-project python -m unittest discover -s tests" [tasks."site:query:check"] @@ -22,4 +22,4 @@ run = "nbb-logseq scripts/check_homepage.cljs" [task_config] # gitp-garden/mise-tasks holds companion tasks imported with garden proxy pages, # such as logseq:entity:proxy:page:sync. -includes = ["mise-tasks", "gitp-garden/mise-tasks"] +includes = ["gitp-garden/mise-tasks"] diff --git a/scripts/build_rss.py b/scripts/build_rss.py index aa05a361..46c46ca5 100644 --- a/scripts/build_rss.py +++ b/scripts/build_rss.py @@ -10,7 +10,7 @@ from urllib.request import Request, urlopen import re -from sync_episode import split_page, properties, page_path +from logseq_page import split_page, properties, page_path import xml.etree.ElementTree as ET diff --git a/scripts/logseq_page.py b/scripts/logseq_page.py new file mode 100644 index 00000000..4fda3414 --- /dev/null +++ b/scripts/logseq_page.py @@ -0,0 +1,26 @@ +"""Read Logseq page files: the property block, the body, and page-name paths.""" +from __future__ import annotations +import re + + +def split_page(text): + lines = text.splitlines(keepends=True) + count = 0 + for line in lines: + if not re.match(r'^[a-z][a-z0-9-]*:: ', line): + break + count += 1 + return lines[:count], ''.join(lines[count:]) + + +def properties(lines): + return dict(line.rstrip('\n').split(':: ', 1) for line in lines) + + +def page_path(garden, name): + if not name or any(part in ('', '.', '..') for part in name.split('/')) or any(c in name for c in '\\\n\r\x00') or '___' in name: + raise ValueError(f'Invalid page name: {name!r}') + path = garden / 'pages' / (name.replace('/', '___') + '.md') + if not path.resolve().is_relative_to((garden / 'pages').resolve()): + raise ValueError(f'Page escapes garden: {name}') + return path diff --git a/scripts/prepare_site.py b/scripts/prepare_site.py index 1376e6b1..8df2cc72 100644 --- a/scripts/prepare_site.py +++ b/scripts/prepare_site.py @@ -6,7 +6,7 @@ import re import shutil from urllib.parse import urlparse -from sync_episode import split_page, properties, page_path +from logseq_page import split_page, properties, page_path ROOT = Path(__file__).resolve().parents[1] diff --git a/scripts/sync_episode.py b/scripts/sync_episode.py deleted file mode 100644 index 3590e2b9..00000000 --- a/scripts/sync_episode.py +++ /dev/null @@ -1,121 +0,0 @@ -"""Mirror episode pages and their embedded asset pages from a Logseq garden.""" -from __future__ import annotations -import argparse -from datetime import date -import os -from pathlib import Path -import re -import tempfile -from urllib.parse import quote, unquote - -PROXY_KEYS = ('logseq-proxy-url', 'logseq-proxy-codeforge-url', 'logseq-proxy-last-sync-date') -ROOT = Path(__file__).resolve().parents[1] - -def split_page(text): - lines = text.splitlines(keepends=True) - count = 0 - for line in lines: - if not re.match(r'^[a-z][a-z0-9-]*:: ', line): - break - count += 1 - return lines[:count], ''.join(lines[count:]) - -def properties(lines): - return dict(line.rstrip('\n').split(':: ', 1) for line in lines) - -def page_path(garden, name): - if not name or any(part in ('', '.', '..') for part in name.split('/')) or any(c in name for c in '\\\n\r\x00') or '___' in name: - raise ValueError(f'Invalid page name: {name!r}') - path = garden / 'pages' / (name.replace('/', '___') + '.md') - if not path.resolve().is_relative_to((garden / 'pages').resolve()): - raise ValueError(f'Page escapes garden: {name}') - return path - -def sync(source, destination, names, graph='logseq-encode-garden', forge='https://github.com/codekiln/logseq-encode-garden/blob/main', today=None): - """Validate the whole batch before replacing any destination files.""" - source, destination = Path(source), Path(destination) - today = today or date.today().isoformat() - pending = list(names) - changes = {} - reports = [] - visited = set() - while pending: - name = pending.pop(0) - if name in visited: - continue - visited.add(name) - src, dest = page_path(source, name), page_path(destination, name) - src_lines, body = split_page(src.read_text(encoding='utf-8')) - source_props = properties(src_lines) - if dest.exists(): - dest_lines, _ = split_page(dest.read_text(encoding='utf-8')) - props = properties(dest_lines) - expected = f'logseq://graph/{quote(graph, safe="")}?page={quote(name, safe="")}' - if props.get('logseq-proxy-url') != expected: - raise ValueError(f'Page collision or different proxy source: {dest}') - lines = [line for line in dest_lines if line.split(':: ', 1)[0] not in PROXY_KEYS] - mode = 'resync' - else: - lines = [line for line in src_lines if line.split(':: ', 1)[0] not in PROXY_KEYS] - entities = source_props.get('logseq-entity', '') - marker = '[[Logseq/Entity/Proxy/Page]]' - if marker not in entities: - entities = f'{entities}, {marker}' if entities else marker - lines = [line for line in lines if not line.startswith('logseq-entity:: ')] - lines.append(f'logseq-entity:: {entities}\n') - mode = 'create' - proxy = { - 'logseq-proxy-url': f'logseq://graph/{quote(graph, safe="")}?page={quote(name, safe="")}', - 'logseq-proxy-codeforge-url': f'{forge}/pages/{quote(src.name, safe="")}', - 'logseq-proxy-last-sync-date': f'[[{today}]]', - } - lines = [line if line.endswith('\n') else line + '\n' for line in lines] - lines.extend(f'{key}:: {value}\n' for key, value in proxy.items()) - changes[dest] = (''.join(lines) + body).encode() - pending.extend(re.findall(r'\{\{embed\s+\[\[([^\]]+)\]\]\s*\}\}', body)) - # Session download links also need their asset-page bodies on the site. - pending.extend(link for link in re.findall(r'\[\[([^\]]+)\]\]', body) if link.startswith('GitP/A/Session/') and '/Asset/' in link) - assets = [] - for relative in re.findall(r'\]\((\.\./assets/[^)]+)\)', body): - relative = unquote(relative.split(' "', 1)[0]) - asset = (src.parent / relative).resolve() - if not asset.is_relative_to((source / 'assets').resolve()): - raise ValueError(f'Asset escapes garden: {relative}') - target = destination / 'assets' / asset.relative_to((source / 'assets').resolve()) - if not target.resolve().is_relative_to((destination / 'assets').resolve()): - raise ValueError(f'Asset destination escapes garden: {target}') - if asset.is_file(): - changes[target] = asset.read_bytes() - assets.append(str(target)) - else: - reports.append(f'Missing source asset: {asset}') - reports.append(f'{mode}: {src} -> {dest}; {proxy}; assets: {assets}') - temporary = [] - try: - for target, content in changes.items(): - target.parent.mkdir(parents=True, exist_ok=True) - fd, path = tempfile.mkstemp(dir=target.parent, prefix='.proxy-') - with os.fdopen(fd, 'wb') as handle: - handle.write(content) - temporary.append((Path(path), target)) - for path, target in temporary: - path.replace(target) - finally: - for path, _ in temporary: - path.unlink(missing_ok=True) - return reports - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--source', type=Path, required=True, help='Source garden root') - parser.add_argument('--destination', type=Path, default=ROOT / 'gitp-garden') - parser.add_argument('pages', nargs='+', help='Exact logical page names') - args = parser.parse_args() - try: - for report in sync(args.source, args.destination, args.pages): - print(report) - except (ValueError, OSError) as error: - parser.exit(1, f'{error}\n') - -if __name__ == '__main__': - main() diff --git a/tests/test_sync_episode.py b/tests/test_sync_episode.py deleted file mode 100644 index 835352f6..00000000 --- a/tests/test_sync_episode.py +++ /dev/null @@ -1,71 +0,0 @@ -import sys -from pathlib import Path -import tempfile -import unittest -sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts')) -from sync_episode import sync - -class SyncTests(unittest.TestCase): - def setUp(self): - self.tmp = tempfile.TemporaryDirectory() - self.addCleanup(self.tmp.cleanup) - self.source, self.dest = [Path(self.tmp.name) / name for name in ('source', 'dest')] - for graph in (self.source, self.dest): - (graph / 'pages').mkdir(parents=True) - self.source_page = self.source / 'pages/Session.md' - self.dest_page = self.dest / 'pages/Session.md' - self.source_page.write_text('tags:: [[Music]]\nlogseq-entity:: [[Logseq/Entity/Podcast/Episode]]\n- # Session\n\t- Description.\n') - def run_sync(self, names=None): - return sync(self.source, self.dest, names or ['Session'], today='2026-10-04') - def test_create_then_resync_preserves_destination_properties(self): - self.run_sync() - text = self.dest_page.read_text().replace('tags:: [[Music]]', 'tags:: [[Curated]]') - self.dest_page.write_text('public:: true\npodcast-guid:: stable\n' + text) - self.source_page.write_text('tags:: [[Changed]]\n- # New body\n') - self.run_sync() - result = self.dest_page.read_text() - self.assertIn('tags:: [[Curated]]', result) - self.assertIn('public:: true', result) - self.assertIn('podcast-guid:: stable', result) - self.assertIn('[[Logseq/Entity/Proxy/Page]]', result) - self.assertIn('- # New body', result) - self.assertNotIn('Description.', result) - def test_collision_aborts_entire_batch(self): - (self.source / 'pages/Other.md').write_text('- # Other\n') - (self.dest / 'pages/Other.md').write_text('- # Local page\n') - with self.assertRaisesRegex(ValueError, 'collision'): - self.run_sync(['Session', 'Other']) - self.assertFalse(self.dest_page.exists()) - def test_recursive_embeds_and_asset_copy(self): - self.source_page.write_text('- # Session\n\t- {{embed [[Asset]]}}\n') - (self.source / 'pages/Asset.md').write_text('- # Asset\n\t- [Recording](../assets/music/test.mp3)\n\t- {{embed [[Session]]}}\n') - (self.source / 'assets/music').mkdir(parents=True) - (self.source / 'assets/music/test.mp3').write_bytes(b'audio') - self.run_sync() - self.assertTrue((self.dest / 'pages/Asset.md').exists()) - self.assertEqual((self.dest / 'assets/music/test.mp3').read_bytes(), b'audio') - def test_session_download_pages_sync_without_following_notes(self): - self.source_page.write_text("- [[GitP/A/Session/26/09/24-Thu/Asset/Preset/A/mfpz]]\n- [[Music/Log]]\n") - (self.source / "pages/GitP___A___Session___26___09___24-Thu___Asset___Preset___A___mfpz.md").write_text("- # Patch\n") - self.run_sync() - asset = self.dest / "pages/GitP___A___Session___26___09___24-Thu___Asset___Preset___A___mfpz.md" - self.assertTrue(asset.exists()) - self.assertNotIn("public::", asset.read_text()) - def test_missing_asset_reported(self): - self.source_page.write_text('- [Absent](../assets/absent.mp3)\n') - self.assertIn('Missing source asset', '\n'.join(self.run_sync())) - def test_traversal_rejected_before_write(self): - self.source_page.write_text('- [Escape](../assets/../../secret)\n') - with self.assertRaisesRegex(ValueError, 'escapes'): - self.run_sync() - self.assertFalse(self.dest_page.exists()) - def test_invalid_name_rejected(self): - with self.assertRaises(ValueError): - self.run_sync(['../secret']) - def test_other_graph_proxy_rejected(self): - self.dest_page.write_text('logseq-proxy-url:: logseq://graph/other?page=Session\n- # Other\n') - with self.assertRaisesRegex(ValueError, 'different proxy source'): - self.run_sync() - -if __name__ == '__main__': - unittest.main()