fix(studio): safe markdown-folder persistence that follows outside edits

CHANGELOG.md

@@ -28,6 +28,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Welcome screen: on a first visit the sample model gets its own card (preview, what it contains, "Explore the sample") in place of the placeholder diagram, and opens on the System Context view instead of the all-elements canvas
 - Desktop installers for macOS (dmg, Apple Silicon + Intel), Windows (NSIS) and Linux (AppImage), built by electron-builder and drafted as a GitHub Release when a `v*` tag is pushed
 - App icon for the desktop builds
+- Markdown-folder models follow edits made outside Studio (another editor, `git pull`): the active document's folder is polled (desktop every second, browser every two) and reloaded from disk, keeping the open view and selection; a save never overwrites a file changed on disk since Studio read it
 
 ### Changed
 - The Studio app is named radical.studio (installers, window and page title); radical.tools names the family of apps
@@ -39,6 +40,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Production desktop build showed a blank window: the CSP header handler dropped every other response header (including `Content-Type`), so the renderer's module scripts were refused
 - Hub concept files' `views` were ignored by the viewer and on import
 - Importing from the Hub no longer stacks one "Imported …" notification per concept over the template-parameter form
+- Markdown-folder models: renaming or moving an element whose description was never opened lost that description on the next save
+- Markdown-folder models: saving deleted files Studio did not write — any top-level `.json` (e.g. `package.json`) and any file under `nodes/`, such as a hand-written `README.md`; "Save as folder…" into a folder that holds other files but no model now asks first
+- Markdown-folder models: two saves overlapping (a slow disk, a large model) could delete an element's file altogether
+- Switching documents while a slow one was still loading could load it into — and autosave it over — the document switched to; an edit made just before a switch now lands in the document being left
+- Markdown-folder saves write only the files that changed, and only the model's own files are read (a folder inside a repository no longer walks `node_modules`)
 - Presentation slides without a captured canvas (the sample model's walkthrough, imported slides) lay nodes out with their view's saved positions instead of whatever layout was on screen, so they no longer need a Smart Layout to look right; a slide after a milestone slide shows the live model again instead of the milestone's subset
 
 [Unreleased]: https://github.com/radicaltools/radical.tools/compare/HEAD

apps/e2e/README.md

@@ -28,6 +28,7 @@ The first time, install Chromium: `npx playwright install chromium`.
 | `studio/layout` | Smart Layout: no overlapping siblings, children inside their parent, same result on every run |
 | `studio/presentation` | Playing slides, keyboard navigation, slide deep links |
 | `studio/export` | PNG and SVG export produce real files |
+| `studio/folders` | Models stored as a folder of Markdown files: save as folder, reload, renames that move files, edits made outside Studio, files Studio must not touch |
 | `studio/known-issues` | Open bugs, marked `test.fail` (see below) |
 | `hub/hub` | Landing page, categories, search, concept canvas / wiki / table, `.radical` download |
 
@@ -39,6 +40,13 @@ tests the catalogue that is about to be deployed.
 Tests start from a known document instead of clicking one together: see
 `fixtures/bookstore.radical` and `Studio.seed()` in `support/fixtures.ts`.
 
+Folder tests (`support/folder.ts`) run the web build's File System Access
+code against a directory in the browser's origin-private file system: the
+directory picker is replaced with one that returns it, and the test edits it
+directly to play another editor. They run in a persistent browser profile,
+because Chromium crashes when an off-the-record page (Playwright's default)
+reads a stored directory handle back from IndexedDB.
+
 ## Screenshots
 
 Visual checks use `toHaveScreenshot`. Baselines live in

apps/e2e/support/folder.ts

@@ -0,0 +1,81 @@
+import type { Page } from '@playwright/test'
+import { mkdtemp, rm } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { test as base } from './fixtures'
+
+/**
+ * The Studio `test`, with its page in a persistent (on-disk) browser profile
+ * instead of Playwright's usual off-the-record one. Chromium takes the whole
+ * browser down when an off-the-record page reads a stored OPFS directory handle
+ * back from IndexedDB — which the app does on every folder save and load. A
+ * regular profile, like a user's, is not affected.
+ */
+export const test = base.extend<{ page: Page }>({
+  page: async ({ playwright }, use, testInfo) => {
+    const { viewport, deviceScaleFactor, locale, timezoneId, colorScheme, baseURL, userAgent, headless } = testInfo.project.use
+    const profile = await mkdtemp(join(tmpdir(), 'radical-e2e-'))
+    const context = await playwright.chromium.launchPersistentContext(profile, {
+      headless: headless ?? true, viewport, deviceScaleFactor, locale, timezoneId, colorScheme, baseURL, userAgent,
+    })
+    await use(context.pages()[0] ?? await context.newPage())
+    await context.close()
+    await rm(profile, { recursive: true, force: true })
+  },
+})
+
+/**
+ * A model folder for Studio's web build, which works on real directories
+ * through the File System Access API. Playwright cannot drive the native
+ * directory picker, so `showDirectoryPicker()` is replaced with one that hands
+ * out a directory in the page's origin-private file system (OPFS): a real
+ * directory handle the app reads, writes, stores in IndexedDB and polls, and
+ * one the test can edit behind the app's back. Each test's profile has its
+ * own, empty OPFS.
+ */
+export class ModelFolder {
+  constructor(readonly page: Page, readonly name = 'model') {}
+
+  /** Make the directory picker return this folder. Call before navigating. */
+  async install(): Promise<void> {
+    await this.page.addInitScript((name) => {
+      ;(window as unknown as { showDirectoryPicker: () => Promise<FileSystemDirectoryHandle> })
+        .showDirectoryPicker = async () =>
+          (await navigator.storage.getDirectory()).getDirectoryHandle(name, { create: true })
+    }, this.name)
+  }
+
+  /** Every file in the folder: relative path → content. */
+  async files(): Promise<Record<string, string>> {
+    return this.page.evaluate(async (name) => {
+      const out: Record<string, string> = {}
+      const walk = async (dir: FileSystemDirectoryHandle, prefix: string): Promise<void> => {
+        for await (const [entryName, entry] of (dir as unknown as AsyncIterable<[string, FileSystemHandle]>)) {
+          const rel = prefix ? `${prefix}/${entryName}` : entryName
+          if (entry.kind === 'directory') await walk(entry as FileSystemDirectoryHandle, rel)
+          else out[rel] = await (await (entry as FileSystemFileHandle).getFile()).text()
+        }
+      }
+      const root = await (await navigator.storage.getDirectory()).getDirectoryHandle(name, { create: true })
+      await walk(root, '')
+      return out
+    }, this.name)
+  }
+
+  /** Paths of the files in the folder, sorted. */
+  async paths(): Promise<string[]> {
+    return Object.keys(await this.files()).sort()
+  }
+
+  /** Write a file as another program would (an editor, git), outside Studio. */
+  async write(path: string, content: string): Promise<void> {
+    await this.page.evaluate(async ([name, rel, text]) => {
+      let dir = await (await navigator.storage.getDirectory()).getDirectoryHandle(name, { create: true })
+      const parts = rel.split('/')
+      for (const part of parts.slice(0, -1)) dir = await dir.getDirectoryHandle(part, { create: true })
+      const writable = await (await dir.getFileHandle(parts[parts.length - 1], { create: true })).createWritable()
+      await writable.write(text)
+      await writable.close()
+    }, [this.name, path, content] as const)
+  }
+}

apps/e2e/tests/studio/folders.spec.ts

@@ -0,0 +1,152 @@
+import { expect } from '../../support/fixtures'
+import { ModelFolder, test } from '../../support/folder'
+import type { Page } from '@playwright/test'
+
+// Models persisted as a folder of Markdown files (web build: File System
+// Access API over an OPFS directory, see support/folder.ts). Systems and
+// containers get a directory with an _index.md, other elements a file: the
+// bookstore fixture becomes nodes/customer.md, nodes/payment-provider/_index.md
+// and nodes/bookstore/_index.md with api/_index.md, web-app.md, orders-db.md.
+
+let folder: ModelFolder
+
+test.beforeEach(async ({ page, studio }) => {
+  folder = new ModelFolder(page)
+  await folder.install()
+  await studio.seed()
+  await studio.open('v-context')
+  await expect(studio.nodes).toHaveCount(3)
+})
+
+async function openModels(page: Page): Promise<void> {
+  await page.getByTitle('Menu', { exact: true }).click()
+  await page.getByRole('menuitem', { name: /Manage models/ }).click()
+  await expect(page.getByRole('dialog', { name: 'Models' })).toBeVisible()
+}
+
+/** Convert the seeded document to a folder through the Models dialog. */
+async function saveAsFolder(page: Page): Promise<void> {
+  await openModels(page)
+  await page.getByRole('button', { name: 'Save as folder…' }).click()
+  const models = page.getByRole('dialog', { name: 'Models' })
+  await expect(models.getByRole('tab', { name: /Folders/ })).toHaveAttribute('aria-selected', 'true')
+  await expect(models.locator('.docmgr-badge.md')).toBeVisible()
+  await models.getByRole('button', { name: 'Close' }).click()
+}
+
+/** The properties-panel Label field of the selected element. */
+function labelField(page: Page) {
+  return page.locator('.props-field').filter({ has: page.getByText('Label', { exact: true }) }).locator('input')
+}
+
+async function rename(page: Page, from: string, to: string): Promise<void> {
+  await expect(labelField(page)).toHaveValue(from)
+  await labelField(page).fill(to)
+  await labelField(page).press('Tab')
+}
+
+test('save as folder writes one Markdown file per element', async ({ page }) => {
+  await saveAsFolder(page)
+  await expect.poll(() => folder.paths()).toEqual(expect.arrayContaining([
+    'radical.md',
+    'nodes/customer.md',
+    'nodes/bookstore/_index.md',
+    'nodes/bookstore/api/_index.md',
+    'nodes/bookstore/web-app.md',
+    'nodes/bookstore/orders-db.md',
+    'nodes/payment-provider/_index.md',
+    'relations.json',
+    '_layout.json',
+  ]))
+  const files = await folder.files()
+  expect(files['radical.md']).toContain('radicalFormat: "md-folder"')
+  expect(files['nodes/customer.md']).toContain('label: "Customer"')
+  expect(files['nodes/customer.md']).toContain('Buys books online')
+})
+
+test('a folder-backed model survives a reload', async ({ page, studio }) => {
+  await saveAsFolder(page)
+  await studio.node('payments').click()
+  await rename(page, 'Payment Provider', 'Card Gateway')
+  await expect.poll(() => folder.paths()).toContain('nodes/card-gateway/_index.md')
+
+  await page.reload()
+  await studio.ready()
+  await expect(studio.node('payments')).toContainText('Card Gateway')
+  await openModels(page)
+  await expect(page.getByRole('dialog', { name: 'Models' }).locator('.docmgr-badge.md')).toBeVisible()
+})
+
+test('moving files keeps the descriptions of elements never opened', async ({ page, studio }) => {
+  await saveAsFolder(page)
+  // After a reload, descriptions are read from the files only on demand.
+  await page.reload()
+  await studio.ready()
+
+  // Renaming the system moves its children's files, whose bodies were
+  // never loaded; a second save must not lose them. (The reload lands on the
+  // full canvas, where Bookstore is expanded: click its header.)
+  await studio.node('bookstore').click({ position: { x: 24, y: 12 } })
+  await rename(page, 'Bookstore', 'Online Shop')
+  await expect.poll(() => folder.paths()).toContain('nodes/online-shop/api/_index.md')
+  await studio.node('customer').click()
+  await rename(page, 'Customer', 'Reader')
+  await expect.poll(() => folder.paths()).toContain('nodes/reader.md')
+
+  const files = await folder.files()
+  expect(files['nodes/online-shop/api/_index.md']).toContain('Orders and catalogue API')
+  expect(files['nodes/online-shop/web-app.md']).toContain('Catalogue and checkout UI')
+  expect(files['nodes/online-shop/orders-db.md']).toContain('Orders and stock')
+  expect(Object.keys(files).some((p) => p.startsWith('nodes/bookstore/'))).toBe(false)
+})
+
+test('an edit made outside the app shows up on the canvas', async ({ page, studio }) => {
+  await saveAsFolder(page)
+  await expect.poll(() => folder.paths()).toContain('nodes/customer.md')
+
+  const file = (await folder.files())['nodes/customer.md']
+  await folder.write('nodes/customer.md', file.replace('label: "Customer"', 'label: "Book Lover"'))
+
+  // The folder is polled every couple of seconds.
+  await expect(studio.node('customer')).toContainText('Book Lover', { timeout: 10_000 })
+  // The app never wrote its own version back over the edit; its next save
+  // just files the element under its new name.
+  await expect.poll(() => folder.paths()).toContain('nodes/book-lover.md')
+  await page.waitForTimeout(3000)
+  const files = await folder.files()
+  expect(files['nodes/book-lover.md']).toContain('label: "Book Lover"')
+  expect(Object.values(files).some((c) => c.includes('label: "Customer"'))).toBe(false)
+})
+
+test('files the model does not own are left alone', async ({ page, studio }) => {
+  await folder.write('package.json', '{"name":"shop"}')
+  await folder.write('nodes/README.md', '# Notes\n')
+
+  // The folder holds files but no model, so the app asks first.
+  const asked = page.waitForEvent('dialog').then(async (d) => {
+    expect(d.message()).toContain('is not a Radical model folder')
+    await d.accept()
+  })
+  await saveAsFolder(page)
+  await asked
+  await expect.poll(() => folder.paths()).toContain('nodes/payment-provider/_index.md')
+
+  await studio.node('payments').click()
+  await page.getByRole('button', { name: 'Delete', exact: true }).click()
+  await expect.poll(() => folder.paths()).not.toContain('nodes/payment-provider/_index.md')
+
+  const files = await folder.files()
+  expect(files['package.json']).toBe('{"name":"shop"}')
+  expect(files['nodes/README.md']).toBe('# Notes\n')
+})
+
+test('declining the prompt leaves a non-model folder untouched', async ({ page }) => {
+  await folder.write('nodes/README.md', '# Notes\n')
+  page.once('dialog', (d) => d.dismiss())
+  await openModels(page)
+  await page.getByRole('button', { name: 'Save as folder…' }).click()
+  const models = page.getByRole('dialog', { name: 'Models' })
+  await expect(models.locator('.docmgr-badge.ls')).toBeVisible()
+  await expect(models.locator('.docmgr-badge.md')).toHaveCount(0)
+  expect(await folder.paths()).toEqual(['nodes/README.md'])
+})

apps/studio/src/main/diskFolderStorage.ts

@@ -0,0 +1,56 @@
+// ─── Md-folder storage on the local disk ─────────────────────────────────────
+//
+// The `MdFolderStorage` behind Electron's md-folder sessions (see
+// @radical/common/formats/mdFolderSync). Kept free of Electron imports so it
+// can be tested against a real temporary directory.
+
+import type { Stats } from 'fs'
+import { readFile, writeFile, mkdir, readdir, rm, stat } from 'fs/promises'
+import { join, resolve, dirname, sep } from 'path'
+import { isMdFolderModelPath } from '@radical/common/formats/mdFolder'
+import type { MdFolderStorage } from '@radical/common/formats/mdFolderSync'
+
+/** Absolute path of `rel` inside `root`; throws if `rel` would escape it. */
+function inside(root: string, rel: string): string {
+  const abs = resolve(root, rel)
+  if (!abs.startsWith(resolve(root) + sep)) throw new Error(`path outside the model folder: ${rel}`)
+  return abs
+}
+
+/** A model folder on the local disk. Only the manifest, the top-level
+ *  sidecars and `nodes/**` are listed — the rest of the folder (a repo's
+ *  `node_modules`, say) is never walked. Hidden files are ignored. */
+export function diskFolderStorage(root: string): MdFolderStorage {
+  const stampOf = (s: Stats): string => `${s.mtimeMs}:${s.size}`
+  return {
+    async list() {
+      const stamps: Record<string, string> = {}
+      const add = async (rel: string): Promise<void> => {
+        try { stamps[rel] = stampOf(await stat(join(root, rel))) } catch { /* gone meanwhile */ }
+      }
+      const walk = async (rel: string): Promise<void> => {
+        const entries = await readdir(join(root, rel), { withFileTypes: true }).catch(() => [])
+        for (const entry of entries) {
+          if (entry.name.startsWith('.')) continue
+          const child = `${rel}/${entry.name}`
+          if (entry.isDirectory()) await walk(child)
+          else if (entry.isFile() && isMdFolderModelPath(child)) await add(child)
+        }
+      }
+      const top = await readdir(root, { withFileTypes: true }).catch(() => [])
+      for (const entry of top) {
+        if (entry.isFile() && isMdFolderModelPath(entry.name)) await add(entry.name)
+        else if (entry.isDirectory() && entry.name === 'nodes') await walk('nodes')
+      }
+      return stamps
+    },
+    read: (rel) => readFile(inside(root, rel), 'utf-8'),
+    async write(rel, content) {
+      const abs = inside(root, rel)
+      await mkdir(dirname(abs), { recursive: true })
+      await writeFile(abs, content, 'utf-8')
+      return stampOf(await stat(abs))
+    },
+    remove: (rel) => rm(inside(root, rel), { force: true }),
+  }
+}