feat: derive in-memory layer from another FileSystem

README.md

@@ -58,6 +58,37 @@ Effect.runPromise(
 )
 ```
 
+Snapshot a real directory tree into a fresh in-memory layer. Files, empty
+directories, symlinks, file mode, owner, and access/modification times are
+all replicated. The caller provides the `FileSystem.FileSystem` service used
+for the walk, so the source can live on Node, Bun, Deno, or any other
+Effect-TS `FileSystem` backend.
+
+```ts
+import { NodeFileSystem } from "@effect/platform-node-shared"
+import { Effect, FileSystem } from "effect"
+import { layerFromPath } from "@lucas-bur/effect-memfs"
+
+const program = Effect.gen(function* () {
+  const fs = yield* FileSystem.FileSystem
+  return yield* fs.readFileString("/data/fixtures/test.json")
+}).pipe(
+  Effect.provide(layerFromPath("/data/fixtures")),
+  Effect.provide(NodeFileSystem.layer),
+)
+```
+
+The source path is mounted at the same location in the in-memory
+filesystem by default; pass `{ mountAt: "/some/other/path" }` to remap.
+
+## Roadmap
+
+- [ ] **1:1 parity with Node's filesystem** — `@lucas-bur/effect-memfs` must be a
+  drop-in replacement for `NodeFileSystem`: identical behavior, error tags, and
+  edge cases. Enforced by `test/file-system-suite.test.ts`, which runs the same
+  suite against both backends. Any divergence is a bug to fix, not a quirk to
+  document. Pinned via the `TODO(1:1-parity)` marker in `src/index.ts`.
+
 ## License
 
 MIT

playground/scratch.ts

@@ -0,0 +1,52 @@
+import { NodeFileSystem, NodeRuntime } from "@effect/platform-node-shared"
+import { Console, Effect, FileSystem } from "effect"
+
+import { layerFromPath } from "../src/index.js"
+
+// it.effect("scratch: write + read roundtrip", () =>
+//   Effect.gen(function* () {
+//     const fs = yield* FileSystem.FileSystem
+
+//     yield* fs.makeDirectory("/scratch")
+//     yield* fs.writeFileString("/scratch/hello.txt", "world")
+//     const content = yield* fs.readFileString("/scratch/hello.txt")
+//     yield* Console.log(`read back: ${content}`)
+
+//     expect(content).toBe("world")
+//   }).pipe(Effect.provide(layer())),
+// )
+
+// it.effect("scratch: layer pre-populates files", () =>
+//   Effect.gen(function* () {
+//     const fs = yield* FileSystem.FileSystem
+
+//     const a = yield* fs.readFileString("/a.txt")
+//     const b = yield* fs.readFileString("/nested/b.txt")
+
+//     expect(a).toBe("A")
+//     expect(b).toBe("B")
+//   }).pipe(
+//     Effect.provide(
+//       layer({
+//         "/a.txt": "A",
+//         "/nested/b.txt": "B",
+//       }),
+//     ),
+//   ),
+// )
+
+const main = Effect.gen(function* () {
+  const fs = yield* FileSystem.FileSystem
+  const dir = yield* fs.readDirectory("/", { recursive: true })
+  yield* Console.log(dir)
+  for (let i = 0; i < dir.length; i++) {
+    yield* Console.log(yield* fs.stat("/" + dir[i]))
+  }
+
+  // yield* Console.log(fs)
+}).pipe(
+  Effect.provide(layerFromPath("./test/../test", { mountAt: "app" })),
+  Effect.provide(NodeFileSystem.layer),
+)
+
+NodeRuntime.runMain(main)

src/helpers/volume.ts

@@ -1,8 +1,155 @@
+import { Effect, Option, type FileSystem, type PlatformError } from "effect"
 import { Volume } from "memfs"
 
+import { fromSync } from "./promise.js"
+
 export interface FileTree {
   [key: string]: string | null | FileTree
 }
 
 export const makeVol = (initialFiles: FileTree = {}): Volume =>
   Volume.fromNestedJSON(initialFiles, "/")
+
+const copyMode = (vol: Volume, dst: string, info: FileSystem.File.Info, isLink: boolean): void => {
+  const mode = info.mode & 0o777
+  if (isLink) {
+    vol.lchmodSync(dst, mode)
+  } else {
+    vol.chmodSync(dst, mode)
+  }
+}
+
+const copyOwner = (vol: Volume, dst: string, info: FileSystem.File.Info, isLink: boolean): void => {
+  const uid = Option.getOrUndefined(info.uid)
+  const gid = Option.getOrUndefined(info.gid)
+  if (uid === undefined || gid === undefined) return
+  if (isLink) {
+    vol.lchownSync(dst, uid, gid)
+  } else {
+    vol.chownSync(dst, uid, gid)
+  }
+}
+
+const copyTimestamps = (
+  vol: Volume,
+  dst: string,
+  info: FileSystem.File.Info,
+  isLink: boolean,
+): void => {
+  const atime = Option.getOrUndefined(info.atime)
+  const mtime = Option.getOrUndefined(info.mtime)
+  if (atime === undefined || mtime === undefined) return
+  if (isLink) {
+    vol.lutimesSync(dst, atime, mtime)
+  } else {
+    vol.utimesSync(dst, atime, mtime)
+  }
+}
+
+const toUnix = (p: string): string => p.replace(/\\/g, "/")
+
+// memfs resolves relative paths against `process.cwd()`, which would land
+// the in-memory tree under the OS working directory. Force a leading `/`
+// so a bare relative path (e.g. `mountAt: "app"`) is treated as absolute
+// inside the isolated Volume. Paths that are already absolute — either
+// POSIX (`/foo`) or Windows with a drive letter (`C:/foo`) — are kept as-is
+// because the extra `/` would corrupt the drive-letter form on Windows
+// (`path.resolve("/C:/foo")` becomes `E:\C:\foo`).
+const toVolumePath = (p: string): string => {
+  const normalized = toUnix(p)
+  if (normalized.startsWith("/")) return normalized
+  if (/^[A-Za-z]:/.test(normalized)) return normalized
+  return "/" + normalized
+}
+
+const populate = (
+  vol: Volume,
+  fs: FileSystem.FileSystem,
+  src: string,
+  dst: string,
+): Effect.Effect<void, PlatformError.PlatformError> =>
+  Effect.gen(function* () {
+    // Detect symlinks first: readLink only succeeds on actual links,
+    // so we use that as the discriminator instead of stat (which follows).
+    const linkOpt = yield* Effect.option(fs.readLink(src))
+    if (Option.isSome(linkOpt)) {
+      yield* fromSync(() => vol.symlinkSync(linkOpt.value, dst), "symlink", dst)
+      // Apply the link's own metadata via l* variants so we never follow.
+      const stat = yield* Effect.option(fs.stat(src))
+      if (Option.isSome(stat)) {
+        copyMode(vol, dst, stat.value, true)
+        copyOwner(vol, dst, stat.value, true)
+        copyTimestamps(vol, dst, stat.value, true)
+      }
+      return
+    }
+
+    const stat = yield* fs.stat(src)
+
+    if (stat.type === "Directory") {
+      // Always grant the user write+execute bits so the recursive mkdir
+      // chain stays traversable AND children can be written. Some
+      // platforms (notably Windows) report `0o444`/`0o666` for read-only
+      // directories (e.g. `.git/objects/...`), which makes memfs reject
+      // child writes with EACCES without this.
+      const dirMode = (stat.mode & 0o777) | 0o300
+      yield* fromSync(() => vol.mkdirSync(dst, { mode: dirMode, recursive: true }), "mkdir", dst)
+      copyOwner(vol, dst, stat, false)
+      copyTimestamps(vol, dst, stat, false)
+      const children = yield* fs.readDirectory(src)
+      for (const child of children) {
+        const childSrc = src.endsWith("/") ? src + child : src + "/" + child
+        const childDst = dst.endsWith("/") ? dst + child : dst + "/" + child
+        yield* populate(vol, fs, childSrc, childDst)
+      }
+      return
+    }
+
+    if (stat.type === "File") {
+      const content = yield* fs.readFile(src)
+      // memfs rejects the post-create `open` for write on a file that was
+      // just created with a read-only mode (e.g. `0o444` for git objects).
+      // Write with a writable mode first, then restore the source mode.
+      yield* fromSync(() => vol.writeFileSync(dst, content, { mode: 0o666 }), "writeFile", dst)
+      yield* fromSync(() => vol.chmodSync(dst, stat.mode & 0o777), "chmod", dst)
+      copyOwner(vol, dst, stat, false)
+      copyTimestamps(vol, dst, stat, false)
+      return
+    }
+
+    // BlockDevice, FIFO, Socket, etc. — memfs can represent them but
+    // replicating their content/behavior is rarely useful in tests.
+    // Silently skip rather than fail loudly; the user can layer in
+    // custom handling later if needed.
+  })
+
+/**
+ * Build a `Volume` that exactly mirrors a path in another `FileSystem`.
+ *
+ * Walks the source recursively and reproduces every entry — files, empty directories, and symlinks
+ * — into the returned `Volume`. File mode, owner, and access/modification times are copied for
+ * every entry; symlinks are preserved (not followed) and their own metadata is set via the `l*`
+ * variants of the `Volume` API.
+ *
+ * The source path is mounted at the same path inside the Volume, so a source at `/data/fixtures`
+ * lands at `/data/fixtures` in the resulting in-memory filesystem. To remap, use the `mountAt`
+ * option.
+ *
+ * The `FileSystem` service is supplied by the caller, so this works with Node's filesystem, Bun's,
+ * an in-memory mock, or any other Effect-TS `FileSystem` implementation.
+ */
+export const makeVolFromFileSystem = (
+  fs: FileSystem.FileSystem,
+  path: string,
+  options?: { readonly mountAt?: string },
+): Effect.Effect<Volume, PlatformError.PlatformError> =>
+  Effect.gen(function* () {
+    const vol = new Volume()
+    // Normalize to forward slashes so the source path (which may use
+    // OS-native separators, e.g. backslashes on Windows) lands at the
+    // same place inside the memfs Volume, which only speaks POSIX.
+    const src = toUnix(path)
+    const dst = toVolumePath(options?.mountAt ?? path)
+    yield* populate(vol, fs, src, dst)
+    return vol
+  })

src/index.ts

@@ -1,6 +1,7 @@
-import { Effect, FileSystem, Layer } from "effect"
+import { Effect, FileSystem, Layer, type PlatformError } from "effect"
+import type { Volume } from "memfs"
 
-import { makeVol, type FileTree } from "./helpers/volume.js"
+import { makeVol, makeVolFromFileSystem, type FileTree } from "./helpers/volume.js"
 import { access } from "./operations/access.js"
 import { chmod } from "./operations/chmod.js"
 import { chown } from "./operations/chown.js"
@@ -26,36 +27,75 @@ import { writeFile } from "./operations/write-file.js"
 
 export { type FileTree }
 
-const makeFileSystem = (initialFiles: FileTree = {}): Effect.Effect<FileSystem.FileSystem> =>
-  Effect.sync(() => {
-    const vol = makeVol(initialFiles)
-    return FileSystem.make({
-      access: access(vol),
-      chmod: chmod(vol),
-      chown: chown(vol),
-      copy: copy(vol),
-      copyFile: copyFile(vol),
-      link: link(vol),
-      makeDirectory: makeDirectory(vol),
-      makeTempDirectory: makeTempDirectory(vol),
-      makeTempDirectoryScoped: makeTempDirectoryScoped(vol),
-      makeTempFile: makeTempFile(vol),
-      makeTempFileScoped: makeTempFileScoped(vol),
-      open: open(vol),
-      readDirectory: readDirectory(vol),
-      readFile: readFile(vol),
-      readLink: readLink(vol),
-      realPath: realPath(vol),
-      remove: remove(vol),
-      rename: rename(vol),
-      stat: stat(vol),
-      symlink: symlink(vol),
-      truncate: truncate(vol),
-      utimes: utimes(vol),
-      watch: watch(vol),
-      writeFile: writeFile(vol),
-    })
+// TODO(1:1-parity): memfs must be 1:1 interchangeable with NodeFileSystem — same
+// behavior, same error tags, same edge cases. Enforce this via the cross-validation
+// suite in `test/file-system-suite.test.ts`, which runs every case against both
+// backends. Any divergence is a bug, not a platform quirk; fix it and add a
+// regression test that pins the corrected behavior. Tracked at the project level
+// because every operation in this module depends on it.
+
+const makeFileSystem = (vol: Volume): FileSystem.FileSystem =>
+  FileSystem.make({
+    access: access(vol),
+    chmod: chmod(vol),
+    chown: chown(vol),
+    copy: copy(vol),
+    copyFile: copyFile(vol),
+    link: link(vol),
+    makeDirectory: makeDirectory(vol),
+    makeTempDirectory: makeTempDirectory(vol),
+    makeTempDirectoryScoped: makeTempDirectoryScoped(vol),
+    makeTempFile: makeTempFile(vol),
+    makeTempFileScoped: makeTempFileScoped(vol),
+    open: open(vol),
+    readDirectory: readDirectory(vol),
+    readFile: readFile(vol),
+    readLink: readLink(vol),
+    realPath: realPath(vol),
+    remove: remove(vol),
+    rename: rename(vol),
+    stat: stat(vol),
+    symlink: symlink(vol),
+    truncate: truncate(vol),
+    utimes: utimes(vol),
+    watch: watch(vol),
+    writeFile: writeFile(vol),
   })
 
 export const layer = (initialFiles: FileTree = {}): Layer.Layer<FileSystem.FileSystem> =>
-  Layer.effect(FileSystem.FileSystem)(makeFileSystem(initialFiles))
+  Layer.sync(FileSystem.FileSystem, () => makeFileSystem(makeVol(initialFiles)))
+
+/**
+ * Build a `FileSystem` layer pre-populated from a real directory.
+ *
+ * Walks `path` using the caller-provided `FileSystem.FileSystem` service (e.g.
+ * `NodeFileSystem.layer`) and copies the entire tree — files, empty directories, symlinks, plus
+ * mode, owner, and timestamps — into a fresh in-memory `Volume`. The in-memory `FileSystem` service
+ * is then built on top of that `Volume`.
+ *
+ * The source path is mounted at the same path inside the in-memory filesystem; pass `mountAt` to
+ * remap it.
+ *
+ * @example
+ *   ```ts
+ *   import { NodeFileSystem } from "@effect/platform-node-shared"
+ *   import { Effect, FileSystem } from "effect"
+ *   import { layerFromPath } from "@lucas-bur/effect-memfs"
+ *
+ *   const program = Effect.gen(function* () {
+ *     const fs = yield* FileSystem.FileSystem
+ *     return yield* fs.readFileString("/data/fixtures/test.json")
+ *   }).pipe(Effect.provide(layerFromPath("/data/fixtures")), Effect.provide(NodeFileSystem.layer))
+ *   ```
+ */
+export const layerFromPath = (
+  path: string,
+  options?: { readonly mountAt?: string },
+): Layer.Layer<FileSystem.FileSystem, PlatformError.PlatformError, FileSystem.FileSystem> =>
+  Layer.unwrap(
+    Effect.gen(function* () {
+      const fs = yield* FileSystem.FileSystem
+      const vol = yield* makeVolFromFileSystem(fs, path, options)
+      return Layer.succeed(FileSystem.FileSystem, makeFileSystem(vol))
+    }),
+  )

src/operations/read-directory.ts

@@ -3,21 +3,23 @@ import type { Volume } from "memfs"
 
 import { fromPromise } from "../helpers/promise.js"
 
-// NOTE: memfs returns paths with a leading "/" in recursive mode
-// when the root directory is read, so we normalize them.
+// Match the shape of `node:fs/promises.readdir(path, { recursive: true })`:
+// paths are returned relative to the input, separated by forward slashes, and
+// intermediate directories are included. memfs returns relative paths only
+// when the input has no trailing slash, so we strip one (keeping the root `/`).
+// For root itself, memfs returns absolute paths — we drop the leading `/`.
+const stripTrailingSlash = (p: string): string =>
+  p === "/" ? p : p.endsWith("/") ? p.slice(0, -1) : p
 
 export const readDirectory =
   (vol: Volume): FileSystem.FileSystem["readDirectory"] =>
   (path, options) =>
     fromPromise(
       () => {
-        const dir = vol.promises.readdir(path, {
+        const dir = vol.promises.readdir(stripTrailingSlash(path), {
           recursive: options?.recursive ?? false,
         }) as Promise<string[]>
-
-        return options?.recursive
-          ? dir.then((entries) => entries.map((e) => (e.startsWith("/") ? e.slice(1) : e)))
-          : dir
+        return dir.then((entries) => entries.map((e) => (e.startsWith("/") ? e.slice(1) : e)))
       },
       "readDirectory",
       path,