From c09d9e63d6e0889e6bb377da21c9330401be0633 Mon Sep 17 00:00:00 2001 From: thiagomajesk <13930952+thiagomajesk@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:04:23 -0300 Subject: [PATCH] Add filtered fetch and explicit object hydration --- docs/CLI.md | 15 +- docs/CLIENT.md | 25 ++ docs/FILE_REFERENCE.md | 4 + docs/REPO.md | 4 +- src/commands/clone.ts | 21 +- src/commands/fetch.ts | 28 +- src/commands/gc.ts | 2 + src/commands/pull.ts | 2 + src/hooks.ts | 2 + src/lib/fetch-helpers.ts | 13 +- src/lib/partial-clone.ts | 46 ++++ src/lib/repack.ts | 2 + src/lib/transport/remote.ts | 4 + src/lib/transport/smart-http.ts | 10 + src/lib/transport/transport.ts | 9 +- src/repo/fetching.ts | 60 +++++ src/repo/index.ts | 2 + test/integration/partial-clone-native.test.ts | 255 ++++++++++++++++++ 18 files changed, 489 insertions(+), 15 deletions(-) create mode 100644 src/lib/partial-clone.ts create mode 100644 src/repo/fetching.ts create mode 100644 test/integration/partial-clone-native.test.ts diff --git a/docs/CLI.md b/docs/CLI.md index e52b0db0..dc6c4d06 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -378,6 +378,7 @@ Options: --bare Create a bare clone -b, --branch Checkout this branch instead of HEAD --depth Create a shallow clone with history truncated to N commits + --filter Omit blob contents (requires --no-checkout or --bare) --single-branch Clone only the history of the specified or default branch --no-single-branch Clone all branches even with --depth --no-tags Don't clone any tags @@ -485,12 +486,14 @@ Arguments: refspec... Refspec(s) to fetch Options: - --all Fetch from all remotes - -p, --prune Remove stale remote-tracking refs - --tags Also fetch tags - --depth Limit fetching to the specified number of commits - --unshallow Convert a shallow repository to a complete one - -q, --quiet be more quiet + --all Fetch from all remotes + -p, --prune Remove stale remote-tracking refs + --tags Also fetch tags + --no-tags Do not fetch tags + --filter Omit blob contents (blob:none) + --depth Limit fetching to the specified number of commits + --unshallow Convert a shallow repository to a complete one + -q, --quiet be more quiet ``` ## git gc diff --git a/docs/CLIENT.md b/docs/CLIENT.md index 334405f7..f74340e7 100644 --- a/docs/CLIENT.md +++ b/docs/CLIENT.md @@ -35,6 +35,31 @@ const git = createGit({ ## Repository access +### Partial clone + +Use `fetch --filter=blob:none` or `clone --filter=blob:none --no-checkout` (or `--bare`) to transfer commits and trees without blobs. The remote must advertise the `filter` capability. The filter and promisor remote are saved in repository configuration and reused by later fetches. `fetch --no-tags` and the remote's saved `tagOpt` suppress automatic tag downloads. + +Use `fetchObjects(repo, objectIds, { remote: "origin", batchSize: 128 })` from `just-git/repo` to retrieve selected missing objects. Requests use the configured transport, credentials, network policy and fetch hooks without updating refs, `FETCH_HEAD` or shallow boundaries. The server must allow requests for reachable object IDs. Existing objects and duplicates are skipped. Request blob IDs from trees to avoid downloading the graph reachable from a commit. + +Object reads never download implicitly; retrieve required blobs before reading them. Partial repositories support filtered fetch and the repository API, but not general checkout, diff or merge operations. Pull, GC and repack reject partial repositories before modifying them. The local/cross-VFS transport filters its outgoing pack but still reads source blobs during enumeration; Smart HTTP avoids transferring them. + +```ts +import { fetchObjects, flattenTree, readCommit, resolveRef } from "just-git/repo"; + +await git.exec("fetch --depth=1 --filter=blob:none --no-tags origin"); +const repo = await git.findRepo(); +if (repo) { + const revision = await resolveRef(repo, "refs/remotes/origin/main"); + if (!revision) throw new Error("Remote main branch not found"); + const entries = await flattenTree(repo, (await readCommit(repo, revision)).tree); + const selected = entries.filter(({ path }) => path.startsWith("src/")); + await fetchObjects( + repo, + selected.map(({ hash }) => hash), + ); +} +``` + `git.findRepo()` returns a `GitContext` for the current working directory, the bridge between command execution via `exec` and programmatic access via the [repo module](REPO.md). Like `exec`, it uses the instance's default `fs` and `cwd`, with optional per-call overrides. The returned context carries all operator-level extensions (hooks, identity, credentials, config overrides) configured on the instance. ```ts diff --git a/docs/FILE_REFERENCE.md b/docs/FILE_REFERENCE.md index 3811d1df..b4dff23b 100644 --- a/docs/FILE_REFERENCE.md +++ b/docs/FILE_REFERENCE.md @@ -164,6 +164,10 @@ Exported symbols from `src/lib/`. Generated by `bun scripts/gen-lib-reference.ts - `deflate`, `inflate`, `inflateObject` +### lib/partial-clone.ts + +- `validateObjectFilter`, `configurePartialClone`, `requireCompleteRepository` + ### lib/patch-id.ts - `computePatchId` diff --git a/docs/REPO.md b/docs/REPO.md index 8d144bf6..96fe3d1b 100644 --- a/docs/REPO.md +++ b/docs/REPO.md @@ -10,7 +10,7 @@ For command-execution configuration, see [CLIENT.md](CLIENT.md). For the embedda ## GitRepo -Everything in this module operates on `GitRepo`, a minimal interface representing a git repository as an object store and a ref store: +Most operations in this module accept `GitRepo`, a minimal interface representing a git repository as an object store and a ref store: ```ts interface GitRepo { @@ -21,6 +21,8 @@ interface GitRepo { This is the type that unifies the client and server sides of just-git. Any function that accepts `GitRepo` works identically regardless of what's behind it: an in-memory VFS, a SQLite database, Postgres, or a custom implementation. +`fetchObjects(ctx, objectIds, options?)` requires a `GitContext` from `git.findRepo()` because it uses configured remotes, credentials and network policy. It retrieves missing objects in batches without changing refs or shallow boundaries; a storage-backed `GitRepo` alone is insufficient. See [Partial clone](CLIENT.md#partial-clone). + ### How you get a GitRepo **From a virtual filesystem**: `findRepo` walks up from a path looking for a `.git` directory and returns a `GitContext` (which extends `GitRepo` with filesystem access, paths, and credentials): diff --git a/src/commands/clone.ts b/src/commands/clone.ts index a1abee51..b8f6fa69 100644 --- a/src/commands/clone.ts +++ b/src/commands/clone.ts @@ -2,6 +2,7 @@ import type { GitExtensions } from "../git.ts"; import { isRejection } from "../hooks.ts"; import { err, fatal, quietFlag } from "../lib/command-utils.ts"; import { readConfig, writeConfig } from "../lib/config.ts"; +import { validateObjectFilter } from "../lib/partial-clone.ts"; import { buildIndex, defaultStat, @@ -32,6 +33,7 @@ export function registerCloneCommand(parent: Command, ext?: GitExtensions) { bare: f().describe("Create a bare clone"), branch: o.string().alias("b").describe("Checkout this branch instead of HEAD"), depth: o.number().describe("Create a shallow clone with history truncated to N commits"), + filter: o.string().describe("Omit blob contents (requires --no-checkout or --bare)"), singleBranch: f().describe("Clone only the history of the specified or default branch"), noSingleBranch: f().describe("Clone all branches even with --depth"), noTags: f().describe("Don't clone any tags"), @@ -39,6 +41,9 @@ export function registerCloneCommand(parent: Command, ext?: GitExtensions) { quiet: quietFlag("be more quiet"), }, handler: async (args, ctx) => { + const filter = args.filter === undefined ? undefined : validateObjectFilter(args.filter); + if (filter && !args.noCheckout && !args.bare) + return fatal("Filtered clone requires --no-checkout or --bare; hydrate objects explicitly"); const repository = args.repository; if (!repository) { return fatal("You must specify a repository to clone."); @@ -145,6 +150,12 @@ export function registerCloneCommand(parent: Command, ext?: GitExtensions) { url: sourcePath, fetch: "+refs/heads/*:refs/remotes/origin/*", }; + if (filter) { + config['remote "origin"']!.promisor = "true"; + config['remote "origin"']!.partialclonefilter = filter; + config.core = { ...config.core, repositoryformatversion: "1" }; + config.extensions = { ...config.extensions, partialclone: "origin" }; + } await writeConfig(newCtx, config); await ext?.hooks?.postClone?.({ repo: newCtx, @@ -220,6 +231,12 @@ export function registerCloneCommand(parent: Command, ext?: GitExtensions) { remoteSection.tagOpt = "--no-tags"; } config['remote "origin"'] = remoteSection; + if (filter) { + remoteSection.promisor = "true"; + remoteSection.partialclonefilter = filter; + config.core = { ...config.core, repositoryformatversion: "1" }; + config.extensions = { ...config.extensions, partialclone: "origin" }; + } // Build wants list — single-branch limits to target branch only const wants: ObjectId[] = []; @@ -244,7 +261,9 @@ export function registerCloneCommand(parent: Command, ext?: GitExtensions) { } const shallowOpts: ShallowFetchOptions | undefined = - depthOpt !== undefined && depthOpt > 0 ? { depth: depthOpt } : undefined; + (depthOpt !== undefined && depthOpt > 0) || filter + ? { depth: depthOpt, filter } + : undefined; if (wants.length > 0) { const fetchResult = await transport.fetch(wants, [], shallowOpts); diff --git a/src/commands/fetch.ts b/src/commands/fetch.ts index 394107dc..5ed869d2 100644 --- a/src/commands/fetch.ts +++ b/src/commands/fetch.ts @@ -11,6 +11,7 @@ import { type TransferRefLine, } from "../lib/command-utils.ts"; import { readConfig } from "../lib/config.ts"; +import { configurePartialClone, validateObjectFilter } from "../lib/partial-clone.ts"; import { autoFollowReachableTags, collectFetchHaves, @@ -47,11 +48,14 @@ export function registerFetchCommand(parent: Command, ext?: GitExtensions) { all: f().describe("Fetch from all remotes"), prune: f().alias("p").describe("Remove stale remote-tracking refs"), tags: f().describe("Also fetch tags"), + noTags: f().describe("Do not fetch tags"), + filter: o.string().describe("Omit blob contents (blob:none)"), depth: o.number().describe("Limit fetching to the specified number of commits"), unshallow: f().describe("Convert a shallow repository to a complete one"), quiet: quietFlag("be more quiet"), }, handler: async (args, ctx) => { + if (args.tags && args.noTags) return fatal("--tags and --no-tags cannot be used together"); const gitCtxOrError = await requireGitContext(ctx.fs, ctx.cwd, ext); if (isCommandError(gitCtxOrError)) return gitCtxOrError; const gitCtx = gitCtxOrError; @@ -86,6 +90,8 @@ export function registerFetchCommand(parent: Command, ext?: GitExtensions) { ext, depth, args.quiet, + args.filter, + args.noTags, ); if (result.stderr) allStderr.push(result.stderr); if (result.exitCode !== 0) lastExit = result.exitCode; @@ -104,6 +110,8 @@ export function registerFetchCommand(parent: Command, ext?: GitExtensions) { ext, depth, args.quiet, + args.filter, + args.noTags, ); }, }); @@ -282,10 +290,16 @@ async function fetchOneRemote( ext?: GitExtensions, depth?: number, quiet = false, + requestedFilter?: string, + noTags = false, ): Promise { const resolved = await resolveRemoteTransportOrError(gitCtx, remoteName, env); if (isCommandError(resolved)) return resolved; const { transport, config } = resolved; + const selectedFilter = requestedFilter ?? config.partialCloneFilter; + const filter = selectedFilter === undefined ? undefined : validateObjectFilter(selectedFilter); + if (filter && config.anonymous) return fatal("Filtered fetch requires a configured remote"); + const suppressTags = noTags || config.noTags; const hasExplicitRefspecs = !!rawRefspecs && rawRefspecs.length > 0; // A raw-URL (anonymous) remote has no remote-tracking namespace. Like git, @@ -316,6 +330,7 @@ async function fetchOneRemote( } } } + if (filter) await configurePartialClone(gitCtx, remoteName, filter); return { stdout: "", stderr: "", exitCode: 0 }; } @@ -354,11 +369,16 @@ async function fetchOneRemote( const haveSet = new Set(haves); const filteredWants = wants.filter((w) => !haveSet.has(w)); - const { existingShallows, shallowOpts } = await prepareShallowFetch(gitCtx, depth); + const { existingShallows, shallowOpts } = await prepareShallowFetch(gitCtx, depth, filter); // When depth/unshallow is requested, we must call fetch even with no // new wants so the shallow boundary negotiation can happen. - const effectiveWants = filteredWants.length > 0 ? filteredWants : shallowOpts ? wants : []; + const effectiveWants = + filteredWants.length > 0 + ? filteredWants + : depth !== undefined || (filter && !config.partialCloneFilter) + ? wants + : []; if (effectiveWants.length > 0) { const fetchResult = await transport.fetch(effectiveWants, haves, shallowOpts); @@ -367,6 +387,7 @@ async function fetchOneRemote( await applyShallowUpdates(gitCtx, fetchResult.shallowUpdates, existingShallows); } } + if (filter) await configurePartialClone(gitCtx, remoteName, filter); const ident = await getReflogIdentity(gitCtx, env); const { refLines, hadTagRejection, appliedUpdates } = await applyFetchRefUpdates( @@ -381,13 +402,14 @@ async function fetchOneRemote( refLines.push({ prefix: " * branch", from: "HEAD", to: "FETCH_HEAD" }); } - if (!tags) { + if (!tags && !suppressTags) { const tagLines = await autoFollowReachableTags({ gitCtx, transport, remoteRefs, ident, reflogAction: "fetch", + filter, }); if (!quiet) refLines.push(...tagLines); } diff --git a/src/commands/gc.ts b/src/commands/gc.ts index 81c5ab5b..bc043161 100644 --- a/src/commands/gc.ts +++ b/src/commands/gc.ts @@ -1,4 +1,5 @@ import type { GitExtensions } from "../git.ts"; +import { requireCompleteRepository } from "../lib/partial-clone.ts"; import { isCommandError, quietFlag, requireGitContext } from "../lib/command-utils.ts"; import { type GitConfig, readConfig } from "../lib/config.ts"; import { collectAllRoots } from "../lib/gc-roots.ts"; @@ -22,6 +23,7 @@ export function registerGcCommand(parent: Command, ext?: GitExtensions) { const gitCtxOrError = await requireGitContext(ctx.fs, ctx.cwd, ext); if (isCommandError(gitCtxOrError)) return gitCtxOrError; const gitCtx = gitCtxOrError; + await requireCompleteRepository(gitCtx); // Step 1: Pack refs await writePackedRefs(gitCtx); diff --git a/src/commands/pull.ts b/src/commands/pull.ts index 8bf24609..46b06b54 100644 --- a/src/commands/pull.ts +++ b/src/commands/pull.ts @@ -1,4 +1,5 @@ import type { GitExtensions } from "../git.ts"; +import { requireCompleteRepository } from "../lib/partial-clone.ts"; import { isRejection } from "../hooks.ts"; import { buildAbbrevResolver, @@ -87,6 +88,7 @@ export function registerPullCommand(parent: Command, ext?: GitExtensions) { const gitCtxOrError = await requireGitContext(ctx.fs, ctx.cwd, ext); if (isCommandError(gitCtxOrError)) return gitCtxOrError; const gitCtx = gitCtxOrError; + await requireCompleteRepository(gitCtx, "Pull; use filtered fetch and the repo SDK instead"); const depthResult = await normalizeFetchDepth(gitCtx, args); if (isCommandError(depthResult)) return depthResult; diff --git a/src/hooks.ts b/src/hooks.ts index d4865549..3d5c4883 100644 --- a/src/hooks.ts +++ b/src/hooks.ts @@ -194,6 +194,8 @@ export interface PreCheckoutEvent { /** Fired before a fetch begins. Return a {@link Rejection} to block. */ export interface PreFetchEvent { + /** Explicit object hydration requests; refspecs is empty for these calls. */ + readonly objectIds?: readonly ObjectId[]; readonly repo: GitRepo; readonly remote: string; readonly url: string; diff --git a/src/lib/fetch-helpers.ts b/src/lib/fetch-helpers.ts index 5e6e52de..935b9f9e 100644 --- a/src/lib/fetch-helpers.ts +++ b/src/lib/fetch-helpers.ts @@ -71,11 +71,13 @@ export async function collectFetchHaves(gitCtx: GitContext): Promise export async function prepareShallowFetch( gitCtx: GitContext, depth?: number, + filter?: "blob:none", ): Promise { - const existingShallows = depth !== undefined ? await readShallowCommits(gitCtx) : undefined; + const existingShallows = + depth !== undefined || filter ? await readShallowCommits(gitCtx) : undefined; return { existingShallows, - shallowOpts: depth !== undefined ? { depth, existingShallows } : undefined, + shallowOpts: depth !== undefined || filter ? { depth, existingShallows, filter } : undefined, }; } @@ -85,6 +87,7 @@ export async function autoFollowReachableTags(options: { remoteRefs: RemoteRef[]; ident: ReflogWriteIdentity; reflogAction: "fetch" | "pull"; + filter?: "blob:none"; }): Promise { const { gitCtx, transport, remoteRefs, ident, reflogAction } = options; @@ -106,7 +109,11 @@ export async function autoFollowReachableTags(options: { } } if (tagObjectWants.length > 0) { - await transport.fetch(tagObjectWants, await collectFetchHaves(gitCtx)); + await transport.fetch( + tagObjectWants, + await collectFetchHaves(gitCtx), + options.filter ? { filter: options.filter } : undefined, + ); } const refLines: TransferRefLine[] = []; diff --git a/src/lib/partial-clone.ts b/src/lib/partial-clone.ts new file mode 100644 index 00000000..edeb920a --- /dev/null +++ b/src/lib/partial-clone.ts @@ -0,0 +1,46 @@ +import { parseConfig, readConfig, setConfigValueRaw } from "./config.ts"; +import { join } from "./path.ts"; +import type { GitContext } from "./types.ts"; + +export function validateObjectFilter(filter: string): "blob:none" { + if (filter !== "blob:none") throw new Error("Only the blob:none object filter is supported"); + return filter; +} + +/** Persist the promise that deliberately omitted objects remain on this remote. */ +export async function configurePartialClone(ctx: GitContext, remote: string, filter: "blob:none") { + const path = join(ctx.commonDir, "config"); + let raw = await ctx.fs.readFile(path); + const config = parseConfig(raw); + const remoteSection = `remote "${remote}"`; + const section = config[remoteSection]; + if (!section?.url) throw new Error("Filtered fetch requires a configured remote"); + if ( + section.promisor === "true" && + section.partialclonefilter === filter && + config.extensions?.partialclone === remote + ) + return; + for (const [section, key, value] of [ + [remoteSection, "promisor", "true"], + [remoteSection, "partialclonefilter", filter], + ["core", "repositoryformatversion", "1"], + ["extensions", "partialclone", remote], + ] as const) + raw = setConfigValueRaw(raw, section, key, value); + await ctx.fs.writeFile(path, raw); +} + +/** Maintenance must not mistake promised-but-absent objects for corruption. */ +export async function requireCompleteRepository( + ctx: GitContext, + operation = "Repository maintenance", +): Promise { + const config = await readConfig(ctx); + if ( + config.extensions?.partialclone || + Object.values(config).some((section) => section.promisor === "true") + ) { + throw new Error(`${operation} is not supported for partial clones`); + } +} diff --git a/src/lib/repack.ts b/src/lib/repack.ts index dd40a690..3a88c050 100644 --- a/src/lib/repack.ts +++ b/src/lib/repack.ts @@ -1,4 +1,5 @@ import type { FileSystem } from "../fs.ts"; +import { requireCompleteRepository } from "./partial-clone.ts"; import { findBestDeltas } from "./pack/delta.ts"; import { buildPackIndexFromMeta, PackIndex } from "./pack/pack-index.ts"; import { type DeltaPackInput, writePackDeltified } from "./pack/packfile.ts"; @@ -62,6 +63,7 @@ interface RepackResult { */ export async function repackFromTips(options: RepackOptions): Promise { const { gitCtx, fs, tips, cleanup } = options; + await requireCompleteRepository(gitCtx); const window = options.window ?? 10; const depth = options.depth ?? 50; diff --git a/src/lib/transport/remote.ts b/src/lib/transport/remote.ts index e6c03fd5..7f64a29f 100644 --- a/src/lib/transport/remote.ts +++ b/src/lib/transport/remote.ts @@ -49,6 +49,8 @@ export function stripAndCacheCredentials( } interface RemoteConfig { + partialCloneFilter?: string; + noTags?: boolean; name: string; url: string; fetchRefspec: string; @@ -71,6 +73,8 @@ async function getRemoteConfig(ctx: GitContext, remoteName: string): Promise, clientShallowBoundary?: Set, + filter?: "blob:none", ): Promise { const enumResult = await enumerateObjectsWithContent( ctx, @@ -484,7 +488,10 @@ async function buildDeltifiedPack( ); if (enumResult.count === 0) return undefined; - const objects = await collectEnumeration(enumResult); + const objects = (await collectEnumeration(enumResult)).filter( + (object) => filter !== "blob:none" || object.type !== "blob", + ); + if (objects.length === 0) return undefined; const deltas = findBestDeltas(objects); const inputs: DeltaPackInput[] = deltas.map((r) => ({ hash: r.hash, diff --git a/src/repo/fetching.ts b/src/repo/fetching.ts new file mode 100644 index 00000000..c9cc5776 --- /dev/null +++ b/src/repo/fetching.ts @@ -0,0 +1,60 @@ +import { isRejection } from "../hooks.ts"; +import { objectExists } from "../lib/object-db.ts"; +import { resolveRemoteTransport } from "../lib/transport/remote.ts"; +import type { GitContext, ObjectId } from "../lib/types.ts"; + +export interface FetchObjectsOptions { + /** Configured remote name (defaults to origin). */ + remote?: string; + /** Environment used by the normal credential resolver. */ + env?: Map; + /** Maximum requested objects per round trip (defaults to 128, maximum 1024). */ + batchSize?: number; +} + +/** + * Explicitly fetch missing objects by SHA-1, without changing refs, FETCH_HEAD, + * or the shallow boundary. Intended for hydrating selected blobs after a + * blobless fetch. Servers must allow requests for reachable object IDs. + * Existing objects are skipped; reads and existence checks never fetch implicitly. + */ +export async function fetchObjects( + repo: GitContext, + objectIds: readonly ObjectId[], + options: FetchObjectsOptions = {}, +): Promise<{ fetched: number }> { + const batchSize = options.batchSize ?? 128; + if (!Number.isInteger(batchSize) || batchSize < 1 || batchSize > 1024) { + throw new Error("Object fetch batch size must be an integer between 1 and 1024"); + } + const ids = [...new Set(objectIds)]; + if (ids.some((id) => !/^[a-f0-9]{40}$/.test(id))) throw new Error("Invalid SHA-1 object ID"); + const missing: ObjectId[] = []; + for (const id of ids) if (!(await objectExists(repo, id))) missing.push(id); + if (missing.length === 0) return { fetched: 0 }; + const remote = options.remote ?? "origin"; + const resolved = await resolveRemoteTransport(repo, remote, options.env ?? new Map()); + if (!resolved) throw new Error(`Cannot resolve remote '${remote}'`); + const rejection = await repo.hooks?.preFetch?.({ + repo, + remote, + url: resolved.config.url, + refspecs: [], + prune: false, + tags: false, + objectIds: missing, + }); + if (isRejection(rejection)) throw new Error(rejection.message ?? "Object fetch rejected"); + for (let offset = 0; offset < missing.length; offset += batchSize) { + const batch = missing.slice(offset, offset + batchSize); + // A commit "have" implies its blobs are present. That is deliberately + // false in a partial clone, so hydration sends no commit haves. + await resolved.transport.fetch(batch, []); + for (const id of batch) { + if (!(await objectExists(repo, id))) + throw new Error(`Remote did not provide requested object ${id}`); + } + } + await repo.hooks?.postFetch?.({ repo, remote, url: resolved.config.url, updatedRefCount: 0 }); + return { fetched: missing.length }; +} diff --git a/src/repo/index.ts b/src/repo/index.ts index 8fb74fab..d35a91df 100644 --- a/src/repo/index.ts +++ b/src/repo/index.ts @@ -1,5 +1,7 @@ // Repo operations SDK — high-level functions for working with GitRepo +export { fetchObjects, type FetchObjectsOptions } from "./fetching.ts"; + // Reading export { branchNameFromRef, diff --git a/test/integration/partial-clone-native.test.ts b/test/integration/partial-clone-native.test.ts new file mode 100644 index 00000000..b48f2202 --- /dev/null +++ b/test/integration/partial-clone-native.test.ts @@ -0,0 +1,255 @@ +import { test, expect } from "bun:test"; +import { execFileSync } from "node:child_process"; +import { mkdtemp, mkdir, writeFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { randomBytes } from "node:crypto"; +import { createGit, MemoryFileSystem } from "../../src/index.ts"; +import { setConfigValue } from "../../src/lib/config.ts"; +import { + buildCommit, + fetchObjects, + readCommit, + readBlob, + resolveRef, + flattenTree, +} from "../../src/repo/index.ts"; + +test("filtered native HTTP fetch hydrates only requested blobs and preserves unrequested files on push", async () => { + const directory = await mkdtemp(join(tmpdir(), "just-git-partial-")); + const remote = join(directory, "remote.git"); + const source = join(directory, "source"); + const git = (cwd: string, args: string[]) => + execFileSync("git", args, { cwd, encoding: "utf8" }).trim(); + git(directory, ["init", "--bare", "--initial-branch=main", remote]); + for (const [key, value] of [ + ["http.receivepack", "true"], + ["uploadpack.allowFilter", "true"], + ["uploadpack.allowReachableSHA1InWant", "true"], + ]) + git(remote, ["config", key!, value!]); + git(directory, ["init", "--initial-branch=main", source]); + git(source, ["config", "user.name", "test"]); + git(source, ["config", "user.email", "test@example.com"]); + await mkdir(join(source, "A")); + await mkdir(join(source, "B")); + await writeFile(join(source, "A/file.md"), "Local file.\n"); + await writeFile(join(source, "A/state.bin"), randomBytes(1024)); + await writeFile(join(source, "B/large.bin"), randomBytes(2 * 1024 * 1024)); + await writeFile(join(source, "B/state.bin"), randomBytes(128 * 1024)); + git(source, ["add", "."]); + git(source, ["commit", "-m", "fixture"]); + git(source, ["remote", "add", "origin", remote]); + git(source, ["push", "origin", "main"]); + git(remote, [ + "-c", + "user.name=test", + "-c", + "user.email=test@example.com", + "tag", + "-a", + "fixture", + "-m", + "fixture", + "main", + ]); + let downloaded = 0; + let requests = 0; + const server = Bun.serve({ + port: 0, + async fetch(request) { + if (request.headers.get("authorization") !== "Basic " + btoa("test:secret")) + return new Response("Unauthorized", { status: 401 }); + const url = new URL(request.url); + const bytes = new Uint8Array(await request.arrayBuffer()); + const process = Bun.spawn(["git", "http-backend"], { + stdin: bytes, + stdout: "pipe", + stderr: "pipe", + env: { + ...Bun.env, + GIT_PROJECT_ROOT: directory, + GIT_HTTP_EXPORT_ALL: "1", + PATH_INFO: url.pathname, + QUERY_STRING: url.search.slice(1), + REQUEST_METHOD: request.method, + CONTENT_TYPE: request.headers.get("content-type") ?? "", + CONTENT_LENGTH: String(bytes.length), + REMOTE_USER: "test", + }, + }); + const [data] = await Promise.all([ + new Response(process.stdout).arrayBuffer(), + new Response(process.stderr).arrayBuffer(), + process.exited, + ]); + const raw = Buffer.from(data); + let offset = raw.indexOf("\r\n\r\n"), + length = 4; + if (offset < 0) { + offset = raw.indexOf("\n\n"); + length = 2; + } + if (offset < 0) return new Response("Invalid CGI response", { status: 500 }); + const headers = new Headers(); + let status = 200; + for (const line of raw.subarray(0, offset).toString().split(/\r?\n/)) { + const colon = line.indexOf(":"); + if (line.slice(0, colon).toLowerCase() === "status") + status = Number( + line + .slice(colon + 1) + .trim() + .split(" ")[0], + ); + else headers.set(line.slice(0, colon), line.slice(colon + 1).trim()); + } + return new Response(raw.subarray(offset + length), { status, headers }); + }, + }); + try { + const fs = new MemoryFileSystem(); + const makeClient = () => + createGit({ + fs, + cwd: "/repo", + credentials: () => ({ type: "basic", username: "test", password: "secret" }), + network: { + allowed: ["localhost"], + fetch: async (input, init) => { + requests++; + const response = await fetch(input, init); + downloaded += (await response.clone().arrayBuffer()).byteLength; + return response; + }, + }, + }); + const client = makeClient(); + const run = async (command: string) => { + const result = await client.exec(command); + expect(result.exitCode, result.stderr).toBe(0); + return result; + }; + await fs.mkdir("/repo", { recursive: true }); + await run("init -b main"); + await run(`remote add origin ${server.url}remote.git`); + await run("config remote.origin.fetch +refs/heads/main:refs/remotes/origin/main"); + await setConfigValue((await client.findRepo())!, "remote.origin.tagOpt", "--no-tags"); + const preserved = + "\n# Unrelated configuration\n[credential]\n\thelper = first\n\thelper = second\n"; + await fs.writeFile("/repo/.git/config", (await fs.readFile("/repo/.git/config")) + preserved); + const configBefore = await fs.readFile("/repo/.git/config"); + const rejected = createGit({ + fs, + cwd: "/repo", + hooks: { preFetch: () => ({ reject: true, message: "Denied by hook" }) }, + }); + expect((await rejected.exec("fetch --filter blob:none origin")).exitCode).not.toBe(0); + expect(await fs.readFile("/repo/.git/config")).toBe(configBefore); + await run("fetch --depth 1 --filter blob:none --no-tags origin"); + expect(await fs.readFile("/repo/.git/config")).toContain(preserved); + const repo = await client.findRepo(); + if (!repo) throw new Error("No partial repo"); + const revision = (await resolveRef(repo, "refs/remotes/origin/main"))!; + const entries = await flattenTree(repo, (await readCommit(repo, revision)).tree); + const own = entries.filter((entry) => entry.path.startsWith("A/")); + const other = entries.filter((entry) => entry.path.startsWith("B/")); + expect(downloaded).toBeLessThan(12 * 1024); + for (const entry of entries) expect(await repo.objectStore.exists(entry.hash)).toBe(false); + expect(await resolveRef(repo, "refs/tags/fixture")).toBeNull(); + const refsBefore = await repo.refStore.listRefs(); + const headBefore = await fs.readFile("/repo/.git/FETCH_HEAD"); + const boundaryBefore = await fs.readFile("/repo/.git/shallow"); + expect( + await fetchObjects( + repo, + own.map((entry) => entry.hash), + { batchSize: 1 }, + ), + ).toEqual({ fetched: 2 }); + for (const entry of own) expect(await repo.objectStore.exists(entry.hash)).toBe(true); + for (const entry of other) expect(await repo.objectStore.exists(entry.hash)).toBe(false); + expect(await repo.refStore.listRefs()).toEqual(refsBefore); + expect(await fs.readFile("/repo/.git/FETCH_HEAD")).toBe(headBefore); + expect(await fs.readFile("/repo/.git/shallow")).toBe(boundaryBefore); + const before = requests; + expect(await fetchObjects(repo, [...own.map((entry) => entry.hash), own[0]!.hash])).toEqual({ + fetched: 0, + }); + expect(requests).toBe(before); + await expect(fetchObjects(repo, ["not-a-hash"])).rejects.toThrow("Invalid SHA-1"); + for (const filter of ["tree:0", ""]) { + await setConfigValue(repo, "remote.origin.partialclonefilter", filter); + const requestsBefore = requests; + const invalid = await client.exec("fetch origin"); + expect(invalid.exitCode).not.toBe(0); + expect(invalid.stderr).toContain("Only the blob:none object filter is supported"); + expect(requests).toBe(requestsBefore); + await run("fetch --filter blob:none --no-tags origin"); + } + for (const command of [ + "fetch --filter= origin", + `clone --filter= --no-checkout ${server.url}remote.git /invalid-clone`, + ]) { + const invalid = await client.exec(command); + expect(invalid.exitCode).not.toBe(0); + expect(invalid.stderr).toContain("Only the blob:none object filter is supported"); + } + await writeFile(join(source, "A/file.md"), "Incoming file edit.\n"); + await writeFile(join(source, "B/large.bin"), randomBytes(2 * 1024 * 1024)); + git(source, ["add", "."]); + git(source, ["commit", "-m", "both directories change"]); + git(source, ["push", "origin", "main"]); + downloaded = 0; + const reopened = makeClient(); + const refreshed = await reopened.exec("fetch origin"); + expect(refreshed.exitCode, refreshed.stderr).toBe(0); + const reopenedRepo = (await reopened.findRepo())!; + const incomingRevision = (await resolveRef(reopenedRepo, "refs/remotes/origin/main"))!; + expect(incomingRevision).not.toBe(revision); + const incomingEntries = await flattenTree( + reopenedRepo, + (await readCommit(reopenedRepo, incomingRevision)).tree, + ); + const incomingOther = incomingEntries.filter((entry) => entry.path.startsWith("B/")); + for (const entry of incomingOther) + expect(await reopenedRepo.objectStore.exists(entry.hash)).toBe(false); + expect(downloaded).toBeLessThan(12 * 1024); + const incomingOwn = incomingEntries.filter((entry) => entry.path.startsWith("A/")); + expect( + await fetchObjects( + reopenedRepo, + incomingOwn.map((entry) => entry.hash), + ), + ).toEqual({ fetched: 1 }); + expect(downloaded).toBeLessThan(12 * 1024); + await reopenedRepo.refStore.writeRef("refs/heads/main", incomingRevision); + const next = await buildCommit(reopenedRepo, { + branch: "main", + files: { "A/file.md": "Edited file.\n", "A/state.bin": randomBytes(1024) }, + message: "partial client edit", + author: { name: "test", email: "test@example.com" }, + }); + await reopenedRepo.refStore.writeRef("refs/heads/main", next.hash); + const pushed = await reopened.exec("push origin main:main"); + expect(pushed.exitCode, pushed.stderr).toBe(0); + for (const entry of incomingOther) { + expect(await reopenedRepo.objectStore.exists(entry.hash)).toBe(false); + expect(git(remote, ["rev-parse", "main:" + entry.path])).toBe(entry.hash); + } + git(remote, ["fsck", "--full"]); + for (const command of ["gc", "repack -a", "pull origin main"]) { + const result = await client.exec(command); + expect(result.exitCode).not.toBe(0); + expect(result.stderr).toContain("partial clones"); + } + expect(new TextDecoder().decode(await readBlob(repo, own[0]!.hash)).length).toBeGreaterThan(0); + git(remote, ["config", "uploadpack.allowFilter", "false"]); + const unsupported = await makeClient().exec("fetch --depth 1 --filter blob:none origin"); + expect(unsupported.exitCode).not.toBe(0); + expect(unsupported.stderr).toContain("filter capability missing"); + } finally { + server.stop(true); + await rm(directory, { recursive: true, force: true }); + } +}, 30000);