fix: let the model library scan read large checkpoint metadata

app/electron/agent/engine/library.test.ts

@@ -155,6 +155,59 @@ try {
   const large = await lowMemory.scan()
   assert.equal(large.models.every((model) => !model.loadable), true)
 
+
+  // Large checkpoint metadata: a shard index above the old 4 MiB cap now scans, resolves, and stays bounded.
+  const bigModels = path.join(root, 'Big Metadata')
+  const bigValid = path.join(bigModels, 'big-index')
+  await fs.mkdir(bigValid, { recursive: true })
+  await fs.writeFile(path.join(bigValid, 'config.json'), JSON.stringify({ model_type: 'qwen3_5', quantization: { bits: 4, group_size: 64 } }))
+  await fs.writeFile(path.join(bigValid, 'tokenizer.json'), JSON.stringify({ model: { type: 'BPE', vocab: {} } }))
+  const bigHeader = Buffer.from(JSON.stringify({ weight: { dtype: 'F32', shape: [2], data_offsets: [0, 8] } }))
+  const bigPrefix = Buffer.alloc(8); bigPrefix.writeBigUInt64LE(BigInt(bigHeader.length))
+  const bigWeight = Buffer.concat([bigPrefix, bigHeader, Buffer.alloc(8, 1)])
+  await fs.writeFile(path.join(bigValid, 'model-00001-of-00001.safetensors'), bigWeight)
+  const pad = 5 * 1024 * 1024
+  const bigIndex = `{"metadata":{"total_size":${bigWeight.length},"pad":"${'x'.repeat(pad)}"},"weight_map":{"a":"model-00001-of-00001.safetensors","b":"model-00001-of-00001.safetensors"}}`
+  await fs.writeFile(path.join(bigValid, 'model.safetensors.index.json'), bigIndex)
+  const bigLib = library({ appHome: path.join(root, 'big-metadata-app') })
+  const bigFound = await bigLib.addFolder(bigModels)
+  const bigModel = bigFound.models.find((model) => model.path === bigValid)!
+  assert.equal(bigModel.complete, true, bigModel.reason ?? '')
+  assert.equal(bigModel.compatible, true)
+  assert.equal(bigModel.loadable, true)
+  assert.equal(bigModel.weightsBytes, bigWeight.length)
+  assert.equal(bigModel.memoryFloorBytes, bigWeight.length + 2 * 1024 ** 3)
+  assert.equal((await bigLib.resolve(bigModel.id)).path, bigValid, 'a checkpoint whose shard index exceeds the old cap resolves')
+
+  // An oversized tokenizer.json alone was never blocked: it is checked through safeFile(), not the JSON cap.
+  const bigTokenizer = path.join(bigModels, 'big-tokenizer')
+  await checkpoint(bigTokenizer)
+  await fs.truncate(path.join(bigTokenizer, 'tokenizer.json'), 5 * 1024 * 1024)
+  const tokenizerFound = await library({ appHome: path.join(root, 'big-tokenizer-app') }).addFolder(bigModels)
+  assert.equal(tokenizerFound.models.find((model) => model.path === bigTokenizer)?.loadable, true, 'an oversized tokenizer alone needs no limit change')
+
+  // A file above the new per-file cap is still rejected; sparse files keep the cost to the stat only.
+  const overLimit = path.join(bigModels, 'over-limit')
+  await checkpoint(overLimit)
+  const oversized = path.join(overLimit, 'model.safetensors.index.json')
+  await fs.writeFile(oversized, '')
+  await fs.truncate(oversized, 65 * 1024 * 1024)
+  const overFound = await library({ appHome: path.join(root, 'over-limit-app') }).addFolder(bigModels)
+  assert.match(overFound.models.find((model) => model.path === overLimit)!.reason!, /metadata is too large/)
+
+  // Malformed JSON in an index is still rejected.
+  const corrupt = path.join(bigModels, 'corrupt-index')
+  await checkpoint(corrupt)
+  await fs.writeFile(path.join(corrupt, 'model.safetensors.index.json'), '{"weight_map":')
+  const corruptFound = await library({ appHome: path.join(root, 'corrupt-index-app') }).addFolder(bigModels)
+  assert.match(corruptFound.models.find((model) => model.path === corrupt)!.reason!, /invalid JSON/)
+
+  // The scan-wide metadata budget stays bounded and reports a visible truncation.
+  const budgeted = library({ appHome: path.join(root, 'budget-app'), limits: { metadataBytes: 3 * 1024 * 1024 } })
+  const budgetFound = await budgeted.addFolder(bigModels)
+  assert.equal(budgetFound.truncated, true)
+  assert.match(budgetFound.warnings.join(' '), /safety limit/)
+
   const validRegistry = await fs.readFile(lib.file, 'utf8')
   await fs.writeFile(lib.file, '{')
   await assert.rejects(() => library().snapshot(), /library is invalid/)

app/electron/agent/engine/library.ts

@@ -7,8 +7,9 @@ import { formatName } from './cli'
 import { isRepoId } from './paths'
 
 const STATE_LIMIT = 2 * 1024 * 1024
-const JSON_LIMIT = 4 * 1024 * 1024
-const DEFAULT_LIMITS = { directories: 512, entries: 12_000, models: 256, depth: 4, metadataBytes: 32 * 1024 * 1024, durationMs: 15_000 }
+// The native engine reads checkpoint metadata well above this cap; the limit only guards the scan's bounded read loop.
+const JSON_LIMIT = 64 * 1024 * 1024
+const DEFAULT_LIMITS = { directories: 512, entries: 12_000, models: 256, depth: 4, metadataBytes: 256 * 1024 * 1024, durationMs: 15_000 }
 const INVALID = 'The local model library is invalid. Preserve its settings and repair them before scanning or loading models.'
 const LOCAL_ID = /^local\/[a-f0-9]{64}$/
 const SHA = /^[a-f0-9]{40}$/