diff --git a/README.md b/README.md index e323862..79a742f 100644 --- a/README.md +++ b/README.md @@ -15,11 +15,12 @@ npm install carbites ## Usage -Carbites supports 3 different strategies: +Carbites supports 4 different strategies: 1. [**Simple**](#simple) (default) - fast but naive, only the first CAR output has a root CID, subsequent CARs have a placeholder "empty" CID. 2. [**Rooted**](#rooted) - like simple, but creates a custom root node to ensure all blocks in a CAR are referenced. 3. [**Treewalk**](#treewalk) - walks the DAG to pack sub-graphs into each CAR file that is output. Every CAR has the same root CID, but contains a different portion of the DAG. +4. [**Treeleaf**](#treeleaf) - the first CAR contains all non-leaf nodes. Subsequent CARs contain only leaf-nodes. ### Simple @@ -116,7 +117,55 @@ Every CAR file has the _same_ root CID but a different portion of the DAG. The D -### CLI +### Treeleaf + +
+ Example + + ```js + import { TreeleafCarSplitter } from 'carbites/treeleaf' + import { CarReader } from '@ipld/car/reader' + import { printTree } from 'imaginary-car-tree-lib' + import fs from 'fs' + + const bigCar = await CarReader.fromIterable(fs.createReadStream('/path/to/big.car')) + const targetSize = 1024 * 1024 * 100 // chunk to ~100MB CARs + const splitter = new TreeleafCarSplitter(bigCar, targetSize) + + const count = 0 + for await (const car of splitter.cars()) { + const reader = await CarReader.fromIterable(car) + const [splitCarRootCid] = await reader.getRoots() + console.log(`## car ${count} – root: ${splitCarRootCid}`) + console.log(printTree(reader)) + count++ + } + /* output: + ## car 0 - root: CID(bafybeidpq6auelplqkozwd3kobvgv4jow72r2yt6ffjzenemqhylmmxngi) + bafybeidpq6auelplqkozwd3kobvgv4jow72r2yt6ffjzenemqhylmmxngi + └─┬ bafybeibw77p4afchrvmo6ep4fpv7j4aehrn2yqs3tv3uuofc6oag36zq2q + ├─┬ bafybeih2w5euyf3sodc6efxtpahgwkata46fupjysi4bbnazb4n2b25rry + │ └─┬ bafybeif5xvhik6thha5ykg3g73jo3mqd5mhb7orzeznnvwquashmwryhai + │ ├── bafkreidkmypp3asq3xiu6mmtcfbzmmhcobpmnaqcyns6535w3u7bz7el64 ❌ missing + │ ├── bafkreid3pefuwyvhsnlrz6xk67f4f6opgqhcrle4bf47kvachwojdgt7re ❌ missing + │ └── bafkreigr3beehu5ebbgoisx4rl2vyaqebmohubauc6avefodb5jauz3tyi ❌ missing + └─┬ bafybeia2i6eqwfqrixh446pavwev37kywowmdpgvx34fbv7asdh3qwpm3y + └── bafkreih2bhak5yv7g4vft5c37j7dw5rqnnsyyuzsifczehhhpm3t655oae ❌ missing + + ## car 1 - root: CID(bafkqaaa) + bafkreidkmypp3asq3xiu6mmtcfbzmmhcobpmnaqcyns6535w3u7bz7el64 + bafkreid3pefuwyvhsnlrz6xk67f4f6opgqhcrle4bf47kvachwojdgt7re + bafkreigr3beehu5ebbgoisx4rl2vyaqebmohubauc6avefodb5jauz3tyi + bafkreih2bhak5yv7g4vft5c37j7dw5rqnnsyyuzsifczehhhpm3t655oae + */ + ``` + + ⚠️ Note: The first CAR output has roots in the header, subsequent CARs have an empty root CID [`bafkqaaa`](https://cid.ipfs.io/#bafkqaaa) as [recommended](https://ipld.io/specs/transport/car/carv1/#number-of-roots). + +
+ + +## CLI Install the CLI tool to use Carbites from the comfort of your terminal: diff --git a/lib/treeleaf/index.js b/lib/treeleaf/index.js new file mode 100644 index 0000000..45cd3a6 --- /dev/null +++ b/lib/treeleaf/index.js @@ -0,0 +1,7 @@ +import { TreeleafCarSplitter } from './splitter.js' +import { TreeleafCarJoiner } from './joiner.js' + +export { + TreeleafCarSplitter, + TreeleafCarJoiner +} diff --git a/lib/treeleaf/joiner.js b/lib/treeleaf/joiner.js new file mode 100644 index 0000000..9da4aa8 --- /dev/null +++ b/lib/treeleaf/joiner.js @@ -0,0 +1,41 @@ +import { CarWriter } from '@ipld/car' + +/** + * @typedef {import('@ipld/car/api').BlockReader & import('@ipld/car/api').RootsReader} ICarReader + */ + +export class TreeleafCarJoiner { + /** + * @param {Iterable} cars + */ + constructor (cars) { + /** @type {ICarReader[]} */ + this._cars = Array.from(cars) + if (!this._cars.length) throw new Error('missing CARs') + } + + async * car () { + const reader = this._cars[0] + const roots = await reader.getRoots() + const { writer, out } = CarWriter.create(roots) + const writeCar = async () => { + try { + for await (const b of reader.blocks()) { + await writer.put(b) + } + for (const reader of this._cars.slice(1)) { + for await (const b of reader.blocks()) { + await writer.put(b) + } + } + } catch (err) { + console.error(err) // TODO: how to forward this on? + } finally { + await writer.close() + } + } + + writeCar() + yield * out + } +} diff --git a/lib/treeleaf/splitter.js b/lib/treeleaf/splitter.js new file mode 100644 index 0000000..ec2bc47 --- /dev/null +++ b/lib/treeleaf/splitter.js @@ -0,0 +1,112 @@ +import { CarReader, CarWriter } from '@ipld/car' +import * as Block from 'multiformats/block' +import { CID } from 'multiformats/cid' +import * as raw from 'multiformats/codecs/raw' +import * as cbor from '@ipld/dag-cbor' +import * as json from '@ipld/dag-json' +import * as pb from '@ipld/dag-pb' + +/** + * @typedef {import('@ipld/car/api').BlockReader & import('@ipld/car/api').RootsReader} ICarReader + * @typedef {import('@ipld/car/api').WriterChannel} WriterChannel + * @typedef {import('multiformats/codecs/interface').BlockDecoder} BlockDecoder + * @typedef {{ decoders?: BlockDecoder[] }} Options + * @typedef {number} Code + */ + +/** + * A work-around for use-cases where the inclusion of a root CID is difficult + * but needing to be safely within the "at least one" recommendation is to use + * an empty CID: \x01\x55\x00\x00 (zero-length "identity" multihash with "raw" + * codec). Since current implementations for this version of the CAR + * specification don't check for the existence of root CIDs (see Root CID block + * existence), this will be safe as far as CAR implementations are concerned. + * However, there is no guarantee that applications that use CAR files will + * correctly consume (ignore) this empty root CID. + * + * https://github.com/ipld/specs/blob/master/block-layer/content-addressable-archives.md#number-of-roots + */ +const empty = CID.parse('bafkqaaa') + +export class TreeleafCarSplitter { + /** + * @param {ICarReader} reader + * @param {number} targetSize + * @param {Options} [options] + */ + constructor (reader, targetSize, options = {}) { + if (typeof targetSize !== 'number' || targetSize <= 0) { + throw new Error('invalid target chunk size') + } + this._reader = reader + this._targetSize = targetSize + const codecs = [pb, raw, cbor, json, ...(options.decoders || [])] + /** @type Map */ + this._decoderMap = new Map(codecs.map(d => [d.code, d])) + } + + async * cars () { + /** @type {(roots?: CID[]) => WriterChannel} */ + const createCarWriter = (roots = [empty]) => { + const car = CarWriter.create(roots) + Object.assign(car.out, { version: 1, getRoots: async () => roots }) + return car + } + const leaves = [] + const originRoots = await this._reader.getRoots() + const tree = createCarWriter(originRoots) + + for await (const block of this._reader.blocks()) { + if (block.cid.code === raw.code) { + leaves.push(block) + continue + } + const codec = this._decoderMap.get(block.cid.code) + if (!codec) throw new Error(`missing decoder for ${block.cid.code}`) + const decoded = Block.createUnsafe({ cid: block.cid, bytes: block.bytes, codec }) + const hasLinks = Array.from(decoded.links()).length > 0 + if (hasLinks) { + tree.writer.put(block) + } else { + leaves.push(block) + } + } + tree.writer.close() + yield tree.out + + // finished tree car, now write multiple leaf cars + + let size = 0 + let car = createCarWriter() + for (const block of leaves) { + if (size >= this._targetSize) { + car.writer.close() + yield car.out + car = createCarWriter() + } + car.writer.put(block) + size += block.bytes.length + } + car.writer.close() + yield car.out + } + + /** + * @param {Blob} blob + * @param {number} targetSize + */ + static async fromBlob (blob, targetSize) { + const buffer = await blob.arrayBuffer() + const reader = await CarReader.fromBytes(new Uint8Array(buffer)) + return new TreeleafCarSplitter(reader, targetSize) + } + + /** + * @param {AsyncIterable} iterable + * @param {number} targetSize + */ + static async fromIterable (iterable, targetSize) { + const reader = await CarReader.fromIterable(iterable) + return new TreeleafCarSplitter(reader, targetSize) + } +} diff --git a/package-lock.json b/package-lock.json index 89f7649..aa459e9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5,11 +5,13 @@ "requires": true, "packages": { "": { + "name": "carbites", "version": "1.0.6", "license": "(Apache-2.0 AND MIT)", "dependencies": { "@ipld/car": "^3.0.1", "@ipld/dag-cbor": "^6.0.3", + "@ipld/dag-json": "^8.0.11", "@ipld/dag-pb": "^2.0.2", "multiformats": "^9.0.4" }, @@ -135,6 +137,15 @@ "multiformats": "^9.0.0" } }, + "node_modules/@ipld/dag-json": { + "version": "8.0.11", + "resolved": "https://registry.npmjs.org/@ipld/dag-json/-/dag-json-8.0.11.tgz", + "integrity": "sha512-Pea7JXeYHTWXRTIhBqBlhw7G53PJ7yta3G/sizGEZyzdeEwhZRr0od5IQ0r2ZxOt1Do+2czddjeEPp+YTxDwCA==", + "dependencies": { + "cborg": "^1.5.4", + "multiformats": "^9.5.4" + } + }, "node_modules/@ipld/dag-pb": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/@ipld/dag-pb/-/dag-pb-2.0.2.tgz", @@ -1010,9 +1021,9 @@ } }, "node_modules/cborg": { - "version": "1.3.4", - "resolved": "https://registry.npmjs.org/cborg/-/cborg-1.3.4.tgz", - "integrity": "sha512-w/sCTLy2k4AI6XbDMiaPsGc07bBAcX/It3VYQBwKJ5fvAAyUOXi5nC1wjW0OXIfY5ROkOGJCAUOR5AW1ykkiCQ==", + "version": "1.9.5", + "resolved": "https://registry.npmjs.org/cborg/-/cborg-1.9.5.tgz", + "integrity": "sha512-fLBv8wmqtlXqy1Yu+pHzevAIkW6k2K0ZtMujNzWphLsA34vzzg9BHn+5GmZqOJkSA9V7EMKsWrf6K976c1QMjQ==", "bin": { "cborg": "cli.js" } @@ -3738,9 +3749,9 @@ "dev": true }, "node_modules/multiformats": { - "version": "9.0.4", - "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-9.0.4.tgz", - "integrity": "sha512-QWGCSOO4aSEZO4fUCZC1tkvd6rtPTzkAT5AaEshWWJIwd544YTBrrJFmICTp/hEUiOhOyDBJuwnjsFQCTIKLeQ==" + "version": "9.9.0", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-9.9.0.tgz", + "integrity": "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg==" }, "node_modules/natural-compare": { "version": "1.4.0", @@ -5778,6 +5789,15 @@ "multiformats": "^9.0.0" } }, + "@ipld/dag-json": { + "version": "8.0.11", + "resolved": "https://registry.npmjs.org/@ipld/dag-json/-/dag-json-8.0.11.tgz", + "integrity": "sha512-Pea7JXeYHTWXRTIhBqBlhw7G53PJ7yta3G/sizGEZyzdeEwhZRr0od5IQ0r2ZxOt1Do+2czddjeEPp+YTxDwCA==", + "requires": { + "cborg": "^1.5.4", + "multiformats": "^9.5.4" + } + }, "@ipld/dag-pb": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/@ipld/dag-pb/-/dag-pb-2.0.2.tgz", @@ -6491,9 +6511,9 @@ "dev": true }, "cborg": { - "version": "1.3.4", - "resolved": "https://registry.npmjs.org/cborg/-/cborg-1.3.4.tgz", - "integrity": "sha512-w/sCTLy2k4AI6XbDMiaPsGc07bBAcX/It3VYQBwKJ5fvAAyUOXi5nC1wjW0OXIfY5ROkOGJCAUOR5AW1ykkiCQ==" + "version": "1.9.5", + "resolved": "https://registry.npmjs.org/cborg/-/cborg-1.9.5.tgz", + "integrity": "sha512-fLBv8wmqtlXqy1Yu+pHzevAIkW6k2K0ZtMujNzWphLsA34vzzg9BHn+5GmZqOJkSA9V7EMKsWrf6K976c1QMjQ==" }, "chalk": { "version": "2.4.2", @@ -8676,9 +8696,9 @@ "dev": true }, "multiformats": { - "version": "9.0.4", - "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-9.0.4.tgz", - "integrity": "sha512-QWGCSOO4aSEZO4fUCZC1tkvd6rtPTzkAT5AaEshWWJIwd544YTBrrJFmICTp/hEUiOhOyDBJuwnjsFQCTIKLeQ==" + "version": "9.9.0", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-9.9.0.tgz", + "integrity": "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg==" }, "natural-compare": { "version": "1.4.0", diff --git a/package.json b/package.json index 1f8a02b..c637960 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ "dependencies": { "@ipld/car": "^3.0.1", "@ipld/dag-cbor": "^6.0.3", + "@ipld/dag-json": "^8.0.11", "@ipld/dag-pb": "^2.0.2", "multiformats": "^9.0.4" }, diff --git a/test/treeleaf-splitter.spec.js b/test/treeleaf-splitter.spec.js new file mode 100644 index 0000000..cf1270d --- /dev/null +++ b/test/treeleaf-splitter.spec.js @@ -0,0 +1,135 @@ +import test from 'ava' +import { CarReader } from '@ipld/car' +import * as Block from 'multiformats/block' +import { CID } from 'multiformats/cid' +import * as raw from 'multiformats/codecs/raw' +import * as cbor from '@ipld/dag-cbor' +import * as json from '@ipld/dag-json' +import * as pb from '@ipld/dag-pb' +import { Blob } from '@web-std/blob' +import { collect, collectBytes, randomCar, toAsyncIterable } from './_helpers.js' +import { TreeleafCarSplitter } from '../lib/treeleaf/splitter.js' + +const decoders = new Map([raw, cbor, json, pb].map(d => [d.code, d])) + +const empty = CID.parse('bafkqaaa') + +test('split in ~two', async t => { + const targetSize = 500 + const bytes = await collectBytes(await randomCar(1000)) + const blocks = await collect((await CarReader.fromBytes(bytes)).blocks()) + const splitter = new TreeleafCarSplitter(await CarReader.fromBytes(bytes), targetSize) + const cars = [] + for await (const car of splitter.cars()) { + cars.push(car) + } + t.true(cars.length >= 2) + const chunkedBlocks = [] + for (const c of cars) { + const bs = await collect((await CarReader.fromIterable(c)).blocks()) + chunkedBlocks.push(...bs) + } + t.is(blocks.length, chunkedBlocks.length) +}) + +test('target size bigger than file size', async t => { + const bytes = await collectBytes(await randomCar(1000)) + const splitter = new TreeleafCarSplitter(await CarReader.fromBytes(bytes), Infinity) + const cars = [] + for await (const car of splitter.cars()) { + cars.push(car) + } + t.is(cars.length, 2, 'treeleaf always splits tree from leaf nodes') + + const treeCar = await CarReader.fromIterable(cars[0]) + for await (const { cid, bytes } of treeCar.blocks()) { + const codec = decoders.get(cid.code) + const decoded = Block.createUnsafe({ cid, bytes, codec }) + t.true(Array.from(decoded.links()).length > 0, 'All blocks in tree car should have links') + } + + const leafCar = await CarReader.fromIterable(cars[1]) + for await (const { cid, bytes } of leafCar.blocks()) { + const codec = decoders.get(cid.code) + const decoded = Block.createUnsafe({ cid, bytes, codec }) + t.is(Array.from(decoded.links()).length, 0, 'All blocks in leaf car should have no links') + } +}) + +test('roots in first CAR, empty CID root in other CARs', async t => { + const targetSize = 100 + const bytes = await collectBytes(await randomCar(1000)) + const splitter = new TreeleafCarSplitter(await CarReader.fromBytes(bytes), targetSize) + let i = 0 + for await (const car of splitter.cars()) { + const reader = await CarReader.fromIterable(car) + const roots = await reader.getRoots() + if (i === 0) { + t.true(roots.length > 0) + } else { + t.is(roots.length, 1) + t.true(empty.equals(roots[0])) + } + i++ + } +}) + +test('bad target size', t => { + t.throws(() => new TreeleafCarSplitter(null, -1)) +}) + +test('split CARs are RootsReader', async t => { + const targetSize = 500 + const bytes = await collectBytes(await randomCar(1000)) + const reader = await CarReader.fromBytes(bytes) + const splitter = new TreeleafCarSplitter(reader, targetSize) + const splitRoots = [] + for await (const car of splitter.cars()) { + const roots = await car.getRoots() + splitRoots.push(...roots) + } + const originRoots = await reader.getRoots() + for (const [i, r] of originRoots.entries()) { + t.true(r.equals(splitRoots[i])) + } + // non-origin roots should be empty CID + splitRoots.slice(originRoots.length).forEach(r => { + t.true(r.equals(empty)) + }) +}) + +test('fromBlob', async t => { + const targetSize = 500 + const bytes = await collectBytes(await randomCar(1000)) + const blocks = await collect((await CarReader.fromBytes(bytes)).blocks()) + const splitter = await TreeleafCarSplitter.fromBlob(new Blob([bytes]), targetSize) + const cars = [] + for await (const car of splitter.cars()) { + cars.push(car) + } + t.true(cars.length >= 2) + const chunkedBlocks = [] + for (const c of cars) { + const bs = await collect((await CarReader.fromIterable(c)).blocks()) + chunkedBlocks.push(...bs) + } + t.is(blocks.length, chunkedBlocks.length) +}) + +test('fromIterable', async t => { + const targetSize = 500 + const bytes = await collectBytes(await randomCar(1000)) + const blocks = await collect((await CarReader.fromBytes(bytes)).blocks()) + const splitter = await TreeleafCarSplitter.fromIterable(toAsyncIterable([bytes]), targetSize) + const cars = [] + for await (const car of splitter.cars()) { + cars.push(car) + } + t.true(cars.length >= 2) + const chunkedBlocks = [] + for (const c of cars) { + const bs = await collect((await CarReader.fromIterable(c)).blocks()) + chunkedBlocks.push(...bs) + } + t.is(blocks.length, chunkedBlocks.length) +})