From 30a75073d6c143904d5478733cf26291fb5a438f Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Mon, 21 Feb 2022 17:15:09 +0000 Subject: [PATCH] feat: add origin root filler to simpole strategy --- README.md | 10 ++- lib/rooted/joiner.js | 8 +-- lib/rooted/splitter.js | 15 ++++- lib/simple/joiner.js | 8 +-- lib/simple/splitter.js | 21 ++++--- lib/treewalk/joiner.js | 10 +-- lib/treewalk/splitter.js | 3 +- package-lock.json | 117 ++++++++++++++--------------------- package.json | 8 +-- test/splitter-simple.spec.js | 14 +++++ 10 files changed, 106 insertions(+), 108 deletions(-) diff --git a/README.md b/README.md index e323862..17df6ba 100644 --- a/README.md +++ b/README.md @@ -135,7 +135,7 @@ carbites join big-0.car big-1.car ... ## API * [`class CarSplitter`](#class-carsplitter) - * [`constructor(car: AsyncIterable, targetSize: number)`](#constructorcar-asynciterableuint8array-targetsize-number) + * [`constructor(car: AsyncIterable, targetSize: number, options?: Object)`](#constructorcar-asynciterableuint8array-targetsize-number-options-object) * [`cars(): AsyncGenerator & RootsReader>`](#cars-asyncgeneratorasynciterableuint8array--rootsreader) * [`static async fromBlob(blob: Blob, targetSize: number): CarSplitter`](#static-async-fromblobblob-blob-targetsize-number-carsplitter) * [`static async fromIterable(iterable: AsyncIterable, targetSize: number): CarSplitter`](#static-async-fromiterableiterable-asynciterableuint8array-targetsize-number-carsplitter) @@ -165,10 +165,16 @@ import { CarSplitter } from 'carbites' Note: This is an alias of `SimpleCarSplitter` - the default strategy for splitting CARs. -#### `constructor(car: CarReader, targetSize: number)` +#### `constructor(car: CarReader, targetSize: number, options?: Object)` Create a new `CarSplitter` for the passed CAR file, aiming to generate CARs of around `targetSize` bytes in size. +The following options are available: + +* `fillRoots: 'empty'|'origin'` - How to populate header roots. Default 'empty'. + * Specify 'empty' to populate split CAR header roots (after the first) with the empty CID "bafkqaaa". + * Specify 'origin' to populate all split CAR header roots with the roots found in the origin CAR file header. + #### `cars(): AsyncGenerator & RootsReader>` Split the CAR file and create multiple smaller CAR files. Returns an `AsyncGenerator` that yields the split CAR files (of type `AsyncIterable`). diff --git a/lib/rooted/joiner.js b/lib/rooted/joiner.js index 276c7e9..f138997 100644 --- a/lib/rooted/joiner.js +++ b/lib/rooted/joiner.js @@ -4,10 +4,6 @@ import { Block } from 'multiformats/block' import { isRootNode } from './root-node.js' import { SimpleCarJoiner } from '../simple/joiner.js' -/** - * @typedef {import('@ipld/car/api').BlockReader & import('@ipld/car/api').RootsReader} ICarReader - */ - export class RootedCarJoiner extends SimpleCarJoiner { async * car () { const reader = this._cars[0] @@ -38,7 +34,7 @@ export class RootedCarJoiner extends SimpleCarJoiner { } /** - * @param {ICarReader} reader + * @param {import('@ipld/car/api').CarReader} reader * @returns {Promise>} */ async function getRootedBlock (reader) { @@ -54,7 +50,7 @@ async function getRootedBlock (reader) { /** * @param {import('multiformats').CID} cid - * @param {ICarReader} reader + * @param {import('@ipld/car/api').CarReader} reader */ async function * filterBlock (cid, reader) { for await (const b of reader.blocks()) { diff --git a/lib/rooted/splitter.js b/lib/rooted/splitter.js index 37873f0..582b805 100644 --- a/lib/rooted/splitter.js +++ b/lib/rooted/splitter.js @@ -1,8 +1,19 @@ import { CarReader, CarWriter } from '@ipld/car' import { mkRootNode } from './root-node.js' -import { SimpleCarSplitter } from '../simple/splitter.js' -export class RootedCarSplitter extends SimpleCarSplitter { +export class RootedCarSplitter { + /** + * @param {import('@ipld/car/api').CarReader} reader + * @param {number} targetSize + */ + constructor (reader, targetSize) { + if (typeof targetSize !== 'number' || targetSize <= 0) { + throw new Error('invalid target chunk size') + } + this._reader = reader + this._targetSize = targetSize + } + async * cars () { const allBlocks = this._reader.blocks()[Symbol.asyncIterator]() const roots = await this._reader.getRoots() diff --git a/lib/simple/joiner.js b/lib/simple/joiner.js index e4f5cbd..d138bd8 100644 --- a/lib/simple/joiner.js +++ b/lib/simple/joiner.js @@ -1,15 +1,11 @@ import { CarWriter } from '@ipld/car' -/** - * @typedef {import('@ipld/car/api').BlockReader & import('@ipld/car/api').RootsReader} ICarReader - */ - export class SimpleCarJoiner { /** - * @param {Iterable} cars + * @param {Iterable} cars */ constructor (cars) { - /** @type {ICarReader[]} */ + /** @type {import('@ipld/car/api').CarReader[]} */ this._cars = Array.from(cars) if (!this._cars.length) throw new Error('missing CARs') } diff --git a/lib/simple/splitter.js b/lib/simple/splitter.js index b529911..934cd39 100644 --- a/lib/simple/splitter.js +++ b/lib/simple/splitter.js @@ -1,10 +1,6 @@ import { CarReader, CarWriter } from '@ipld/car' import { CID } from 'multiformats/cid' -/** - * @typedef {import('@ipld/car/api').BlockReader & import('@ipld/car/api').RootsReader} ICarReader - */ - /** * A work-around for use-cases where the inclusion of a root CID is difficult * but needing to be safely within the "at least one" recommendation is to use @@ -21,15 +17,23 @@ const empty = CID.parse('bafkqaaa') export class SimpleCarSplitter { /** - * @param {ICarReader} reader + * @param {import('@ipld/car/api').CarReader} reader * @param {number} targetSize + * @param {Object} [options] + * @param {'empty'|'origin'} [options.fillRoots] How to populate header roots. Default 'empty'. + * Specify 'empty' to populate split CARs (after the first) with the empty CID "bafkqaaa". + * Specify 'origin' to populate all split CAR header roots with the roots found in the origin CAR file header. */ - constructor (reader, targetSize) { + constructor (reader, targetSize, options = {}) { if (typeof targetSize !== 'number' || targetSize <= 0) { throw new Error('invalid target chunk size') } this._reader = reader this._targetSize = targetSize + this._fillRoots = options.fillRoots || 'empty' + if (!['empty', 'origin'].includes(this._fillRoots)) { + throw new Error('invalid root filler') + } } async * cars () { @@ -50,7 +54,10 @@ export class SimpleCarSplitter { blocks.push(block) size += block.bytes.length } - const roots = first ? originRoots : [empty] + let roots = first ? originRoots : [empty] // default is empty roots + if (this._fillRoots === 'origin') { + roots = originRoots + } const { writer, out } = CarWriter.create(roots) Object.assign(out, { version: 1, getRoots: async () => roots }) blocks.forEach(b => writer.put(b)) diff --git a/lib/treewalk/joiner.js b/lib/treewalk/joiner.js index f80e621..142787d 100644 --- a/lib/treewalk/joiner.js +++ b/lib/treewalk/joiner.js @@ -1,15 +1,11 @@ import { CarWriter } from '@ipld/car' -/** - * @typedef {import('@ipld/car/api').BlockReader & import('@ipld/car/api').RootsReader} ICarReader - */ - export class TreewalkCarJoiner { /** - * @param {Iterable} cars + * @param {Iterable} cars */ constructor (cars) { - /** @type {ICarReader[]} */ + /** @type {import('@ipld/car/api').CarReader[]} */ this._cars = Array.from(cars) if (!this._cars.length) throw new Error('missing CARs') } @@ -20,7 +16,7 @@ export class TreewalkCarJoiner { const { writer, out } = CarWriter.create(roots) const writeCar = async () => { const written = new Set() - const writeBlocks = async (/** @type {ICarReader} */ reader) => { + const writeBlocks = async (/** @type {import('@ipld/car/api').CarReader} */ reader) => { for await (const b of reader.blocks()) { if (written.has(b.cid.toString())) continue await writer.put(b) diff --git a/lib/treewalk/splitter.js b/lib/treewalk/splitter.js index 1b68d5c..4563a6e 100644 --- a/lib/treewalk/splitter.js +++ b/lib/treewalk/splitter.js @@ -8,12 +8,11 @@ import * as pb from '@ipld/dag-pb' * @typedef {{ decoders?: import('multiformats/codecs/interface').BlockDecoder[] }} Options * @typedef {import('@ipld/car/api').WriterChannel & { size: number }} WriterChannel * @typedef {import('multiformats/cid').CID} CID - * @typedef {import('@ipld/car/api').BlockReader & import('@ipld/car/api').RootsReader} ICarReader */ export class TreewalkCarSplitter { /** - * @param {ICarReader} reader + * @param {import('@ipld/car/api').CarReader} reader * @param {number} targetSize * @param {Options} [options] */ diff --git a/package-lock.json b/package-lock.json index 89f7649..52bda37 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5,13 +5,14 @@ "requires": true, "packages": { "": { + "name": "carbites", "version": "1.0.6", "license": "(Apache-2.0 AND MIT)", "dependencies": { - "@ipld/car": "^3.0.1", - "@ipld/dag-cbor": "^6.0.3", - "@ipld/dag-pb": "^2.0.2", - "multiformats": "^9.0.4" + "@ipld/car": "^3.2.3", + "@ipld/dag-cbor": "^7.0.0", + "@ipld/dag-pb": "^2.1.15", + "multiformats": "^9.6.4" }, "devDependencies": { "@web-std/blob": "^2.1.0", @@ -116,31 +117,30 @@ } }, "node_modules/@ipld/car": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@ipld/car/-/car-3.0.3.tgz", - "integrity": "sha512-c5/LmjjWFDicqcRvJNfhC394wt9ZUKDzzdzMZdrXI1xCBCCtimlKhVaIlxh2+9lFvV50alPBqGK+jhnjXPbmEQ==", + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/@ipld/car/-/car-3.2.3.tgz", + "integrity": "sha512-pXE5mFJlXzJVaBwqAJKGlKqMmxq8H2SLEWBJgkeBDPBIN8ZbscPc3I9itkSQSlS/s6Fgx35Ri3LDTDtodQjCCQ==", "dependencies": { - "@ipld/dag-cbor": "^6.0.0", - "@types/varint": "^6.0.0", - "multiformats": "^9.0.0", + "@ipld/dag-cbor": "^7.0.0", + "multiformats": "^9.5.4", "varint": "^6.0.0" } }, "node_modules/@ipld/dag-cbor": { - "version": "6.0.4", - "resolved": "https://registry.npmjs.org/@ipld/dag-cbor/-/dag-cbor-6.0.4.tgz", - "integrity": "sha512-rfvNBGBGb1cTEQVaTd2V3i9nRuTyhKbgdvss8vdAXJt3ZM1istxT5n1jUeVuUaMESObmS2JBHTGZ5L8CCIEtaQ==", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/@ipld/dag-cbor/-/dag-cbor-7.0.0.tgz", + "integrity": "sha512-us/dagGvfQ+acO8uyAfozUQ21xxvI6ZrCWwfbOuk+o+cSpCIKY30lUYRuN3kzWLvTJHvbuCVPVEH38ynM1ZBgw==", "dependencies": { - "cborg": "^1.2.1", - "multiformats": "^9.0.0" + "cborg": "^1.6.0", + "multiformats": "^9.5.4" } }, "node_modules/@ipld/dag-pb": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/@ipld/dag-pb/-/dag-pb-2.0.2.tgz", - "integrity": "sha512-m0XOtq2zx+F6FqPCrBh4tjTxyAzRE8IIjAPZsfXoXcqOC8HhOiH3nrvk3sXgLuU3Ttxkitkn5ntS4r8LX0ZDVA==", + "version": "2.1.15", + "resolved": "https://registry.npmjs.org/@ipld/dag-pb/-/dag-pb-2.1.15.tgz", + "integrity": "sha512-qkoUIiuQDx2ZN+YmYFdSNNHRt15p1XTYbqsseb8DgA0ACcqCUurbiNVd0jt5GuiBm76t2mOV2cZsNu6rykRFBQ==", "dependencies": { - "multiformats": "^9.0.0" + "multiformats": "^9.5.4" } }, "node_modules/@istanbuljs/schema": { @@ -229,25 +229,12 @@ "integrity": "sha1-7ihweulOEdK4J7y+UnC86n8+ce4=", "dev": true }, - "node_modules/@types/node": { - "version": "15.12.2", - "resolved": "https://registry.npmjs.org/@types/node/-/node-15.12.2.tgz", - "integrity": "sha512-zjQ69G564OCIWIOHSXyQEEDpdpGl+G348RAKY0XXy9Z5kU9Vzv1GMNnkar/ZJ8dzXB3COzD9Mo9NtRZ4xfgUww==" - }, "node_modules/@types/normalize-package-data": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.0.tgz", "integrity": "sha512-f5j5b/Gf71L+dbqxIpQ4Z2WlmI/mPJ0fOkGGmFgtb6sAu97EPczzbS3/tJKxmcYDj55OX6ssqwDAWOHIYDRDGA==", "dev": true }, - "node_modules/@types/varint": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/@types/varint/-/varint-6.0.0.tgz", - "integrity": "sha512-2jBazyxGl4644tvu3VAez8UA/AtrcEetT9HOeAbqZ/vAcRVL/ZDFQjSS7rkWusU5cyONQVUz+nwwrNZdMva4ow==", - "dependencies": { - "@types/node": "*" - } - }, "node_modules/@web-std/blob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/@web-std/blob/-/blob-2.1.1.tgz", @@ -1010,9 +997,9 @@ } }, "node_modules/cborg": { - "version": "1.3.4", - "resolved": "https://registry.npmjs.org/cborg/-/cborg-1.3.4.tgz", - "integrity": "sha512-w/sCTLy2k4AI6XbDMiaPsGc07bBAcX/It3VYQBwKJ5fvAAyUOXi5nC1wjW0OXIfY5ROkOGJCAUOR5AW1ykkiCQ==", + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/cborg/-/cborg-1.7.0.tgz", + "integrity": "sha512-BVounZzg0940FGbK+vgQCjlXBoczu8t4s2GLsi2N9QRRzerMBbisQQbDszHhnBOtSpi8+umU++xpuIDYIs6cIg==", "bin": { "cborg": "cli.js" } @@ -3738,9 +3725,9 @@ "dev": true }, "node_modules/multiformats": { - "version": "9.0.4", - "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-9.0.4.tgz", - "integrity": "sha512-QWGCSOO4aSEZO4fUCZC1tkvd6rtPTzkAT5AaEshWWJIwd544YTBrrJFmICTp/hEUiOhOyDBJuwnjsFQCTIKLeQ==" + "version": "9.6.4", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-9.6.4.tgz", + "integrity": "sha512-fCCB6XMrr6CqJiHNjfFNGT0v//dxOBMrOMqUIzpPc/mmITweLEyhvMpY9bF+jZ9z3vaMAau5E8B68DW77QMXkg==" }, "node_modules/natural-compare": { "version": "1.4.0", @@ -5759,31 +5746,30 @@ } }, "@ipld/car": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@ipld/car/-/car-3.0.3.tgz", - "integrity": "sha512-c5/LmjjWFDicqcRvJNfhC394wt9ZUKDzzdzMZdrXI1xCBCCtimlKhVaIlxh2+9lFvV50alPBqGK+jhnjXPbmEQ==", + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/@ipld/car/-/car-3.2.3.tgz", + "integrity": "sha512-pXE5mFJlXzJVaBwqAJKGlKqMmxq8H2SLEWBJgkeBDPBIN8ZbscPc3I9itkSQSlS/s6Fgx35Ri3LDTDtodQjCCQ==", "requires": { - "@ipld/dag-cbor": "^6.0.0", - "@types/varint": "^6.0.0", - "multiformats": "^9.0.0", + "@ipld/dag-cbor": "^7.0.0", + "multiformats": "^9.5.4", "varint": "^6.0.0" } }, "@ipld/dag-cbor": { - "version": "6.0.4", - "resolved": "https://registry.npmjs.org/@ipld/dag-cbor/-/dag-cbor-6.0.4.tgz", - "integrity": "sha512-rfvNBGBGb1cTEQVaTd2V3i9nRuTyhKbgdvss8vdAXJt3ZM1istxT5n1jUeVuUaMESObmS2JBHTGZ5L8CCIEtaQ==", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/@ipld/dag-cbor/-/dag-cbor-7.0.0.tgz", + "integrity": "sha512-us/dagGvfQ+acO8uyAfozUQ21xxvI6ZrCWwfbOuk+o+cSpCIKY30lUYRuN3kzWLvTJHvbuCVPVEH38ynM1ZBgw==", "requires": { - "cborg": "^1.2.1", - "multiformats": "^9.0.0" + "cborg": "^1.6.0", + "multiformats": "^9.5.4" } }, "@ipld/dag-pb": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/@ipld/dag-pb/-/dag-pb-2.0.2.tgz", - "integrity": "sha512-m0XOtq2zx+F6FqPCrBh4tjTxyAzRE8IIjAPZsfXoXcqOC8HhOiH3nrvk3sXgLuU3Ttxkitkn5ntS4r8LX0ZDVA==", + "version": "2.1.15", + "resolved": "https://registry.npmjs.org/@ipld/dag-pb/-/dag-pb-2.1.15.tgz", + "integrity": "sha512-qkoUIiuQDx2ZN+YmYFdSNNHRt15p1XTYbqsseb8DgA0ACcqCUurbiNVd0jt5GuiBm76t2mOV2cZsNu6rykRFBQ==", "requires": { - "multiformats": "^9.0.0" + "multiformats": "^9.5.4" } }, "@istanbuljs/schema": { @@ -5851,25 +5837,12 @@ "integrity": "sha1-7ihweulOEdK4J7y+UnC86n8+ce4=", "dev": true }, - "@types/node": { - "version": "15.12.2", - "resolved": "https://registry.npmjs.org/@types/node/-/node-15.12.2.tgz", - "integrity": "sha512-zjQ69G564OCIWIOHSXyQEEDpdpGl+G348RAKY0XXy9Z5kU9Vzv1GMNnkar/ZJ8dzXB3COzD9Mo9NtRZ4xfgUww==" - }, "@types/normalize-package-data": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.0.tgz", "integrity": "sha512-f5j5b/Gf71L+dbqxIpQ4Z2WlmI/mPJ0fOkGGmFgtb6sAu97EPczzbS3/tJKxmcYDj55OX6ssqwDAWOHIYDRDGA==", "dev": true }, - "@types/varint": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/@types/varint/-/varint-6.0.0.tgz", - "integrity": "sha512-2jBazyxGl4644tvu3VAez8UA/AtrcEetT9HOeAbqZ/vAcRVL/ZDFQjSS7rkWusU5cyONQVUz+nwwrNZdMva4ow==", - "requires": { - "@types/node": "*" - } - }, "@web-std/blob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/@web-std/blob/-/blob-2.1.1.tgz", @@ -6491,9 +6464,9 @@ "dev": true }, "cborg": { - "version": "1.3.4", - "resolved": "https://registry.npmjs.org/cborg/-/cborg-1.3.4.tgz", - "integrity": "sha512-w/sCTLy2k4AI6XbDMiaPsGc07bBAcX/It3VYQBwKJ5fvAAyUOXi5nC1wjW0OXIfY5ROkOGJCAUOR5AW1ykkiCQ==" + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/cborg/-/cborg-1.7.0.tgz", + "integrity": "sha512-BVounZzg0940FGbK+vgQCjlXBoczu8t4s2GLsi2N9QRRzerMBbisQQbDszHhnBOtSpi8+umU++xpuIDYIs6cIg==" }, "chalk": { "version": "2.4.2", @@ -8676,9 +8649,9 @@ "dev": true }, "multiformats": { - "version": "9.0.4", - "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-9.0.4.tgz", - "integrity": "sha512-QWGCSOO4aSEZO4fUCZC1tkvd6rtPTzkAT5AaEshWWJIwd544YTBrrJFmICTp/hEUiOhOyDBJuwnjsFQCTIKLeQ==" + "version": "9.6.4", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-9.6.4.tgz", + "integrity": "sha512-fCCB6XMrr6CqJiHNjfFNGT0v//dxOBMrOMqUIzpPc/mmITweLEyhvMpY9bF+jZ9z3vaMAau5E8B68DW77QMXkg==" }, "natural-compare": { "version": "1.4.0", diff --git a/package.json b/package.json index 1f8a02b..bc8aa14 100644 --- a/package.json +++ b/package.json @@ -19,10 +19,10 @@ "author": "Alan Shaw", "license": "(Apache-2.0 AND MIT)", "dependencies": { - "@ipld/car": "^3.0.1", - "@ipld/dag-cbor": "^6.0.3", - "@ipld/dag-pb": "^2.0.2", - "multiformats": "^9.0.4" + "@ipld/car": "^3.2.3", + "@ipld/dag-cbor": "^7.0.0", + "@ipld/dag-pb": "^2.1.15", + "multiformats": "^9.6.4" }, "devDependencies": { "@web-std/blob": "^2.1.0", diff --git a/test/splitter-simple.spec.js b/test/splitter-simple.spec.js index 5737100..107dac0 100644 --- a/test/splitter-simple.spec.js +++ b/test/splitter-simple.spec.js @@ -114,3 +114,17 @@ test('fromIterable', async t => { } t.is(blocks.length, chunkedBlocks.length) }) + +test('split with origin roots filler', async t => { + const targetSize = 100 + const bytes = await collectBytes(await randomCar(1000)) + const reader = await CarReader.fromBytes(bytes) + const originRoots = await reader.getRoots() + t.is(originRoots.length, 1) + const splitter = new SimpleCarSplitter(reader, targetSize, { fillRoots: 'origin' }) + for await (const car of splitter.cars()) { + const reader = await CarReader.fromIterable(car) + const roots = await reader.getRoots() + t.true(originRoots[0].equals(roots[0])) + } +})