feat: add origin root filler to simple strategy

README.md

@@ -135,7 +135,7 @@ carbites join big-0.car big-1.car ...
 ## API
 
 * [`class CarSplitter`](#class-carsplitter)
-    * [`constructor(car: AsyncIterable<Uint8Array>, targetSize: number)`](#constructorcar-asynciterableuint8array-targetsize-number)
+    * [`constructor(car: AsyncIterable<Uint8Array>, targetSize: number, options?: Object)`](#constructorcar-asynciterableuint8array-targetsize-number-options-object)
     * [`cars(): AsyncGenerator<AsyncIterable<Uint8Array> & RootsReader>`](#cars-asyncgeneratorasynciterableuint8array--rootsreader)
     * [`static async fromBlob(blob: Blob, targetSize: number): CarSplitter`](#static-async-fromblobblob-blob-targetsize-number-carsplitter)
     * [`static async fromIterable(iterable: AsyncIterable<Uint8Array>, targetSize: number): CarSplitter`](#static-async-fromiterableiterable-asynciterableuint8array-targetsize-number-carsplitter)
@@ -165,10 +165,16 @@ import { CarSplitter } from 'carbites'
 
 Note: This is an alias of `SimpleCarSplitter` - the default strategy for splitting CARs.
 
-#### `constructor(car: CarReader, targetSize: number)`
+#### `constructor(car: CarReader, targetSize: number, options?: Object)`
 
 Create a new `CarSplitter` for the passed CAR file, aiming to generate CARs of around `targetSize` bytes in size.
 
+The following options are available:
+
+* `fillRoots: 'empty'|'origin'` - How to populate header roots. Default 'empty'.
+    * Specify 'empty' to populate split CAR header roots (after the first) with the empty CID "bafkqaaa".
+    * Specify 'origin' to populate all split CAR header roots with the roots found in the origin CAR file header.
+
 #### `cars(): AsyncGenerator<AsyncIterable<Uint8Array> & RootsReader>`
 
 Split the CAR file and create multiple smaller CAR files. Returns an `AsyncGenerator` that yields the split CAR files (of type `AsyncIterable<Uint8Array>`).

lib/rooted/joiner.js

@@ -4,10 +4,6 @@ import { Block } from 'multiformats/block'
 import { isRootNode } from './root-node.js'
 import { SimpleCarJoiner } from '../simple/joiner.js'
 
-/**
- * @typedef {import('@ipld/car/api').BlockReader & import('@ipld/car/api').RootsReader} ICarReader
- */
-
 export class RootedCarJoiner extends SimpleCarJoiner {
   async * car () {
     const reader = this._cars[0]
@@ -38,7 +34,7 @@ export class RootedCarJoiner extends SimpleCarJoiner {
 }
 
 /**
- * @param {ICarReader} reader
+ * @param {import('@ipld/car/api').CarReader} reader
  * @returns {Promise<import('multiformats/block').Block<import('./root-node').RootNode>>}
  */
 async function getRootedBlock (reader) {
@@ -54,7 +50,7 @@ async function getRootedBlock (reader) {
 
 /**
  * @param {import('multiformats').CID} cid
- * @param {ICarReader} reader
+ * @param {import('@ipld/car/api').CarReader} reader
  */
 async function * filterBlock (cid, reader) {
   for await (const b of reader.blocks()) {

lib/rooted/splitter.js

@@ -1,8 +1,19 @@
 import { CarReader, CarWriter } from '@ipld/car'
 import { mkRootNode } from './root-node.js'
-import { SimpleCarSplitter } from '../simple/splitter.js'
 
-export class RootedCarSplitter extends SimpleCarSplitter {
+export class RootedCarSplitter {
+  /**
+   * @param {import('@ipld/car/api').CarReader} reader
+   * @param {number} targetSize
+   */
+  constructor (reader, targetSize) {
+    if (typeof targetSize !== 'number' || targetSize <= 0) {
+      throw new Error('invalid target chunk size')
+    }
+    this._reader = reader
+    this._targetSize = targetSize
+  }
+
   async * cars () {
     const allBlocks = this._reader.blocks()[Symbol.asyncIterator]()
     const roots = await this._reader.getRoots()

lib/simple/joiner.js

@@ -1,15 +1,11 @@
 import { CarWriter } from '@ipld/car'
 
-/**
- * @typedef {import('@ipld/car/api').BlockReader & import('@ipld/car/api').RootsReader} ICarReader
- */
-
 export class SimpleCarJoiner {
   /**
-   * @param {Iterable<ICarReader>} cars
+   * @param {Iterable<import('@ipld/car/api').CarReader>} cars
    */
   constructor (cars) {
-    /** @type {ICarReader[]} */
+    /** @type {import('@ipld/car/api').CarReader[]} */
     this._cars = Array.from(cars)
     if (!this._cars.length) throw new Error('missing CARs')
   }

lib/simple/splitter.js

@@ -1,10 +1,6 @@
 import { CarReader, CarWriter } from '@ipld/car'
 import { CID } from 'multiformats/cid'
 
-/**
- * @typedef {import('@ipld/car/api').BlockReader & import('@ipld/car/api').RootsReader} ICarReader
- */
-
 /**
  * A work-around for use-cases where the inclusion of a root CID is difficult
  * but needing to be safely within the "at least one" recommendation is to use
@@ -21,15 +17,23 @@ const empty = CID.parse('bafkqaaa')
 
 export class SimpleCarSplitter {
   /**
-   * @param {ICarReader} reader
+   * @param {import('@ipld/car/api').CarReader} reader
    * @param {number} targetSize
+   * @param {Object} [options]
+   * @param {'empty'|'origin'} [options.fillRoots] How to populate header roots. Default 'empty'.
+   * Specify 'empty' to populate split CARs (after the first) with the empty CID "bafkqaaa".
+   * Specify 'origin' to populate all split CAR header roots with the roots found in the origin CAR file header.
    */
-  constructor (reader, targetSize) {
+  constructor (reader, targetSize, options = {}) {
     if (typeof targetSize !== 'number' || targetSize <= 0) {
       throw new Error('invalid target chunk size')
     }
     this._reader = reader
     this._targetSize = targetSize
+    this._fillRoots = options.fillRoots || 'empty'
+    if (!['empty', 'origin'].includes(this._fillRoots)) {
+      throw new Error('invalid root filler')
+    }
   }
 
   async * cars () {
@@ -50,7 +54,10 @@ export class SimpleCarSplitter {
         blocks.push(block)
         size += block.bytes.length
       }
-      const roots = first ? originRoots : [empty]
+      let roots = first ? originRoots : [empty] // default is empty roots
+      if (this._fillRoots === 'origin') {
+        roots = originRoots
+      }
       const { writer, out } = CarWriter.create(roots)
       Object.assign(out, { version: 1, getRoots: async () => roots })
       blocks.forEach(b => writer.put(b))

lib/treewalk/joiner.js

@@ -1,15 +1,11 @@
 import { CarWriter } from '@ipld/car'
 
-/**
- * @typedef {import('@ipld/car/api').BlockReader & import('@ipld/car/api').RootsReader} ICarReader
- */
-
 export class TreewalkCarJoiner {
   /**
-   * @param {Iterable<ICarReader>} cars
+   * @param {Iterable<import('@ipld/car/api').CarReader>} cars
    */
   constructor (cars) {
-    /** @type {ICarReader[]} */
+    /** @type {import('@ipld/car/api').CarReader[]} */
     this._cars = Array.from(cars)
     if (!this._cars.length) throw new Error('missing CARs')
   }
@@ -20,7 +16,7 @@ export class TreewalkCarJoiner {
     const { writer, out } = CarWriter.create(roots)
     const writeCar = async () => {
       const written = new Set()
-      const writeBlocks = async (/** @type {ICarReader} */ reader) => {
+      const writeBlocks = async (/** @type {import('@ipld/car/api').CarReader} */ reader) => {
         for await (const b of reader.blocks()) {
           if (written.has(b.cid.toString())) continue
           await writer.put(b)

lib/treewalk/splitter.js

@@ -8,12 +8,11 @@ import * as pb from '@ipld/dag-pb'
  * @typedef {{ decoders?: import('multiformats/codecs/interface').BlockDecoder<any, any>[] }} Options
  * @typedef {import('@ipld/car/api').WriterChannel & { size: number }} WriterChannel
  * @typedef {import('multiformats/cid').CID} CID
- * @typedef {import('@ipld/car/api').BlockReader & import('@ipld/car/api').RootsReader} ICarReader
  */
 
 export class TreewalkCarSplitter {
   /**
-   * @param {ICarReader} reader
+   * @param {import('@ipld/car/api').CarReader} reader
    * @param {number} targetSize
    * @param {Options} [options]
    */