feat: add treeleaf strategy

README.md

@@ -15,11 +15,12 @@ npm install carbites
 
 ## Usage
 
-Carbites supports 3 different strategies:
+Carbites supports 4 different strategies:
 
 1. [**Simple**](#simple) (default) - fast but naive, only the first CAR output has a root CID, subsequent CARs have a placeholder "empty" CID.
 2. [**Rooted**](#rooted) - like simple, but creates a custom root node to ensure all blocks in a CAR are referenced.
 3. [**Treewalk**](#treewalk) - walks the DAG to pack sub-graphs into each CAR file that is output. Every CAR has the same root CID, but contains a different portion of the DAG.
+4. [**Treeleaf**](#treeleaf) - the first CAR contains all non-leaf nodes. Subsequent CARs contain only leaf-nodes.
 
 ### Simple
 
@@ -116,7 +117,55 @@ Every CAR file has the _same_ root CID but a different portion of the DAG. The D
 
 </details>
 
-### CLI
+### Treeleaf
+
+<details>
+  <summary>Example</summary>
+
+  ```js
+  import { TreeleafCarSplitter } from 'carbites/treeleaf'
+  import { CarReader } from '@ipld/car/reader'
+  import { printTree } from 'imaginary-car-tree-lib'
+  import fs from 'fs'
+
+  const bigCar = await CarReader.fromIterable(fs.createReadStream('/path/to/big.car'))
+  const targetSize = 1024 * 1024 * 100 // chunk to ~100MB CARs
+  const splitter = new TreeleafCarSplitter(bigCar, targetSize)
+
+  const count = 0
+  for await (const car of splitter.cars()) {
+    const reader = await CarReader.fromIterable(car)
+    const [splitCarRootCid] = await reader.getRoots()
+    console.log(`## car ${count} – root: ${splitCarRootCid}`)
+    console.log(printTree(reader))
+    count++
+  }
+  /* output:
+  ## car 0 - root: CID(bafybeidpq6auelplqkozwd3kobvgv4jow72r2yt6ffjzenemqhylmmxngi)
+  bafybeidpq6auelplqkozwd3kobvgv4jow72r2yt6ffjzenemqhylmmxngi
+  └─┬ bafybeibw77p4afchrvmo6ep4fpv7j4aehrn2yqs3tv3uuofc6oag36zq2q
+    ├─┬ bafybeih2w5euyf3sodc6efxtpahgwkata46fupjysi4bbnazb4n2b25rry
+    │ └─┬ bafybeif5xvhik6thha5ykg3g73jo3mqd5mhb7orzeznnvwquashmwryhai
+    │   ├── bafkreidkmypp3asq3xiu6mmtcfbzmmhcobpmnaqcyns6535w3u7bz7el64  ❌ missing
+    │   ├── bafkreid3pefuwyvhsnlrz6xk67f4f6opgqhcrle4bf47kvachwojdgt7re  ❌ missing
+    │   └── bafkreigr3beehu5ebbgoisx4rl2vyaqebmohubauc6avefodb5jauz3tyi  ❌ missing
+    └─┬ bafybeia2i6eqwfqrixh446pavwev37kywowmdpgvx34fbv7asdh3qwpm3y
+      └── bafkreih2bhak5yv7g4vft5c37j7dw5rqnnsyyuzsifczehhhpm3t655oae    ❌ missing
+
+  ## car 1 - root: CID(bafkqaaa)
+  bafkreidkmypp3asq3xiu6mmtcfbzmmhcobpmnaqcyns6535w3u7bz7el64
+  bafkreid3pefuwyvhsnlrz6xk67f4f6opgqhcrle4bf47kvachwojdgt7re
+  bafkreigr3beehu5ebbgoisx4rl2vyaqebmohubauc6avefodb5jauz3tyi
+  bafkreih2bhak5yv7g4vft5c37j7dw5rqnnsyyuzsifczehhhpm3t655oae
+  */
+  ```
+
+  ⚠️ Note: The first CAR output has roots in the header, subsequent CARs have an empty root CID [`bafkqaaa`](https://cid.ipfs.io/#bafkqaaa) as [recommended](https://ipld.io/specs/transport/car/carv1/#number-of-roots).
+
+</details>
+
+
+## CLI
 
 Install the CLI tool to use Carbites from the comfort of your terminal:
 

lib/treeleaf/index.js

@@ -0,0 +1,7 @@
+import { TreeleafCarSplitter } from './splitter.js'
+import { TreeleafCarJoiner } from './joiner.js'
+
+export {
+  TreeleafCarSplitter,
+  TreeleafCarJoiner
+}

lib/treeleaf/joiner.js

@@ -0,0 +1,41 @@
+import { CarWriter } from '@ipld/car'
+
+/**
+ * @typedef {import('@ipld/car/api').BlockReader & import('@ipld/car/api').RootsReader} ICarReader
+ */
+
+export class TreeleafCarJoiner {
+  /**
+   * @param {Iterable<ICarReader>} cars
+   */
+  constructor (cars) {
+    /** @type {ICarReader[]} */
+    this._cars = Array.from(cars)
+    if (!this._cars.length) throw new Error('missing CARs')
+  }
+
+  async * car () {
+    const reader = this._cars[0]
+    const roots = await reader.getRoots()
+    const { writer, out } = CarWriter.create(roots)
+    const writeCar = async () => {
+      try {
+        for await (const b of reader.blocks()) {
+          await writer.put(b)
+        }
+        for (const reader of this._cars.slice(1)) {
+          for await (const b of reader.blocks()) {
+            await writer.put(b)
+          }
+        }
+      } catch (err) {
+        console.error(err) // TODO: how to forward this on?
+      } finally {
+        await writer.close()
+      }
+    }
+
+    writeCar()
+    yield * out
+  }
+}

lib/treeleaf/splitter.js

@@ -0,0 +1,112 @@
+import { CarReader, CarWriter } from '@ipld/car'
+import * as Block from 'multiformats/block'
+import { CID } from 'multiformats/cid'
+import * as raw from 'multiformats/codecs/raw'
+import * as cbor from '@ipld/dag-cbor'
+import * as json from '@ipld/dag-json'
+import * as pb from '@ipld/dag-pb'
+
+/**
+ * @typedef {import('@ipld/car/api').BlockReader & import('@ipld/car/api').RootsReader} ICarReader
+ * @typedef {import('@ipld/car/api').WriterChannel} WriterChannel
+ * @typedef {import('multiformats/codecs/interface').BlockDecoder<any, any>} BlockDecoder
+ * @typedef {{ decoders?: BlockDecoder[] }} Options
+ * @typedef {number} Code
+ */
+
+/**
+ * A work-around for use-cases where the inclusion of a root CID is difficult
+ * but needing to be safely within the "at least one" recommendation is to use
+ * an empty CID: \x01\x55\x00\x00 (zero-length "identity" multihash with "raw"
+ * codec). Since current implementations for this version of the CAR
+ * specification don't check for the existence of root CIDs (see Root CID block
+ * existence), this will be safe as far as CAR implementations are concerned.
+ * However, there is no guarantee that applications that use CAR files will
+ * correctly consume (ignore) this empty root CID.
+ *
+ * https://github.com/ipld/specs/blob/master/block-layer/content-addressable-archives.md#number-of-roots
+ */
+const empty = CID.parse('bafkqaaa')
+
+export class TreeleafCarSplitter {
+  /**
+   * @param {ICarReader} reader
+   * @param {number} targetSize
+   * @param {Options} [options]
+   */
+  constructor (reader, targetSize, options = {}) {
+    if (typeof targetSize !== 'number' || targetSize <= 0) {
+      throw new Error('invalid target chunk size')
+    }
+    this._reader = reader
+    this._targetSize = targetSize
+    const codecs = [pb, raw, cbor, json, ...(options.decoders || [])]
+    /** @type Map<Code, BlockDecoder> */
+    this._decoderMap = new Map(codecs.map(d => [d.code, d]))
+  }
+
+  async * cars () {
+    /** @type {(roots?: CID[]) => WriterChannel} */
+    const createCarWriter = (roots = [empty]) => {
+      const car = CarWriter.create(roots)
+      Object.assign(car.out, { version: 1, getRoots: async () => roots })
+      return car
+    }
+    const leaves = []
+    const originRoots = await this._reader.getRoots()
+    const tree = createCarWriter(originRoots)
+
+    for await (const block of this._reader.blocks()) {
+      if (block.cid.code === raw.code) {
+        leaves.push(block)
+        continue
+      }
+      const codec = this._decoderMap.get(block.cid.code)
+      if (!codec) throw new Error(`missing decoder for ${block.cid.code}`)
+      const decoded = Block.createUnsafe({ cid: block.cid, bytes: block.bytes, codec })
+      const hasLinks = Array.from(decoded.links()).length > 0
+      if (hasLinks) {
+        tree.writer.put(block)
+      } else {
+        leaves.push(block)
+      }
+    }
+    tree.writer.close()
+    yield tree.out
+
+    // finished tree car, now write multiple leaf cars
+
+    let size = 0
+    let car = createCarWriter()
+    for (const block of leaves) {
+      if (size >= this._targetSize) {
+        car.writer.close()
+        yield car.out
+        car = createCarWriter()
+      }
+      car.writer.put(block)
+      size += block.bytes.length
+    }
+    car.writer.close()
+    yield car.out
+  }
+
+  /**
+   * @param {Blob} blob
+   * @param {number} targetSize
+   */
+  static async fromBlob (blob, targetSize) {
+    const buffer = await blob.arrayBuffer()
+    const reader = await CarReader.fromBytes(new Uint8Array(buffer))
+    return new TreeleafCarSplitter(reader, targetSize)
+  }
+
+  /**
+   * @param {AsyncIterable<Uint8Array>} iterable
+   * @param {number} targetSize
+   */
+  static async fromIterable (iterable, targetSize) {
+    const reader = await CarReader.fromIterable(iterable)
+    return new TreeleafCarSplitter(reader, targetSize)
+  }
+}

package.json

@@ -21,6 +21,7 @@
   "dependencies": {
     "@ipld/car": "^3.0.1",
     "@ipld/dag-cbor": "^6.0.3",
+    "@ipld/dag-json": "^8.0.11",
     "@ipld/dag-pb": "^2.0.2",
     "multiformats": "^9.0.4"
   },