Skip to content

Choose stored blocks using their actual padding cost #33

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rafambn merged 1 commit into from
Sep 11, 2026
Merged

Choose stored blocks using their actual padding cost #33

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 16 additions & 2 deletions kflate/src/commonMain/kotlin/com/rafambn/kflate/util/BitUtils.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,15 +134,15 @@ internal fun writeBlock(
val (codeLengthTree, maxCodeLengthBits) = buildHuffmanTreeFromFrequencies(codeLengthFrequencies, 7)
val numCodeLengthCodes = countCodeLengthCodes(codeLengthTree)

val fixedBlockLength = (blockLength + 5) shl 3
val storedBlockLength = storedBlockBitLength(blockLength, bitPosition)
val fixedTypedLength = calculateCodeLength(literalFrequencies, FIXED_LENGTH_TREE) +
calculateCodeLength(distanceFrequencies, FIXED_DISTANCE_TREE) + extraBits
val dynamicTypedLength = calculateCodeLength(literalFrequencies, dynamicLiteralTree) +
calculateCodeLength(distanceFrequencies, dynamicDistanceTree) + extraBits + 14 + 3 * numCodeLengthCodes +
calculateCodeLength(codeLengthFrequencies, codeLengthTree) + 2 * codeLengthFrequencies[16] +
3 * codeLengthFrequencies[17] + 7 * codeLengthFrequencies[18]

if (shouldUseStoredBlock(blockStart, fixedBlockLength, fixedTypedLength, dynamicTypedLength)) {
if (shouldUseStoredBlock(blockStart, storedBlockLength, fixedTypedLength, dynamicTypedLength)) {
return writeFixedBlock(output, currentBitPosition, data.sliceArray(blockStart until blockStart + blockLength))
}

Expand Down Expand Up @@ -268,3 +268,17 @@ internal fun shouldUseStoredBlock(
): Boolean {
return blockStart >= 0 && storedLength <= fixedLength && storedLength <= dynamicLength
}

/**
* Returns the stored block cost after the three bit block header.
*
* The header itself is present in every block type, while a stored block adds
* padding before its four byte length header. Keeping the common header out of
* this value lets it be compared directly with the fixed and dynamic token
* costs.
*/
internal fun storedBlockBitLength(blockLength: Int, bitPosition: Long): Int {
val headerEnd = bitPosition + 3L
val padding = ((8L - (headerEnd and 7L)) and 7L).toInt()
return (blockLength shl 3) + 32 + padding
}
88 changes: 88 additions & 0 deletions kflate/src/jvmTest/kotlin/InternalHelpersCoverageTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.rafambn.kflate

import com.rafambn.kflate.algorithm.inflate
import com.rafambn.kflate.compression.Gzip as CompressionGzip
import com.rafambn.kflate.decompression.Gzip as DecompressionGzip
import com.rafambn.kflate.error.FlateError
Expand Down Expand Up @@ -29,9 +30,11 @@ import com.rafambn.kflate.util.readFourBytesBE
import com.rafambn.kflate.util.readTwoBytes
import com.rafambn.kflate.util.shiftToNextByte
import com.rafambn.kflate.util.shouldUseStoredBlock
import com.rafambn.kflate.util.storedBlockBitLength
import com.rafambn.kflate.util.toIsoStringBytes
import com.rafambn.kflate.util.writeBits
import com.rafambn.kflate.util.writeBits16
import com.rafambn.kflate.util.writeBlock
import com.rafambn.kflate.util.writeBytes
import com.rafambn.kflate.util.writeBytesBE
import kotlin.test.Test
Expand Down Expand Up @@ -84,6 +87,91 @@ class InternalHelpersCoverageTest {
assertTrue(!shouldUseStoredBlock(-1, 1, 1, 1))
assertTrue(!shouldUseStoredBlock(0, 2, 1, 2))
assertTrue(!shouldUseStoredBlock(0, 2, 2, 1))
assertEquals(117, storedBlockBitLength(10, 0))
assertEquals(112, storedBlockBitLength(10, 5))
assertEquals(119, storedBlockBitLength(10, 6))
assertTrue(shouldUseStoredBlock(0, storedBlockBitLength(10, 6), 119, 119))
}

@Test
fun blockWriterAccountsForStoredBlockAlignment() {
val data = ByteArray(26) { (230 + it).toByte() }
val symbols = IntArray(data.size) { 230 + it }
val literalFrequencies = IntArray(288)
for (symbol in symbols) {
literalFrequencies[symbol]++
}

val output = ByteArray(256)
writeBlock(
data = data,
output = output,
isFinal = true,
symbols = symbols,
literalFrequencies = literalFrequencies,
distanceFrequencies = IntArray(32),
extraBits = 0,
symbolCount = symbols.size,
blockStart = 0,
blockLength = data.size,
bitPosition = 5,
)

val blockType = (output[0].toInt() ushr 5) and 7
assertEquals(1, blockType, "block type=$blockType")
}

@Test
fun blockWriterPreservesWholeStreamStoredBlockSavings() {
val prefix = byteArrayOf(253.toByte(), 254.toByte(), 255.toByte())
val payload = ByteArray(26) { (230 + it).toByte() }
val data = prefix + payload
val prefixSymbols = intArrayOf(253, 254, 255)
val payloadSymbols = IntArray(payload.size) { 230 + it }

val prefixLiteralFrequencies = IntArray(288)
for (symbol in prefixSymbols) {
prefixLiteralFrequencies[symbol]++
}
val payloadLiteralFrequencies = IntArray(288)
for (symbol in payloadSymbols) {
payloadLiteralFrequencies[symbol]++
}

val output = ByteArray(128)
val prefixEndBitPosition = writeBlock(
data = data,
output = output,
isFinal = false,
symbols = prefixSymbols,
literalFrequencies = prefixLiteralFrequencies,
distanceFrequencies = IntArray(32),
extraBits = 0,
symbolCount = prefixSymbols.size,
blockStart = 0,
blockLength = prefix.size,
bitPosition = 0,
)
assertEquals(37L, prefixEndBitPosition)

val finalEndBitPosition = writeBlock(
data = data,
output = output,
isFinal = true,
symbols = payloadSymbols,
literalFrequencies = payloadLiteralFrequencies,
distanceFrequencies = IntArray(32),
extraBits = 0,
symbolCount = payloadSymbols.size,
blockStart = prefix.size,
blockLength = payload.size,
bitPosition = prefixEndBitPosition,
)
assertEquals(280L, finalEndBitPosition)

val compressed = output.copyOf(shiftToNextByte(finalEndBitPosition))
assertEquals(35, compressed.size)
assertContentEquals(data, inflate(compressed, InflateState(validationMode = 2)))
}

@Test
Expand Down
60 changes: 60 additions & 0 deletions performance/2026-09-09-stored/RESULTS.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
# Choose stored blocks using their actual padding cost

Use the current bit position when comparing stored, fixed and dynamic blocks. This corrects near-tie decisions at non-byte-aligned boundaries.

Base: `fe2ff51`, `dev-1.1.0`. Measured on September 9, 2026 on Linux x86_64, Intel Core i7-11800H. JVM uses JBR 17.0.14; Node uses 22.22.1.

Only KFlate was timed. Kompress sizes and timings come unchanged from the trusted September 8 archive in `performance/history.json`. Benchmark setup may produce Kompress streams for decoder validation outside timing.

## Compressed size

The JVM sweep compresses all seven tracked corpora at every listed level. Every output must round-trip through both the JDK RAW inflater and KFlate. Byte counts are deterministic. Single-run durations in the sweep JSON are diagnostic, not throughput benchmarks.

Output reduction is `(beforeBytes - afterBytes) / beforeBytes`. Totals weight each corpus by its compressed bytes; they are not averages of percentage changes.

| Level | Before bytes | After bytes | Output reduction |
| ---: | ---: | ---: | ---: |
| 0 | 76,379,868 | 76,379,868 | 0.0000% |
| 1 | 40,222,941 | 40,222,941 | 0.0000% |
| 2 | 39,269,936 | 39,269,936 | 0.0000% |
| 3 | 38,906,551 | 38,906,551 | 0.0000% |
| 4 | 38,333,169 | 38,333,169 | 0.0000% |
| 5 | 38,293,272 | 38,293,272 | 0.0000% |
| 6 | 37,882,044 | 37,882,044 | 0.0000% |
| 7 | 37,868,330 | 37,868,330 | 0.0000% |
| 8 | 37,737,043 | 37,737,043 | 0.0000% |
| 9 | 37,251,418 | 37,251,418 | 0.0000% |

### Level 6

| Corpus | Before bytes | After bytes | Reduction | Saved JVM Kompress bytes |
| --- | ---: | ---: | ---: | ---: |
| simpleText | 84 | 84 | 0.0000% | 84 |
| text | 506,455 | 506,455 | 0.0000% | 505,318 |
| model3D | 2,153 | 2,153 | 0.0000% | 2,149 |
| Rainier.bmp | 3,283,450 | 3,283,450 | 0.0000% | 3,275,337 |
| Maltese.bmp | 7,158,472 | 7,158,472 | 0.0000% | 7,096,685 |
| Sunrise.bmp | 26,838,825 | 26,838,825 | 0.0000% | 26,698,992 |
| compressed_MVT.pbf | 92,605 | 92,605 | 0.0000% | 91,408 |

### Level 9

| Corpus | Before bytes | After bytes | Reduction | Saved JVM Kompress bytes |
| --- | ---: | ---: | ---: | ---: |
| simpleText | 84 | 84 | 0.0000% | 84 |
| text | 492,523 | 492,523 | 0.0000% | 503,400 |
| model3D | 2,153 | 2,153 | 0.0000% | 2,149 |
| Rainier.bmp | 3,258,570 | 3,258,570 | 0.0000% | 3,270,382 |
| Maltese.bmp | 7,034,092 | 7,034,092 | 0.0000% | 7,090,887 |
| Sunrise.bmp | 26,372,269 | 26,372,269 | 0.0000% | 26,652,810 |
| compressed_MVT.pbf | 91,727 | 91,727 | 0.0000% | 91,400 |

## Boundary case

The regression test writes a complete RAW stream with three high-byte literals in a nonfinal fixed block, followed by 26 high-byte literals. The first block ends at bit 37. The old cost estimate chooses a final fixed block ending at bit 281, requiring 36 bytes. Accounting for padding selects a stored block ending at bit 280, requiring 35 bytes. This saves one byte in this specific case; the seven corpus files at all ten levels retain identical sizes.

## Validation

Linux JVM tests and the 70-case size sweep validate correctness. The whole-stream regression verifies all 29 decoded bytes using strict KFlate validation. No throughput improvement is claimed.

Reproduce with `./gradlew :kflate:jvmTest`; the regression is `InternalHelpersCoverageTest.blockWriterPreservesWholeStreamStoredBlockSavings`. `RatioSweep.java` contains the corpus sweep and JDK interoperability checks. Measurements ran before committing the same implementation with this report.
37 changes: 37 additions & 0 deletions performance/2026-09-09-stored/RatioSweep.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
import com.rafambn.kflate.KFlate;
import com.rafambn.kflate.compression.Raw;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.zip.Inflater;

public class RatioSweep {
public static void main(String[] args) throws Exception {
String[] names = {"simpleText", "text", "model3D", "Rainier.bmp", "Maltese.bmp", "Sunrise.bmp", "compressed_MVT.pbf"};
int first = args.length > 1 ? Integer.parseInt(args[1]) : 0;
int last = args.length > 2 ? Integer.parseInt(args[2]) : 9;
for (String name : names) {
byte[] input = Files.readAllBytes(Path.of(args[0], name));
for (int level = first; level <= last; level++) {
long start = System.nanoTime();
byte[] output = KFlate.INSTANCE.compress(input, new Raw(level, null));
double ms = (System.nanoTime() - start) / 1e6;
Inflater inflater = new Inflater(true);
try {
inflater.setInput(output);
byte[] decoded = new byte[input.length + 1];
int count = inflater.inflate(decoded);
if (!inflater.finished() || count != input.length || !Arrays.equals(input, Arrays.copyOf(decoded, count))) {
throw new AssertionError("Invalid output: " + name + " level " + level);
}
} finally {
inflater.end();
}
if (!Arrays.equals(input, KFlate.INSTANCE.decompress(output, new com.rafambn.kflate.decompression.Raw(null, null)))) {
throw new AssertionError("KFlate round trip: " + name + " level " + level);
}
System.out.printf(java.util.Locale.ROOT, "{\"corpus\":\"%s\",\"level\":%d,\"originalSizeBytes\":%d,\"compressedSizeBytes\":%d,\"singleRunMs\":%.6f}%n", name, level, input.length, output.length, ms);
}
}
}
}
70 changes: 70 additions & 0 deletions performance/2026-09-09-stored/after-sizes.jsonl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
{"corpus":"simpleText","level":0,"originalSizeBytes":100,"compressedSizeBytes":105,"singleRunMs":17.743099}
{"corpus":"simpleText","level":1,"originalSizeBytes":100,"compressedSizeBytes":84,"singleRunMs":9.717891}
{"corpus":"simpleText","level":2,"originalSizeBytes":100,"compressedSizeBytes":84,"singleRunMs":0.338182}
{"corpus":"simpleText","level":3,"originalSizeBytes":100,"compressedSizeBytes":84,"singleRunMs":0.307948}
{"corpus":"simpleText","level":4,"originalSizeBytes":100,"compressedSizeBytes":84,"singleRunMs":0.272352}
{"corpus":"simpleText","level":5,"originalSizeBytes":100,"compressedSizeBytes":84,"singleRunMs":0.224498}
{"corpus":"simpleText","level":6,"originalSizeBytes":100,"compressedSizeBytes":84,"singleRunMs":0.267385}
{"corpus":"simpleText","level":7,"originalSizeBytes":100,"compressedSizeBytes":84,"singleRunMs":0.202015}
{"corpus":"simpleText","level":8,"originalSizeBytes":100,"compressedSizeBytes":84,"singleRunMs":0.198751}
{"corpus":"simpleText","level":9,"originalSizeBytes":100,"compressedSizeBytes":84,"singleRunMs":0.478129}
{"corpus":"text","level":0,"originalSizeBytes":1232923,"compressedSizeBytes":1233018,"singleRunMs":4.361065}
{"corpus":"text","level":1,"originalSizeBytes":1232923,"compressedSizeBytes":599813,"singleRunMs":52.279190}
{"corpus":"text","level":2,"originalSizeBytes":1232923,"compressedSizeBytes":581519,"singleRunMs":36.646794}
{"corpus":"text","level":3,"originalSizeBytes":1232923,"compressedSizeBytes":565252,"singleRunMs":36.735742}
{"corpus":"text","level":4,"originalSizeBytes":1232923,"compressedSizeBytes":531466,"singleRunMs":65.162215}
{"corpus":"text","level":5,"originalSizeBytes":1232923,"compressedSizeBytes":521837,"singleRunMs":63.198932}
{"corpus":"text","level":6,"originalSizeBytes":1232923,"compressedSizeBytes":506455,"singleRunMs":74.787364}
{"corpus":"text","level":7,"originalSizeBytes":1232923,"compressedSizeBytes":506097,"singleRunMs":76.796766}
{"corpus":"text","level":8,"originalSizeBytes":1232923,"compressedSizeBytes":513489,"singleRunMs":81.337974}
{"corpus":"text","level":9,"originalSizeBytes":1232923,"compressedSizeBytes":492523,"singleRunMs":220.988788}
{"corpus":"model3D","level":0,"originalSizeBytes":2478,"compressedSizeBytes":2483,"singleRunMs":0.043470}
{"corpus":"model3D","level":1,"originalSizeBytes":2478,"compressedSizeBytes":2169,"singleRunMs":0.261532}
{"corpus":"model3D","level":2,"originalSizeBytes":2478,"compressedSizeBytes":2168,"singleRunMs":0.149276}
{"corpus":"model3D","level":3,"originalSizeBytes":2478,"compressedSizeBytes":2166,"singleRunMs":0.138922}
{"corpus":"model3D","level":4,"originalSizeBytes":2478,"compressedSizeBytes":2165,"singleRunMs":0.127254}
{"corpus":"model3D","level":5,"originalSizeBytes":2478,"compressedSizeBytes":2153,"singleRunMs":0.614859}
{"corpus":"model3D","level":6,"originalSizeBytes":2478,"compressedSizeBytes":2153,"singleRunMs":0.203552}
{"corpus":"model3D","level":7,"originalSizeBytes":2478,"compressedSizeBytes":2153,"singleRunMs":0.282327}
{"corpus":"model3D","level":8,"originalSizeBytes":2478,"compressedSizeBytes":2153,"singleRunMs":0.254732}
{"corpus":"model3D","level":9,"originalSizeBytes":2478,"compressedSizeBytes":2153,"singleRunMs":0.446036}
{"corpus":"Rainier.bmp","level":0,"originalSizeBytes":6220854,"compressedSizeBytes":6221329,"singleRunMs":6.567415}
{"corpus":"Rainier.bmp","level":1,"originalSizeBytes":6220854,"compressedSizeBytes":3387685,"singleRunMs":109.501732}
{"corpus":"Rainier.bmp","level":2,"originalSizeBytes":6220854,"compressedSizeBytes":3357387,"singleRunMs":99.991340}
{"corpus":"Rainier.bmp","level":3,"originalSizeBytes":6220854,"compressedSizeBytes":3345679,"singleRunMs":98.796737}
{"corpus":"Rainier.bmp","level":4,"originalSizeBytes":6220854,"compressedSizeBytes":3325593,"singleRunMs":102.281264}
{"corpus":"Rainier.bmp","level":5,"originalSizeBytes":6220854,"compressedSizeBytes":3313388,"singleRunMs":109.360161}
{"corpus":"Rainier.bmp","level":6,"originalSizeBytes":6220854,"compressedSizeBytes":3283450,"singleRunMs":120.609099}
{"corpus":"Rainier.bmp","level":7,"originalSizeBytes":6220854,"compressedSizeBytes":3282047,"singleRunMs":125.539764}
{"corpus":"Rainier.bmp","level":8,"originalSizeBytes":6220854,"compressedSizeBytes":3274905,"singleRunMs":140.489890}
{"corpus":"Rainier.bmp","level":9,"originalSizeBytes":6220854,"compressedSizeBytes":3258570,"singleRunMs":4123.953838}
{"corpus":"Maltese.bmp","level":0,"originalSizeBytes":16427390,"compressedSizeBytes":16428645,"singleRunMs":6.887125}
{"corpus":"Maltese.bmp","level":1,"originalSizeBytes":16427390,"compressedSizeBytes":8344362,"singleRunMs":304.151030}
{"corpus":"Maltese.bmp","level":2,"originalSizeBytes":16427390,"compressedSizeBytes":7762283,"singleRunMs":351.255122}
{"corpus":"Maltese.bmp","level":3,"originalSizeBytes":16427390,"compressedSizeBytes":7494606,"singleRunMs":389.044940}
{"corpus":"Maltese.bmp","level":4,"originalSizeBytes":16427390,"compressedSizeBytes":7272625,"singleRunMs":405.927534}
{"corpus":"Maltese.bmp","level":5,"originalSizeBytes":16427390,"compressedSizeBytes":7265831,"singleRunMs":521.798475}
{"corpus":"Maltese.bmp","level":6,"originalSizeBytes":16427390,"compressedSizeBytes":7158472,"singleRunMs":529.408796}
{"corpus":"Maltese.bmp","level":7,"originalSizeBytes":16427390,"compressedSizeBytes":7157735,"singleRunMs":534.236659}
{"corpus":"Maltese.bmp","level":8,"originalSizeBytes":16427390,"compressedSizeBytes":7125327,"singleRunMs":537.898069}
{"corpus":"Maltese.bmp","level":9,"originalSizeBytes":16427390,"compressedSizeBytes":7034092,"singleRunMs":1486.123488}
{"corpus":"Sunrise.bmp","level":0,"originalSizeBytes":52344054,"compressedSizeBytes":52348049,"singleRunMs":32.460883}
{"corpus":"Sunrise.bmp","level":1,"originalSizeBytes":52344054,"compressedSizeBytes":27793016,"singleRunMs":929.263105}
{"corpus":"Sunrise.bmp","level":2,"originalSizeBytes":52344054,"compressedSizeBytes":27471887,"singleRunMs":1111.007610}
{"corpus":"Sunrise.bmp","level":3,"originalSizeBytes":52344054,"compressedSizeBytes":27404419,"singleRunMs":1207.959139}
{"corpus":"Sunrise.bmp","level":4,"originalSizeBytes":52344054,"compressedSizeBytes":27107867,"singleRunMs":1293.493213}
{"corpus":"Sunrise.bmp","level":5,"originalSizeBytes":52344054,"compressedSizeBytes":27096750,"singleRunMs":1539.210897}
{"corpus":"Sunrise.bmp","level":6,"originalSizeBytes":52344054,"compressedSizeBytes":26838825,"singleRunMs":1699.000512}
{"corpus":"Sunrise.bmp","level":7,"originalSizeBytes":52344054,"compressedSizeBytes":26827604,"singleRunMs":1941.963146}
{"corpus":"Sunrise.bmp","level":8,"originalSizeBytes":52344054,"compressedSizeBytes":26728454,"singleRunMs":2157.738874}
{"corpus":"Sunrise.bmp","level":9,"originalSizeBytes":52344054,"compressedSizeBytes":26372269,"singleRunMs":14320.729426}
{"corpus":"compressed_MVT.pbf","level":0,"originalSizeBytes":146224,"compressedSizeBytes":146239,"singleRunMs":0.056534}
{"corpus":"compressed_MVT.pbf","level":1,"originalSizeBytes":146224,"compressedSizeBytes":95812,"singleRunMs":2.755843}
{"corpus":"compressed_MVT.pbf","level":2,"originalSizeBytes":146224,"compressedSizeBytes":94608,"singleRunMs":3.234385}
{"corpus":"compressed_MVT.pbf","level":3,"originalSizeBytes":146224,"compressedSizeBytes":94345,"singleRunMs":3.484409}
{"corpus":"compressed_MVT.pbf","level":4,"originalSizeBytes":146224,"compressedSizeBytes":93369,"singleRunMs":3.746026}
{"corpus":"compressed_MVT.pbf","level":5,"originalSizeBytes":146224,"compressedSizeBytes":93229,"singleRunMs":4.607743}
{"corpus":"compressed_MVT.pbf","level":6,"originalSizeBytes":146224,"compressedSizeBytes":92605,"singleRunMs":4.673451}
{"corpus":"compressed_MVT.pbf","level":7,"originalSizeBytes":146224,"compressedSizeBytes":92610,"singleRunMs":4.873132}
{"corpus":"compressed_MVT.pbf","level":8,"originalSizeBytes":146224,"compressedSizeBytes":92631,"singleRunMs":5.248747}
{"corpus":"compressed_MVT.pbf","level":9,"originalSizeBytes":146224,"compressedSizeBytes":91727,"singleRunMs":10.274067}
Loading