diff --git a/README.md b/README.md index 61c03f3..ca4247b 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,13 @@ # ExCodecs +[![CI](https://github.com/thanos/codecs/actions/workflows/ci.yml/badge.svg)](https://github.com/thanos/codecs/actions/workflows/ci.yml) +[![Hex.pm](https://img.shields.io/hexpm/v/ex_codecs.svg)](https://hex.pm/packages/ex_codecs) +[![Hex.pm Downloads](https://img.shields.io/hexpm/dt/ex_codecs.svg)](https://hex.pm/packages/ex_codecs) +[![Documentation](https://img.shields.io/badge/docs-hex.pm-blue.svg)](https://hexdocs.pm/ex_codecs) +[![License](https://img.shields.io/hexpm/l/ex_codecs.svg)](https://github.com/thanos/codecs/blob/main/LICENSE) +[![Elixir](https://img.shields.io/badge/Elixir-%7E%3E%201.17-purple.svg)](https://elixir-lang.org) +[![Coverage Status](https://coveralls.io/repos/github/thanos/codecs/badge.svg?branch=main)](https://coveralls.io/github/thanos/codecs?branch=main) + An extensible BEAM-native codec framework for Elixir. ExCodecs provides a unified API for compression, decompression, hashing, checksums, @@ -127,7 +135,7 @@ Returns a structured `%ExCodecs.Codec{}` struct with metadata, or | `:lz4` | compression | No | No | -- | | `:snappy` | compression | No | No | -- | | `:bzip2` | compression | Yes | No | `block_size` (1-9, default 9) | -| `:blosc2` | compression | Yes | Yes | `cname`, `clevel` (0-9, default 5), `shuffle` (`:none` / `:byte`, default `:byte`), `typesize` (default 8) | +| `:blosc2` | compression | Yes | Yes | `cname`, `clevel` (0-9, default 5), `shuffle` (`:none` / `:byte` / `:bit`, default `:byte`), `typesize` (default 8) | ## Architecture diff --git a/config/config.exs b/config/config.exs index 82cdd70..0c4c6ae 100644 --- a/config/config.exs +++ b/config/config.exs @@ -1,12 +1,6 @@ import Config -# In dev/test, build the NIF from source so `mix compile` and `mix test` -# work without precompiled artifacts. Release consumers use precompiled NIFs. -# -# CI and release workflows can also set EX_CODECS_BUILD=1 or -# RUSTLER_PRECOMPILED_FORCE_BUILD_ALL=1 to force source compilation -# (e.g. the publish_hex job in .github/workflows/release.yml). if config_env() in [:dev, :test] do config :rustler_precompiled, :force_build, ex_codecs: true import_config "#{config_env()}.exs" -end \ No newline at end of file +end diff --git a/lib/ex_codecs/compression/blosc2.ex b/lib/ex_codecs/compression/blosc2.ex index 0b07dcd..73ef086 100644 --- a/lib/ex_codecs/compression/blosc2.ex +++ b/lib/ex_codecs/compression/blosc2.ex @@ -13,14 +13,16 @@ defmodule ExCodecs.Compression.Blosc2 do `:zstd`, `:snappy`, `:zlib` (default: `:lz4`) * `:clevel` — Compression level 0-9 (default: 5). 0 means no compression (store raw data). - * `:shuffle` — Shuffle filter: `:none`, `:byte` (default: `:byte`). + * `:shuffle` — Shuffle filter: `:none`, `:byte`, `:bit` (default: `:byte`). Byte shuffle reorders data to improve compression ratios for typed data. + Bit shuffle transposes at the bit level for even better compression on numerical data. * `:typesize` — Element size in bytes for shuffle (default: 8) ## Performance Characteristics * Optimized for numerical/array data * Byte shuffle dramatically improves compression ratios for typed data + * Bit shuffle can further improve compression on numerical data with small deltas * Excellent when used with appropriate typesize and shuffle settings ## Examples @@ -47,7 +49,7 @@ defmodule ExCodecs.Compression.Blosc2 do @default_typesize 8 @valid_cnames [:lz4, :lz4hc, :blosclz, :zstd, :snappy, :zlib] - @valid_shuffles [:none, :byte] + @valid_shuffles [:none, :byte, :bit] @doc """ Returns codec metadata for the registry. @@ -73,7 +75,7 @@ defmodule ExCodecs.Compression.Blosc2 do * `:cname` — Internal compressor atom (default: `:lz4`) * `:clevel` — Compression level 0-9 (default: 5) - * `:shuffle` — Shuffle filter: `:none`, `:byte` (default: `:byte`) + * `:shuffle` — Shuffle filter: `:none`, `:byte`, `:bit` (default: `:byte`) * `:typesize` — Element size in bytes (default: 8) """ @impl true @@ -151,4 +153,5 @@ defmodule ExCodecs.Compression.Blosc2 do defp shuffle_to_int(:none), do: 0 defp shuffle_to_int(:byte), do: 1 + defp shuffle_to_int(:bit), do: 2 end diff --git a/livebooks/01_introduction.livemd b/livebooks/01_introduction.livemd index 653526c..7b9bb87 100644 --- a/livebooks/01_introduction.livemd +++ b/livebooks/01_introduction.livemd @@ -1,39 +1,52 @@ # Introduction to ExCodecs ```elixir -Mix.install([ - {:ex_codecs, path: Path.join(__DIR__, "..")} -]) +local_path = Path.join(__DIR__, "../mix.exs") + +ex_codecs_dep = + if File.exists?(local_path) do + [{:ex_codecs, path: Path.join(__DIR__, "..")}, {:rustler, "~> 0.36"}] + else + [{:ex_codecs, "~> 0.1"}] + end + +config = + if File.exists?(local_path) do + [rustler_precompiled: [force_build: [ex_codecs: true]]] + else + [] + end + +Mix.install(ex_codecs_dep, config: config) ``` ## What is a Codec? A **codec** (coder-decoder) is an abstraction for transforming data between two representations. In ExCodecs, the two fundamental operations are: -- **Encode** — transform data into a compact or structured form -- **Decode** — recover the original data from the encoded form +* **Encode** — transform data into a compact or structured form +* **Decode** — recover the original data from the encoded form For compression codecs, encoding means compressing and decoding means decompressing. But the codec abstraction extends beyond compression — it can represent hashing, checksums, binary encodings, and content-addressing transforms, all through the same `encode`/`decode` interface. ```elixir -# The universal codec interface -{:ok, encoded} = ExCodecs.encode(:some_codec, original_data) -{:ok, decoded} = ExCodecs.decode(:some_codec, encoded) - -# Round-trip property: decoded == original_data +# The universal codec interface: +# {:ok, encoded} = ExCodecs.encode(:some_codec, data) +# {:ok, decoded} = ExCodecs.decode(:some_codec, encoded) +# decoded == data # round-trip guarantee ``` ## Why Codecs Matter Compression and encoding are fundamental to production systems: -| Concern | How Codecs Help | -|---|---| -| **Storage cost** | Compressed data uses less disk space | -| **Network bandwidth** | Smaller payloads mean faster transfers | -| **Data integrity** | Decode failures detect corruption | -| **Memory efficiency** | Compressed caches fit more in RAM | -| **Scientific data** | Blosc2 shuffle+compress slashes array sizes | +| Concern | How Codecs Help | +| --------------------- | ------------------------------------------- | +| **Storage cost** | Compressed data uses less disk space | +| **Network bandwidth** | Smaller payloads mean faster transfers | +| **Data integrity** | Decode failures detect corruption | +| **Memory efficiency** | Compressed caches fit more in RAM | +| **Scientific data** | Blosc2 shuffle+compress slashes array sizes | A codec framework gives you a **single consistent API** over multiple algorithms, so you can choose the right tool per workload without rewriting integration code. @@ -123,13 +136,13 @@ end ### Codec Feature Summary -| Codec | Category | Configurable | Streaming | Best For | -|---|---|---|---|---| -| `:zstd` | compression | Yes (level 1–22) | Yes | General-purpose, high ratio | -| `:lz4` | compression | Yes (level 1–16) | No | Real-time, low latency | -| `:snappy` | compression | No | No | Short-lived data, low overhead | -| `:bzip2` | compression | Yes (block_size 1–9) | No | Archival, maximum ratio | -| `:blosc2` | compression | Yes (many options) | Yes | Numerical/array data | +| Codec | Category | Configurable | Streaming | Best For | +| --------- | ----------- | ---------------------- | --------- | ------------------------------ | +| `:zstd` | compression | Yes (level 1–22) | Yes | General-purpose, high ratio | +| `:lz4` | compression | Yes (level 1–16) | No | Real-time, low latency | +| `:snappy` | compression | No | No | Short-lived data, low overhead | +| `:bzip2` | compression | Yes (block_size 1–9) | No | Archival, maximum ratio | +| `:blosc2` | compression | Yes (many options) | Yes | Numerical/array data | ## Error Handling @@ -146,7 +159,7 @@ IO.puts("Reason: #{err.reason}") ## What's Next? -- **[Compression Fundamentals](02_compression_fundamentals.livemd)** — theory, trade-offs, and interactive benchmarks -- **[Codec Comparison](03_codec_comparison.livemd)** — side-by-side performance analysis -- **[Building Storage Systems](04_building_storage_systems.livemd)** — practical patterns for production use -- **[Zarr-Style Workloads](05_zarr_style_workloads.livemd)** — scientific dataset compression with Blosc2 \ No newline at end of file +* **[Compression Fundamentals](02_compression_fundamentals.livemd)** — theory, trade-offs, and interactive benchmarks +* **[Codec Comparison](03_codec_comparison.livemd)** — side-by-side performance analysis +* **[Building Storage Systems](04_building_storage_systems.livemd)** — practical patterns for production use +* **[Zarr-Style Workloads](05_zarr_style_workloads.livemd)** — scientific dataset compression with Blosc2 diff --git a/livebooks/02_compression_fundamentals.livemd b/livebooks/02_compression_fundamentals.livemd index 054b8a4..016262d 100644 --- a/livebooks/02_compression_fundamentals.livemd +++ b/livebooks/02_compression_fundamentals.livemd @@ -1,11 +1,26 @@ # Compression Fundamentals ```elixir -Mix.install([ - {:ex_codecs, path: Path.join(__DIR__, "..")}, - {:kino, "~> 0.14"}, - {:vega_lite, "~> 0.1"} -]) +local_path = Path.join(__DIR__, "../mix.exs") + +ex_codecs_dep = + if File.exists?(local_path) do + [{:ex_codecs, path: Path.join(__DIR__, "..")}, {:rustler, "~> 0.36"}] + else + [{:ex_codecs, "~> 0.1"}] + end + +config = + if File.exists?(local_path) do + [rustler_precompiled: [force_build: [ex_codecs: true]]] + else + [] + end + +Mix.install( + ex_codecs_dep ++ [{:jason, "~> 1.4"}, {:kino, "~> 0.14"}, {:kino_vega_lite, "~> 0.1.13"}], + config: config +) ``` ## How Compression Works @@ -103,19 +118,13 @@ IO.puts("Zstd (plain): #{byte_size(zstd_plain)} bytes") ```elixir # Generate test datasets -datasets = %{ - "JSON" => Jason.encode!(%{for i <- 1..500, do: %{id: i, name: "item_#{i}", value: :rand.uniform(1000)}}), -} - -# For this demo, build JSON-like text manually -json_like = "[" <> Enum.join(for i <- 1..500 do - "{\"id\":#{i},\"name\":\"item_#{i}\",\"value\":#{:rand.uniform(1000)}}" -end, ",") <> "]" +json_like = Jason.encode!(for i <- 1..500, do: %{id: i, name: "item_#{i}", value: :rand.uniform(1000)}) datasets = %{ "Repetitive" => String.duplicate("abcdefghij", 5000), "Natural text" => String.duplicate("The quick brown fox jumps over the lazy dog. ", 300), - "Semi-random" => for _ <- 1..10000, into: <<>>, do: <<:rand.uniform(255)>> + "Semi-random" => (for _ <- 1..10000, into: <<>>, do: <<:rand.uniform(255)>>), + "JSON-like" => json_like } results = for {label, data} <- datasets, codec <- [:lz4, :snappy, :zstd, :bzip2] do @@ -146,7 +155,6 @@ VegaLite.new(width: 600, height: 400) |> VegaLite.encode_field(:y, "ratio", type: :quantitative, title: "Compressed size (%)") |> VegaLite.encode_field(:color, "dataset", type: :nominal) |> VegaLite.encode_field(:column, "dataset", type: :nominal) -|> Kino.VegaLite.new() ``` ### Speed Visualization @@ -159,7 +167,6 @@ VegaLite.new(width: 600, height: 400) |> VegaLite.encode_field(:y, "encode_us", type: :quantitative, title: "Encode time (µs)") |> VegaLite.encode_field(:color, "dataset", type: :nominal) |> VegaLite.encode_field(:column, "dataset", type: :nominal) -|> Kino.VegaLite.new() ``` ## When Not to Compress @@ -175,9 +182,10 @@ IO.puts("Compressed data can actually GROW due to header overhead") ``` Rules of thumb: -- **Don't compress encrypted or already-compressed data** — you waste CPU for no gain -- **Avoid compressing tiny payloads** — the codec header overhead may exceed savings -- **Consider latency** — LZ4/Snappy for hot paths, Bzip2 only for cold storage + +* **Don't compress encrypted or already-compressed data** — you waste CPU for no gain +* **Avoid compressing tiny payloads** — the codec header overhead may exceed savings +* **Consider latency** — LZ4/Snappy for hot paths, Bzip2 only for cold storage ## Memory vs CPU Considerations @@ -221,4 +229,4 @@ end 2. **No single best codec** — each excels for different data and latency requirements 3. **Shuffle transforms** (Blosc2) dramatically improve compression of typed binary data 4. **Compression level is a dial** — higher means smaller output but more CPU/memory -5. **Measure your actual data** — use the [Codec Comparison](03_codec_comparison.livemd) livebook with your own datasets \ No newline at end of file +5. **Measure your actual data** — use the [Codec Comparison](03_codec_comparison.livemd) livebook with your own datasets diff --git a/livebooks/03_codec_comparison.livemd b/livebooks/03_codec_comparison.livemd index 6f2b1c9..b5b372b 100644 --- a/livebooks/03_codec_comparison.livemd +++ b/livebooks/03_codec_comparison.livemd @@ -1,11 +1,26 @@ # Codec Comparison ```elixir -Mix.install([ - {:ex_codecs, path: Path.join(__DIR__, "..")}, - {:kino, "~> 0.14"}, - {:vega_lite, "~> 0.1"} -]) +local_path = Path.join(__DIR__, "../mix.exs") + +ex_codecs_dep = + if File.exists?(local_path) do + [{:ex_codecs, path: Path.join(__DIR__, "..")}, {:rustler, "~> 0.36"}] + else + [{:ex_codecs, "~> 0.1"}] + end + +config = + if File.exists?(local_path) do + [rustler_precompiled: [force_build: [ex_codecs: true]]] + else + [] + end + +Mix.install( + ex_codecs_dep ++ [{:jason, "~> 1.4"}, {:kino, "~> 0.14"}, {:kino_vega_lite, "~> 0.1.13"}], + config: config +) ``` ## Setup: Test Datasets @@ -69,7 +84,6 @@ VegaLite.new(width: 700, height: 350) |> VegaLite.encode_field(:y, "ratio_pct", type: :quantitative, title: "Compressed Size (%)", scale: [domain: [0, 110]]) |> VegaLite.encode_field(:color, "codec", type: :nominal) |> VegaLite.encode_field(:column, "dataset", type: :nominal) -|> Kino.VegaLite.new() ``` ### Space Savings Chart @@ -82,7 +96,6 @@ VegaLite.new(width: 700, height: 350) |> VegaLite.encode_field(:y, "savings_pct", type: :quantitative, title: "Space Saved (%)") |> VegaLite.encode_field(:color, "codec", type: :nominal) |> VegaLite.encode_field(:column, "dataset", type: :nominal) -|> Kino.VegaLite.new() ``` ## Speed Benchmarks @@ -128,7 +141,6 @@ VegaLite.new(width: 700, height: 350) |> VegaLite.encode_field(:y, "encode_time_us", type: :quantitative, title: "Encode Time (µs)") |> VegaLite.encode_field(:color, "codec", type: :nominal) |> VegaLite.encode_field(:column, "dataset", type: :nominal) -|> Kino.VegaLite.new() ``` ### Decode Speed Chart @@ -141,7 +153,6 @@ VegaLite.new(width: 700, height: 350) |> VegaLite.encode_field(:y, "decode_time_us", type: :quantitative, title: "Decode Time (µs)") |> VegaLite.encode_field(:color, "codec", type: :nominal) |> VegaLite.encode_field(:column, "dataset", type: :nominal) -|> Kino.VegaLite.new() ``` ## Memory Usage @@ -201,21 +212,21 @@ end ```elixir use_case = Kino.Input.select("Your use case:", [ - "Real-time / low latency" => :speed, - "General purpose" => :balanced, - "Maximum compression / archival" => :ratio, - "Numerical / scientific data" => :numeric, - "Small payloads / low overhead" => :tiny + speed: "Real-time / low latency", + balanced: "General purpose", + ratio: "Maximum compression / archival", + numeric: "Numerical / scientific data", + tiny: "Small payloads / low overhead" ]) data_type = Kino.Input.select("Data type:", [ - "Text / JSON" => :text, - "Binary blobs" => :binary, - "Typed arrays (floats, ints)" => :array, - "Mixed" => :mixed + text: "Text / JSON", + binary: "Binary blobs", + array: "Typed arrays (floats, ints)", + mixed: "Mixed" ]) -Kino.Layout.grid([use_case, data_type], columns: 2) +Kino.render(Kino.Layout.grid([use_case, data_type], columns: 2)) ``` ```elixir @@ -278,17 +289,17 @@ IO.puts(flowchart) ## Codec Feature Matrix -| Feature | LZ4 | Snappy | Zstd | Bzip2 | Blosc2 | -|---|---|---|---|---|---| -| Speed | Very Fast | Very Fast | Fast | Slow | Medium | -| Ratio | Low | Low | High | Very High | High (arrays) | -| Configurable | Level 1–16 | No | Level 1–22 | Block 1–9 | Many options | -| Streaming | No | No | Yes | No | Yes | -| Best For | Hot paths | Short data | General | Archival | Arrays | -| Shuffle | — | — | — | — | Byte/Bit | -| Multi-thread | No | No | No | No | Yes | +| Feature | LZ4 | Snappy | Zstd | Bzip2 | Blosc2 | +| ------------ | ------------ | ---------- | ------------ | ----------- | ------------- | +| Speed | Very Fast | Very Fast | Fast | Slow | Medium | +| Ratio | Low | Low | High | Very High | High (arrays) | +| Configurable | Level 1–16 | No | Level 1–22 | Block 1–9 | Many options | +| Streaming | No | No | Yes | No | Yes | +| Best For | Hot paths | Short data | General | Archival | Arrays | +| Shuffle | — | — | — | — | Byte/Bit | +| Multi-thread | No | No | No | No | Yes | ## Next Steps -- **[Building Storage Systems](04_building_storage_systems.livemd)** — production patterns and error handling -- **[Zarr-Style Workloads](05_zarr_style_workloads.livemd)** — scientific data with Blosc2 \ No newline at end of file +* **[Building Storage Systems](04_building_storage_systems.livemd)** — production patterns and error handling +* **[Zarr-Style Workloads](05_zarr_style_workloads.livemd)** — scientific data with Blosc2 diff --git a/livebooks/04_building_storage_systems.livemd b/livebooks/04_building_storage_systems.livemd index 0893272..72974f5 100644 --- a/livebooks/04_building_storage_systems.livemd +++ b/livebooks/04_building_storage_systems.livemd @@ -1,10 +1,26 @@ # Building Storage Systems with ExCodecs ```elixir -Mix.install([ - {:ex_codecs, path: Path.join(__DIR__, "..")}, - {:kino, "~> 0.14"} -]) +local_path = Path.join(__DIR__, "../mix.exs") + +ex_codecs_dep = + if File.exists?(local_path) do + [{:ex_codecs, path: Path.join(__DIR__, "..")}, {:rustler, "~> 0.36"}] + else + [{:ex_codecs, "~> 0.1"}] + end + +config = + if File.exists?(local_path) do + [rustler_precompiled: [force_build: [ex_codecs: true]]] + else + [] + end + +Mix.install( + ex_codecs_dep ++ [{:jason, "~> 1.4"}, {:kino, "~> 0.14"}, {:kino_vega_lite, "~> 0.1.13"}], + config: config +) ``` ## Practical Usage Patterns @@ -12,13 +28,9 @@ Mix.install([ ### The Basic Pattern ```elixir -# Every compression operation in ExCodecs follows this pattern: data = "Important data to store" -# Encode (compress) {:ok, compressed} = ExCodecs.encode(:zstd, data) - -# Decode (decompress) — always matches the codec used to encode {:ok, recovered} = ExCodecs.decode(:zstd, compressed) IO.puts("Round-trip OK: #{recovered == data}") @@ -27,7 +39,6 @@ IO.puts("Round-trip OK: #{recovered == data}") ### Choosing a Codec at Runtime ```elixir -# Let the workload decide defmodule StorageConfig do def codec_for_content_type("application/json"), do: :zstd def codec_for_content_type("application/octet-stream"), do: :lz4 @@ -47,11 +58,7 @@ IO.puts("Supported: #{ExCodecs.supports?(codec)}") defmodule CompressedStore do @moduledoc """ A simple key-value store with transparent compression. - - Stores values compressed and decompresses on read. - Codec and options are configurable per-store instance. """ - use Agent defstruct [:codec, :opts, :table] @@ -62,11 +69,7 @@ defmodule CompressedStore do name = Keyword.get(opts, :name, __MODULE__) Agent.start_link(fn -> - %__MODULE__{ - codec: codec, - opts: codec_opts, - table: %{} - } + %__MODULE__{codec: codec, opts: codec_opts, table: %{}} end, name: name) end @@ -79,8 +82,7 @@ defmodule CompressedStore do original_size: byte_size(value), codec: state.codec } - {{:ok, byte_size(compressed)}, Map.put(state.table, key, entry)} - + {{:ok, byte_size(compressed)}, %{state | table: Map.put(state.table, key, entry)}} {:error, err} -> {{:error, err}, state} end @@ -101,7 +103,6 @@ defmodule CompressedStore do entries = Map.values(state.table) total_original = Enum.sum(Enum.map(entries, & &1.original_size)) total_compressed = Enum.sum(Enum.map(entries, &byte_size(&1.compressed))) - %{ codec: state.codec, entry_count: length(entries), @@ -110,11 +111,11 @@ defmodule CompressedStore do savings_pct: Float.round(100 * (1 - total_compressed / max(total_original, 1)), 1) } end) - end) + end def delete(store \\ __MODULE__, key) do Agent.get_and_update(store, fn state -> - {Map.get(state.table, key), Map.delete(state.table, key)} + {Map.get(state.table, key), %{state | table: Map.delete(state.table, key)}} end) end end @@ -123,24 +124,15 @@ end ### Using the Compressed Store ```elixir -{:ok, _pid} = CompressedStore.start_link(codec: :zstd, level: 3) + {:ok, _pid} = CompressedStore.start_link(codec: :zstd, level: 3) -# Store various data types CompressedStore.put(:doc1, String.duplicate("Hello, World! ", 1000)) CompressedStore.put(:doc2, :crypto.strong_rand_bytes(4096) |> Base.encode64()) -CompressedStore.put(:doc3, Jason.encode!(%{for i <- 1..100, do: %{id: i, value: :rand.uniform(1000)}})) - -# Note: building JSON manually since we don't require Jason -json_data = Enum.map_join(1..100, ",", fn i -> - "{\"id\":#{i},\"value\":#{:rand.uniform(1000)}}" -end) -CompressedStore.put(:doc3, "[#{json_data}]") +CompressedStore.put(:doc3, Jason.encode!(for i <- 1..100, do: %{id: i, value: :rand.uniform(1000)})) -# Retrieve and verify {:ok, doc1} = CompressedStore.get(:doc1) IO.puts("Retrieved doc1 (#{byte_size(doc1)} bytes)") -# Check storage efficiency info = CompressedStore.info() IO.puts("\nStorage Summary:") IO.puts(" Codec: #{inspect(info.codec)}") @@ -156,11 +148,7 @@ IO.puts(" Savings: #{info.savings_pct}%") defmodule MultiCodecStore do @moduledoc """ A store that selects the best codec per value. - - Tries multiple codecs and picks the one with the best ratio. - Stores the chosen codec alongside the compressed data. """ - use Agent @codecs [:lz4, :snappy, :zstd, :bzip2] @@ -175,7 +163,7 @@ defmodule MultiCodecStore do results = for codec <- @codecs do {:ok, enc} = ExCodecs.encode(codec, value) {codec, enc, byte_size(enc)} - end) + end {best_codec, best_enc, best_size} = Enum.min_by(results, fn {_, _, size} -> size end) @@ -187,7 +175,7 @@ defmodule MultiCodecStore do compressed_size: best_size } - {:ok, entry_info} = {entry, Map.put(state, key, entry)} + {{:ok, entry}, Map.put(state, key, entry)} end) end @@ -223,8 +211,8 @@ end datasets = %{ repetitive: String.duplicate("abcabcabc", 5000), - semi_random: for(_ <- 1..10000, into: <<>>, do: <<:rand.uniform(255)>>), - floats: for(i <- 1..4096, into: <<>>, do: <>) + semi_random: (for _ <- 1..10000, into: <<>>, do: <<:rand.uniform(255)>>), + floats: (for i <- 1..4096, into: <<>>, do: <>) } for {name, data} <- datasets do @@ -236,8 +224,7 @@ end IO.puts("\nCodec distribution:") for {codec, count, orig, comp} <- MultiCodecStore.stats() do - IO.puts(" #{inspect(codec)}: #{count} entries, " <> - "#{orig} -> #{comp} bytes") + IO.puts(" #{inspect(codec)}: #{count} entries, #{orig} -> #{comp} bytes") end ``` @@ -247,24 +234,17 @@ end defmodule ChunkedCompressor do @moduledoc """ Compresses large data in chunks to limit memory usage. - - Each chunk is compressed independently, enabling: - - Parallel compression - - Random access on decompression - - Memory-bounded processing """ - - @default_chunk_size 64 * 1024 # 64 KB + @default_chunk_size 64 * 1024 def compress(data, codec \\ :zstd, opts \\ [], chunk_size \\ @default_chunk_size) do - chunks = chunk_binary(data, chunk_size) + chunks = for <>, do: chunk compressed_chunks = Enum.map(chunks, fn chunk -> {:ok, enc} = ExCodecs.encode(codec, chunk, opts) enc end) - # Header: number of chunks, then offsets offsets = compressed_chunks |> Enum.scan(0, fn chunk, acc -> acc + byte_size(chunk) end) |> Enum.drop(-1) @@ -290,7 +270,7 @@ defmodule ChunkedCompressor do offsets_list = offsets_list ++ [byte_size(chunks_binary)] chunks = Enum.chunk_every(offsets_list, 2, 1, :discard) - |> Enum.map(fn {start_off, end_off} -> + |> Enum.map(fn [start_off, end_off] -> size = end_off - start_off <<_::binary-size(start_off), chunk::binary-size(size), _::binary>> = chunks_binary chunk @@ -303,15 +283,10 @@ defmodule ChunkedCompressor do {:ok, IO.iodata_to_binary(decompressed)} end - - defp chunk_binary(data, chunk_size) do - for <>, do: chunk - end end ``` ```elixir -# Demo: chunked compression large_data = String.duplicate("This is a chunk of data that repeats. ", 10000) {:ok, compressed} = ChunkedCompressor.compress(large_data, :zstd) @@ -326,60 +301,57 @@ IO.puts("Round-trip: #{recovered == large_data}") ## Error Handling Best Practices ```elixir -# Pattern 1: Always handle both success and error -case ExCodecs.encode(:zstd, my_data) do +case ExCodecs.encode(:zstd, "some data") do {:ok, compressed} -> - :ok = write_to_disk(compressed) {:ok, compressed} {:error, %ExCodecs.Error{reason: :unsupported_codec}} -> - # Codec not available — fall back or fail gracefully {:error, :codec_missing} {:error, %ExCodecs.Error{reason: :invalid_data} = err} -> - # Data is not binary — log and reject IO.puts("Invalid data: #{err.message}") {:error, :bad_input} {:error, %ExCodecs.Error{reason: :compression_failed} = err} -> - # Compression itself failed — possibly corrupted or too large IO.puts("Compression failed: #{err.message}") {:error, :compress_failed} end ``` ```elixir -# Pattern 2: Validate before compress -def safe_compress(codec, data, opts \\ []) do - with {:ok, true} <- {:check_codec, ExCodecs.supports?(codec)}, - {:ok, true} <- {:check_binary, is_binary(data)}, - {:ok, compressed} <- ExCodecs.encode(codec, data, opts), - {:ok, ^data} <- ExCodecs.decode(codec, compressed) do - {:ok, compressed} - else - {:check_codec, false} -> {:error, :unsupported_codec} - {:check_binary, false} -> {:error, :not_binary} - {:error, err} -> {:error, err} +defmodule SafeCompress do + def call(codec, data, opts \\ []) do + with {:check_codec, true} <- {:check_codec, ExCodecs.supports?(codec)}, + {:check_binary, true} <- {:check_binary, is_binary(data)}, + {:ok, compressed} <- ExCodecs.encode(codec, data, opts), + {:ok, ^data} <- ExCodecs.decode(codec, compressed) do + {:ok, compressed} + else + {:check_codec, false} -> {:error, :unsupported_codec} + {:check_binary, false} -> {:error, :not_binary} + {:error, err} -> {:error, err} + end end end -{:ok, result} = safe_compress(:zstd, "verify round-trip") +{:ok, result} = SafeCompress.call(:zstd, "verify round-trip") IO.puts("Safe compress OK: #{is_binary(result)}") ``` ```elixir -# Pattern 3: Fallback codec chain -def compress_with_fallback(data, codecs \\ [:zstd, :lz4, :snappy]) do - codecs - |> Enum.reduce_while({:error, :no_codec_available}, fn codec, _acc -> - case ExCodecs.encode(codec, data) do - {:ok, compressed} -> {:halt, {:ok, {codec, compressed}}} - {:error, _} -> {:cont, {:error, :codec_failed}} - end - end) +defmodule CompressWithFallback do + def call(data, codecs \\ [:zstd, :lz4, :snappy]) do + codecs + |> Enum.reduce_while({:error, :no_codec_available}, fn codec, _acc -> + case ExCodecs.encode(codec, data) do + {:ok, compressed} -> {:halt, {:ok, {codec, compressed}}} + {:error, _} -> {:cont, {:error, :codec_failed}} + end + end) + end end -{:ok, {used_codec, compressed}} = compress_with_fallback("fallback test data") +{:ok, {used_codec, compressed}} = CompressWithFallback.call("fallback test data") IO.puts("Used codec: #{inspect(used_codec)}") IO.puts("Compressed: #{byte_size(compressed)} bytes") ``` @@ -390,11 +362,7 @@ IO.puts("Compressed: #{byte_size(compressed)} bytes") defmodule CompressedETS do @moduledoc """ ETS-backed storage with transparent compression. - - Stores compressed values in ETS for fast in-memory access - with reduced memory footprint. """ - def new(name \\ :compressed_store) do :ets.new(name, [:set, :public, :named_table]) name @@ -419,7 +387,6 @@ defmodule CompressedETS do entries = :ets.tab2list(table) total_orig = Enum.sum(Enum.map(entries, fn {_, _, _, orig} -> orig end)) total_comp = Enum.sum(Enum.map(entries, fn {_, comp, _, _} -> byte_size(comp) end)) - %{ entries: length(entries), original_bytes: total_orig, @@ -445,7 +412,6 @@ IO.puts(" Original: #{stats.original_bytes} bytes") IO.puts(" Compressed: #{stats.compressed_bytes} bytes") IO.puts(" Savings: #{stats.savings_pct}%") -# Verify retrieval {:ok, val} = CompressedETS.lookup(table, "key_1") IO.puts("\nRetrieved #{byte_size(val)} bytes successfully") @@ -458,23 +424,17 @@ IO.puts("\nRetrieved #{byte_size(val)} bytes successfully") defmodule CompressedFileIO do @moduledoc """ File I/O with transparent compression. - - Writes compressed data to disk with a codec header for - self-describing files. """ - @magic "EXCD" def write(path, data, codec \\ :zstd, opts \\ []) do {:ok, compressed} = ExCodecs.encode(codec, data, opts) - header = << @magic::binary, codec_atom_to_bin(codec)::binary, byte_size(data)::unsigned-big-64, byte_size(compressed)::unsigned-big-64 >> - File.write(path, header <> compressed) end @@ -483,10 +443,8 @@ defmodule CompressedFileIO do {:ok, <<@magic::binary, codec_bin::binary-size(8), _orig_size::unsigned-big-64, _comp_size::unsigned-big-64, compressed::binary>>} -> codec = bin_to_codec_atom(codec_bin) ExCodecs.decode(codec, compressed) - {:ok, _} -> {:error, :invalid_format} - {:error, reason} -> {:error, reason} end @@ -503,7 +461,6 @@ end ``` ```elixir -# Demo: write and read compressed file test_data = String.duplicate("Compressed file storage example. ", 2000) path = Path.join(System.tmp_dir!(), "ex_codecs_demo.dat") @@ -526,4 +483,4 @@ File.rm(path) 5. **Chunk large files** — enables random access, parallel processing, and bounded memory 6. **Try multiple codecs** for mixed workloads — different data compresses best with different algorithms -Next: **[Zarr-Style Workloads](05_zarr_style_workloads.livemd)** \ No newline at end of file +Next: **[Zarr-Style Workloads](05_zarr_style_workloads.livemd)** diff --git a/livebooks/05_zarr_style_workloads.livemd b/livebooks/05_zarr_style_workloads.livemd index 90da04a..3c13813 100644 --- a/livebooks/05_zarr_style_workloads.livemd +++ b/livebooks/05_zarr_style_workloads.livemd @@ -1,9 +1,26 @@ # Zarr-Style Workloads ```elixir -Mix.install([ - {:ex_codecs, path: ".."} -]) +local_path = Path.join(__DIR__, "../mix.exs") + +ex_codecs_dep = + if File.exists?(local_path) do + [{:ex_codecs, path: Path.join(__DIR__, "..")}, {:rustler, "~> 0.36"}] + else + [{:ex_codecs, "~> 0.1"}] + end + +config = + if File.exists?(local_path) do + [rustler_precompiled: [force_build: [ex_codecs: true]]] + else + [] + end + +Mix.install( + ex_codecs_dep ++ [{:jason, "~> 1.4"}, {:kino, "~> 0.14"}, {:kino_vega_lite, "~> 0.1.13"}], + config: config +) ``` ## Scientific Dataset Compression @@ -21,9 +38,9 @@ alias ExCodecs.Compression Scientific datasets are typically stored as multi-dimensional arrays with strong regularity: -- Numeric types (integers, floats) with fixed element sizes -- Regular dimension strides -- Column-oriented or row-oriented access patterns +* Numeric types (integers, floats) with fixed element sizes +* Regular dimension strides +* Column-oriented or row-oriented access patterns Blosc2 was specifically designed for this workload: @@ -143,4 +160,4 @@ end 2. **Zstd** is the best general-purpose codec 3. **LZ4 and Snappy** are fastest for real-time compression 4. **Bzip2** gives the best ratios for archival -5. **Chunk-based compression** enables random access to large datasets \ No newline at end of file +5. **Chunk-based compression** enables random access to large datasets diff --git a/mix.lock b/mix.lock index 8bc87cb..89def42 100644 --- a/mix.lock +++ b/mix.lock @@ -1,23 +1,65 @@ %{ + "aws_credentials": {:hex, :aws_credentials, "0.3.2", "ba2ccee4ec6dcb5426cf71830b7afd73795b1f19655f401d4401015b468fec6f", [:rebar3], [{:eini, "~> 2.2.4", [hex: :eini_beam, repo: "hexpm", optional: false]}, {:iso8601, "~> 1.3.4", [hex: :iso8601, repo: "hexpm", optional: false]}, {:jsx, "~> 3.1.0", [hex: :jsx, repo: "hexpm", optional: false]}], "hexpm", "2e748626a935a7a544647fb79d7054f38db8bf378978542c962ccbeab387387b"}, + "bandit": {:hex, :bandit, "1.11.1", "1eb33123cc3c17ae0c3447874eb83399ee530f960c39711ed240342fbd4865fa", [:mix], [{:hpax, "~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}, {:plug, "~> 1.18", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:thousand_island, "~> 1.0", [hex: :thousand_island, repo: "hexpm", optional: false]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "d4401016df9abbc6dcd325c0b78b2b193e7c7c96bb68f31e576112be025d84a5"}, "benchee": {:hex, :benchee, "1.5.1", "b95cbc36c4b98969a5c592a246e171041eb683c56bad1cb4f49a3b081ba66087", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}, {:statistex, "~> 1.1", [hex: :statistex, repo: "hexpm", optional: false]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "a539301f8dfd4efc5c5123bfb9d47ebde20092a863a5b5b16c2a60d2243dfce7"}, "benchee_html": {:hex, :benchee_html, "1.0.1", "1e247c0886c3fdb0d3f4b184b653a8d6fb96e4ad0d0389267fe4f36968772e24", [:mix], [{:benchee, ">= 0.99.0 and < 2.0.0", [hex: :benchee, repo: "hexpm", optional: false]}, {:benchee_json, "~> 1.0", [hex: :benchee_json, repo: "hexpm", optional: false]}], "hexpm", "b00a181af7152431901e08f3fc9f7197ed43ff50421a8347b0c80bf45d5b3fef"}, "benchee_json": {:hex, :benchee_json, "1.0.0", "cc661f4454d5995c08fe10dd1f2f72f229c8f0fb1c96f6b327a8c8fc96a91fe5", [:mix], [{:benchee, ">= 0.99.0 and < 2.0.0", [hex: :benchee, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "da05d813f9123505f870344d68fb7c86a4f0f9074df7d7b7e2bb011a63ec231c"}, + "bun": {:hex, :bun, "1.6.0", "ab49eb8b956f7492e8d6c5a43c0b96e94074aa0ace698a4be7e9b3b3b281d1b3", [:mix], [{:nimble_pool, "~> 1.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}], "hexpm", "6f17b0544b452077da87d14192042744826dabb089d968b8caf8b298968e7a17"}, "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"}, "credo": {:hex, :credo, "1.7.19", "cc52129665fc7c15143d47838fda0f9cd6dac9ceced7bf4da6f85fcbfe64b12a", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "2d8bc95d5a7bb99dd2613621d4f08c6a3575c3fd4b62e6a2b48a100352a557b8"}, + "decimal": {:hex, :decimal, "3.1.1", "430d87b04011ce6cbd4fd205be758311a81f87d552d40904abd00f015935b1d0", [:mix], [], "hexpm", "c5f25f2ced74a0587d03e6023f595db8e924c9d3922c8c8ffd9edfc4498cf1f6"}, "deep_merge": {:hex, :deep_merge, "1.0.2", "476aa7ea61c54de96220051b998d893869069094da65b96101aebf79416f8a1e", [:mix], [], "hexpm", "737a53cdc9758fedbb608bdc213969e65729466c4ef3cd8e8726d0335dff116c"}, "dialyxir": {:hex, :dialyxir, "1.4.7", "dda948fcee52962e4b6c5b4b16b2d8fa7d50d8645bbae8b8685c3f9ecb7f5f4d", [:mix], [{:erlex, ">= 0.2.8", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "b34527202e6eb8cee198efec110996c25c5898f43a4094df157f8d28f27d9efe"}, + "dns_cluster": {:hex, :dns_cluster, "0.1.3", "0bc20a2c88ed6cc494f2964075c359f8c2d00e1bf25518a6a6c7fd277c9b0c66", [:mix], [], "hexpm", "46cb7c4a1b3e52c7ad4cbe33ca5079fbde4840dedeafca2baf77996c2da1bc33"}, "earmark_parser": {:hex, :earmark_parser, "1.4.44", "f20830dd6b5c77afe2b063777ddbbff09f9759396500cdbe7523efd58d7a339c", [:mix], [], "hexpm", "4778ac752b4701a5599215f7030989c989ffdc4f6df457c5f36938cc2d2a2750"}, + "ecto": {:hex, :ecto, "3.13.6", "352135b474f91d1ab99a1b502171d207e9db60421c9e3d0ecab4c7ab96b24d14", [:mix], [{:decimal, "~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "8afa059bc16cd2c94739ec0a11e3e5df69d828125119109bef35f20a21a76af2"}, + "eini": {:hex, :eini_beam, "2.2.5", "28fa0a4eb7ff885cc388877b73fbd56fcca3b2eae4d81e47c47f317b900da1da", [:rebar3], [], "hexpm", "511e9207649f3becb5d945f1813615987899cf78fa130f92be07193a6a74ecb8"}, "erlex": {:hex, :erlex, "0.2.9", "7debbbaa9f4f368b8cd648983e0f1d7963028508e9c59e9d4ed504e94ef52a55", [:mix], [], "hexpm", "8cfffc0ec7159e6d73de2ab28a588064de80f88b2798d5cbe4482cbbc200178b"}, "ex_doc": {:hex, :ex_doc, "0.40.3", "4a972ffe64bc07dc605af487e98fc19b72a4185f55ca031b94c0552d6071c1d9", [:mix], [{:earmark_parser, "~> 1.4.44", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "2756e357742fecd9749b489b85d67c9ce99c465f2e75728d9e6dc8d704b973de"}, "excoveralls": {:hex, :excoveralls, "0.18.5", "e229d0a65982613332ec30f07940038fe451a2e5b29bce2a5022165f0c9b157e", [:mix], [{:castore, "~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "523fe8a15603f86d64852aab2abe8ddbd78e68579c8525ae765facc5eae01562"}, "file_system": {:hex, :file_system, "1.1.1", "31864f4685b0148f25bd3fbef2b1228457c0c89024ad67f7a81a3ffbc0bbad3a", [:mix], [], "hexpm", "7a15ff97dfe526aeefb090a7a9d3d03aa907e100e262a0f8f7746b78f8f87a5d"}, + "finch": {:hex, :finch, "0.22.0", "5c48fa6f9706a78eb9036cacb67b8b996b4e66d111c543f4c29bb0f879a6806b", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.8", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 1.1", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "b94e83c47780fc6813f746a1f1a34ee65cda42da4c5ea26a68f0acc4498e23dc"}, + "hpax": {:hex, :hpax, "1.0.3", "ed67ef51ad4df91e75cc6a1494f851850c0bd98ebc0be6e81b026e765ee535aa", [:mix], [], "hexpm", "8eab6e1cfa8d5918c2ce4ba43588e894af35dbd8e91e6e55c817bca5847df34a"}, + "iso8601": {:hex, :iso8601, "1.3.4", "7b1f095f86f6cf65e1e5a77872e8e8bf69bd58d4c3a415b3f77d9cc9423ecbb9", [:rebar3], [], "hexpm", "a334469c07f1c219326bc891a95f5eec8eb12dd8071a3fff56a7843cb20fae34"}, "jason": {:hex, :jason, "1.4.5", "2e3a008590b0b8d7388c20293e9dcc9cf3e5d642fd2a114e4cbbb52e595d940a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "b0c823996102bcd0239b3c2444eb00409b72f6a140c1950bc8b457d836b30684"}, + "jose": {:hex, :jose, "1.11.12", "06e62b467b61d3726cbc19e9b5489f7549c37993de846dfb3ee8259f9ed208b3", [:mix, :rebar3], [], "hexpm", "31e92b653e9210b696765cdd885437457de1add2a9011d92f8cf63e4641bab7b"}, + "jsx": {:hex, :jsx, "3.1.0", "d12516baa0bb23a59bb35dccaf02a1bd08243fcbb9efe24f2d9d056ccff71268", [:rebar3], [], "hexpm", "0c5cc8fdc11b53cc25cf65ac6705ad39e54ecc56d1c22e4adb8f5a53fb9427f3"}, + "kubereq": {:hex, :kubereq, "0.4.2", "a42cb9ceb86605c2ac63fb7921d5481161c25e266f6c82bd9c0f759841db6dd0", [:mix], [{:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: false]}, {:mint_web_socket, "~> 1.0", [hex: :mint_web_socket, repo: "hexpm", optional: false]}, {:pluggable, "~> 1.0", [hex: :pluggable, repo: "hexpm", optional: false]}, {:req, "~> 0.5.0", [hex: :req, repo: "hexpm", optional: false]}, {:yaml_elixir, "~> 2.0", [hex: :yaml_elixir, repo: "hexpm", optional: false]}], "hexpm", "c52a4a7c8911e10e739291457a2367f38397eada3e904ea6731fc0c34ee99d3e"}, + "livebook": {:hex, :livebook, "0.19.8", "4d4ff37e873a163660bafc7b024d17e3d035de127cf6e00fe39c013e41aab3bf", [:mix], [{:aws_credentials, "0.3.2", [hex: :aws_credentials, repo: "hexpm", optional: false]}, {:bandit, "1.11.1", [hex: :bandit, repo: "hexpm", optional: false]}, {:bun, "1.6.0", [hex: :bun, repo: "hexpm", optional: false]}, {:dns_cluster, "0.1.3", [hex: :dns_cluster, repo: "hexpm", optional: false]}, {:earmark_parser, "1.4.44", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:ecto, "3.13.6", [hex: :ecto, repo: "hexpm", optional: false]}, {:kubereq, "0.4.2", [hex: :kubereq, repo: "hexpm", optional: false]}, {:logger_json, "6.2.1", [hex: :logger_json, repo: "hexpm", optional: false]}, {:mint_web_socket, "1.0.4", [hex: :mint_web_socket, repo: "hexpm", optional: false]}, {:nimble_zta, "0.1.0", [hex: :nimble_zta, repo: "hexpm", optional: false]}, {:phoenix, "1.8.7", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_ecto, "4.6.3", [hex: :phoenix_ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "4.2.1", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:phoenix_live_dashboard, "0.8.6", [hex: :phoenix_live_dashboard, repo: "hexpm", optional: false]}, {:phoenix_live_view, "1.1.24", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}, {:plug, "1.19.2", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_crypto, "2.1.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:protobuf, "0.13.0", [hex: :protobuf, repo: "hexpm", optional: false]}, {:req, "0.5.8", [hex: :req, repo: "hexpm", optional: false]}, {:telemetry_metrics, "1.1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}, {:telemetry_poller, "1.1.0", [hex: :telemetry_poller, repo: "hexpm", optional: false]}, {:yaml_elixir, "2.11.0", [hex: :yaml_elixir, repo: "hexpm", optional: false]}], "hexpm", "881d10b67af069e8f272daf65698c33abbefd5b3195a8c0ea248bde5bac86594"}, + "livebook_test": {:hex, :livebook_test, "0.1.0", "5f76f83c76abecc64827771bfdd252dc07729159d494703e6db79653d07c9166", [:mix], [{:livebook, "~> 0.19.0", [hex: :livebook, repo: "hexpm", optional: false]}], "hexpm", "a270ff36ae0bdda761f9503453e61f65b169f222771d73901104977e4c46f841"}, + "logger_json": {:hex, :logger_json, "6.2.1", "a1db30e1164e6057f2328a1e4d6b632b9583c015574fdf6c38cf73721128edcb", [:mix], [{:decimal, ">= 0.0.0", [hex: :decimal, repo: "hexpm", optional: true]}, {:ecto, "~> 3.11", [hex: :ecto, repo: "hexpm", optional: true]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:plug, "~> 1.15", [hex: :plug, repo: "hexpm", optional: true]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "34acd0bfd419d5fcf08c4108a8a4b59b695fcc60409dc1dd1a868b70c42e1d1f"}, "makeup": {:hex, :makeup, "1.2.1", "e90ac1c65589ef354378def3ba19d401e739ee7ee06fb47f94c687016e3713d1", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "d36484867b0bae0fea568d10131197a4c2e47056a6fbe84922bf6ba71c8d17ce"}, "makeup_elixir": {:hex, :makeup_elixir, "1.0.1", "e928a4f984e795e41e3abd27bfc09f51db16ab8ba1aebdba2b3a575437efafc2", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "7284900d412a3e5cfd97fdaed4f5ed389b8f2b4cb49efc0eb3bd10e2febf9507"}, "makeup_erlang": {:hex, :makeup_erlang, "1.1.0", "835f7e60792e08824cda445639555d7bf1bbbddb1b60b306e33cb6f6db24dc74", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "1cd6780fb1dd1a03979abaed0fe82712b0625118fd5257d3ebbf73f960c73c3c"}, + "mime": {:hex, :mime, "2.0.7", "b8d739037be7cd402aee1ba0306edfdef982687ee7e9859bee6198c1e7e2f128", [:mix], [], "hexpm", "6171188e399ee16023ffc5b76ce445eb6d9672e2e241d2df6050f3c771e80ccd"}, + "mint": {:hex, :mint, "1.9.0", "d6f534c2a3e98b2a8cc749b4796eb77e9e3af79a76f96e4c74035a827de0d318", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "007154c7d8c43916aed3c93afd1f11aebbaa9c5ff4b7ba55ebe0d17ee0296042"}, + "mint_web_socket": {:hex, :mint_web_socket, "1.0.4", "0b539116dbb3d3f861cdf5e15e269a933cb501c113a14db7001a3157d96ffafd", [:mix], [{:mint, ">= 1.4.1 and < 2.0.0-0", [hex: :mint, repo: "hexpm", optional: false]}], "hexpm", "027d4c5529c45a4ba0ce27a01c0f35f284a5468519c045ca15f43decb360a991"}, + "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"}, "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"}, + "nimble_pool": {:hex, :nimble_pool, "1.1.0", "bf9c29fbdcba3564a8b800d1eeb5a3c58f36e1e11d7b7fb2e084a643f645f06b", [:mix], [], "hexpm", "af2e4e6b34197db81f7aad230c1118eac993acc0dae6bc83bac0126d4ae0813a"}, + "nimble_zta": {:hex, :nimble_zta, "0.1.0", "7652c9de1d54fab20e13ed021a60198badb541cc34f77adf99a65691ccd269f0", [:mix], [{:jose, "~> 1.11", [hex: :jose, repo: "hexpm", optional: false]}, {:plug, "~> 1.18", [hex: :plug, repo: "hexpm", optional: false]}, {:req, "~> 0.5", [hex: :req, repo: "hexpm", optional: false]}], "hexpm", "63e985edd570e91df6dbe0dcf0112095a212771d18810b9186c9bb6bff5a54b0"}, + "phoenix": {:hex, :phoenix, "1.8.7", "d8d755b4ff4b449f610223dd706b4ae64155cb720d3dc09c706c079ecea189e4", [:mix], [{:bandit, "~> 1.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.7", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5.3", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "47352f72d6ab31009ef77516b1b3a14745be97b54061fd458031b9d8294869d5"}, + "phoenix_ecto": {:hex, :phoenix_ecto, "4.6.3", "f686701b0499a07f2e3b122d84d52ff8a31f5def386e03706c916f6feddf69ef", [:mix], [{:ecto, "~> 3.5", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.1", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "909502956916a657a197f94cc1206d9a65247538de8a5e186f7537c895d95764"}, + "phoenix_html": {:hex, :phoenix_html, "4.2.1", "35279e2a39140068fc03f8874408d58eef734e488fc142153f055c5454fd1c08", [:mix], [], "hexpm", "cff108100ae2715dd959ae8f2a8cef8e20b593f8dfd031c9cba92702cf23e053"}, + "phoenix_live_dashboard": {:hex, :phoenix_live_dashboard, "0.8.6", "7b1f0327f54c9eb69845fd09a77accf922f488c549a7e7b8618775eb603a62c7", [:mix], [{:ecto, "~> 3.6.2 or ~> 3.7", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_mysql_extras, "~> 0.5", [hex: :ecto_mysql_extras, repo: "hexpm", optional: true]}, {:ecto_psql_extras, "~> 0.7", [hex: :ecto_psql_extras, repo: "hexpm", optional: true]}, {:ecto_sqlite3_extras, "~> 1.1.7 or ~> 1.2.0", [hex: :ecto_sqlite3_extras, repo: "hexpm", optional: true]}, {:mime, "~> 1.6 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:phoenix_live_view, "~> 0.19 or ~> 1.0", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6 or ~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "1681ab813ec26ca6915beb3414aa138f298e17721dc6a2bde9e6eb8a62360ff6"}, + "phoenix_live_view": {:hex, :phoenix_live_view, "1.1.24", "1a000a048d5971b61a9efe29a3c4144ca955afd42224998d841c5011a5354838", [:mix], [{:igniter, ">= 0.6.16 and < 1.0.0-0", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:lazy_html, "~> 0.1.0", [hex: :lazy_html, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.6.15 or ~> 1.7.0 or ~> 1.8.0-rc", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 3.3 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.15", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "0c724e6c65f197841cac49d73be4e0f9b93a7711eaa52d2d4d1b9f859c329267"}, + "phoenix_pubsub": {:hex, :phoenix_pubsub, "2.2.0", "ff3a5616e1bed6804de7773b92cbccfc0b0f473faf1f63d7daf1206c7aeaaa6f", [:mix], [], "hexpm", "adc313a5bf7136039f63cfd9668fde73bba0765e0614cba80c06ac9460ff3e96"}, + "phoenix_template": {:hex, :phoenix_template, "1.0.4", "e2092c132f3b5e5b2d49c96695342eb36d0ed514c5b252a77048d5969330d639", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}], "hexpm", "2c0c81f0e5c6753faf5cca2f229c9709919aba34fab866d3bc05060c9c444206"}, + "plug": {:hex, :plug, "1.19.2", "e4950525b22c6789dfb38a3f95d47171ba159da3fc5a33be9643b43d5e8adb98", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "b6fce20a56af5e60fa5dfecf3f907bb98ec981be43c79a3809a499bc3d133de0"}, + "plug_crypto": {:hex, :plug_crypto, "2.1.0", "f44309c2b06d249c27c8d3f65cfe08158ade08418cf540fd4f72d4d6863abb7b", [:mix], [], "hexpm", "131216a4b030b8f8ce0f26038bc4421ae60e4bb95c5cf5395e1421437824c4fa"}, + "pluggable": {:hex, :pluggable, "1.1.0", "7eba3bc70c0caf4d9056c63c882df8862f7534f0145da7ab3a47ca73e4adb1e4", [:mix], [], "hexpm", "d12eb00ea47b21e92cd2700d6fbe3737f04b64e71b63aad1c0accde87c751637"}, + "protobuf": {:hex, :protobuf, "0.13.0", "7a9d9aeb039f68a81717eb2efd6928fdf44f03d2c0dfdcedc7b560f5f5aae93d", [:mix], [{:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "21092a223e3c6c144c1a291ab082a7ead32821ba77073b72c68515aa51fef570"}, + "req": {:hex, :req, "0.5.8", "50d8d65279d6e343a5e46980ac2a70e97136182950833a1968b371e753f6a662", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "d7fc5898a566477e174f26887821a3c5082b243885520ee4b45555f5d53f40ef"}, "rustler": {:hex, :rustler, "0.38.0", "7a8906998ff0d28e3021c0a73264abcda719bda344b2e58307c6805b0f87c9b4", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "704c03c1bf66be12b031c5a389347b91c81c5cb819a24b068b0de36fe4a5652a"}, "rustler_precompiled": {:hex, :rustler_precompiled, "0.9.0", "3a052eda09f3d2436364645cc1f13279cf95db310eb0c17b0d8f25484b233aa0", [:mix], [{:rustler, "~> 0.23", [hex: :rustler, repo: "hexpm", optional: true]}], "hexpm", "471d97315bd3bf7b64623418b3693eedd8e47de3d1cb79a0ac8f9da7d770d94c"}, "statistex": {:hex, :statistex, "1.1.1", "73612aa7f79e53c30569be065fd121e380f1cf57bc4c2da5b41be9246da18df9", [:mix], [], "hexpm", "310c4b49b34adf683de3103639006bed233ab54c08a4add65a531448e653857c"}, "stream_data": {:hex, :stream_data, "1.3.0", "bde37905530aff386dea1ddd86ecbf00e6642dc074ceffc10b7d4e41dfd6aac9", [:mix], [], "hexpm", "3cc552e286e817dca43c98044c706eec9318083a1480c52ae2688b08e2936e3c"}, + "telemetry": {:hex, :telemetry, "1.4.2", "a0cb522801dffb1c49fe6e30561badffc7b6d0e180db1300df759faa22062855", [:rebar3], [], "hexpm", "928f6495066506077862c0d1646609eed891a4326bee3126ba54b60af61febb1"}, + "telemetry_metrics": {:hex, :telemetry_metrics, "1.1.0", "5bd5f3b5637e0abea0426b947e3ce5dd304f8b3bc6617039e2b5a008adc02f8f", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "e7b79e8ddfde70adb6db8a6623d1778ec66401f366e9a8f5dd0955c56bc8ce67"}, + "telemetry_poller": {:hex, :telemetry_poller, "1.1.0", "58fa7c216257291caaf8d05678c8d01bd45f4bdbc1286838a28c4bb62ef32999", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "9eb9d9cbfd81cbd7cdd24682f8711b6e2b691289a0de6826e58452f28c103c8f"}, + "thousand_island": {:hex, :thousand_island, "1.5.0", "f50a213cac97262b6d5ebb85745aa2c00fec1413191e6e66834788d45425cecb", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "708923d40523e43cf99041ab37a0d4b0ec426ac6438fa3716ab23d919eaeb412"}, + "websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"}, + "websock_adapter": {:hex, :websock_adapter, "0.5.9", "43dc3ba6d89ef5dec5b1d0a39698436a1e856d000d84bf31a3149862b01a287f", [:mix], [{:bandit, ">= 0.6.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "5534d5c9adad3c18a0f58a9371220d75a803bf0b9a3d87e6fe072faaeed76a08"}, + "yamerl": {:hex, :yamerl, "0.10.0", "4ff81fee2f1f6a46f1700c0d880b24d193ddb74bd14ef42cb0bcf46e81ef2f8e", [:rebar3], [], "hexpm", "346adb2963f1051dc837a2364e4acf6eb7d80097c0f53cbdc3046ec8ec4b4e6e"}, + "yaml_elixir": {:hex, :yaml_elixir, "2.11.0", "9e9ccd134e861c66b84825a3542a1c22ba33f338d82c07282f4f1f52d847bd50", [:mix], [{:yamerl, "~> 0.10", [hex: :yamerl, repo: "hexpm", optional: false]}], "hexpm", "53cc28357ee7eb952344995787f4bb8cc3cecbf189652236e9b163e8ce1bc242"}, } diff --git a/native/ex_codecs_native/src/blosc2_codec.rs b/native/ex_codecs_native/src/blosc2_codec.rs index 7f17f7f..52deb90 100644 --- a/native/ex_codecs_native/src/blosc2_codec.rs +++ b/native/ex_codecs_native/src/blosc2_codec.rs @@ -20,6 +20,7 @@ const BLOSC_ZSTD: u8 = 5; const BLOSC_NOSHUFFLE: u8 = 0; const BLOSC_BYTESHUFFLE: u8 = 1; +const BLOSC_BITSHUFFLE: u8 = 2; fn byte_shuffle(data: &[u8], typesize: usize) -> Vec { if typesize <= 1 || data.len() < typesize { @@ -65,6 +66,87 @@ fn byte_unshuffle(data: &[u8], typesize: usize) -> Vec { result } +fn bit_shuffle(data: &[u8], typesize: usize) -> Vec { + if typesize == 0 || data.is_empty() { + return data.to_vec(); + } + + let n_elements = data.len() / typesize; + if n_elements == 0 { + return data.to_vec(); + } + + let n_full_blocks = n_elements / 8; + let block_bytes = n_full_blocks * 8 * typesize; + let mut result = vec![0u8; data.len()]; + let mut out_pos = 0; + + for block in 0..n_full_blocks { + let block_start = block * 8; + for byte_idx in 0..typesize { + for bit_idx in 0..8 { + let mut acc: u8 = 0; + for i in 0..8 { + let src = data[(block_start + i) * typesize + byte_idx]; + if src & (1 << (7 - bit_idx)) != 0 { + acc |= 1 << (7 - i); + } + } + result[out_pos] = acc; + out_pos += 1; + } + } + } + + if block_bytes < data.len() { + result[block_bytes..data.len()].copy_from_slice(&data[block_bytes..data.len()]); + } + + result +} + +fn bit_unshuffle(data: &[u8], typesize: usize) -> Vec { + if typesize == 0 || data.is_empty() { + return data.to_vec(); + } + + let n_elements = data.len() / typesize; + if n_elements == 0 { + return data.to_vec(); + } + + let n_full_blocks = n_elements / 8; + let block_bytes = n_full_blocks * 8 * typesize; + let mut result = vec![0u8; data.len()]; + let mut in_pos = 0; + + for block in 0..n_full_blocks { + let block_start = block * 8; + for byte_idx in 0..typesize { + let mut transposed = [0u8; 8]; + for k in 0..8 { + transposed[k] = data[in_pos]; + in_pos += 1; + } + for i in 0..8 { + let mut byte_val: u8 = 0; + for bit_idx in 0..8 { + if transposed[bit_idx] & (1 << (7 - i)) != 0 { + byte_val |= 1 << (7 - bit_idx); + } + } + result[(block_start + i) * typesize + byte_idx] = byte_val; + } + } + } + + if block_bytes < data.len() { + result[block_bytes..data.len()].copy_from_slice(&data[block_bytes..data.len()]); + } + + result +} + fn internal_compress(data: &[u8], cname: u8, clevel: u8) -> Result, ()> { match cname { BLOSC_BLOSCLZ | BLOSC_LZ4 | BLOSC_LZ4HC => Ok(lz4_flex::compress_prepend_size(data)), @@ -110,7 +192,7 @@ pub fn blosc2_compress<'a>( let clevel = clevel.clamp(0, 9) as u8; let typesize = typesize.max(1); let cname = cname.clamp(0, 5) as u8; - let shuffle = shuffle.clamp(0, 1) as u8; + let shuffle = shuffle.clamp(0, 2) as u8; if data.len() > u32::MAX as usize { return (atoms::error(), atoms::invalid_data()).encode(env); @@ -120,7 +202,7 @@ pub fn blosc2_compress<'a>( let mut header = vec![0u8; BLOSC_MIN_HEADER_LENGTH]; header[0] = BLOSC2_MAGIC; header[1] = BLOSC2_VERSION; - header[2] = 0x01; // flags: stored raw (no decompression needed) + header[2] = 0x01; header[3] = 0; header[4..8].copy_from_slice(&0u32.to_le_bytes()); header[8..12].copy_from_slice(&(BLOSC_MIN_HEADER_LENGTH as u32).to_le_bytes()); @@ -133,6 +215,7 @@ pub fn blosc2_compress<'a>( let shuffled = match shuffle { BLOSC_BYTESHUFFLE => byte_shuffle(data.as_slice(), typesize), + BLOSC_BITSHUFFLE => bit_shuffle(data.as_slice(), typesize), _ => data.as_slice().to_vec(), }; @@ -181,7 +264,7 @@ pub fn blosc2_compress<'a>( let mut output = vec![0u8; total_len]; output[0] = BLOSC2_MAGIC; output[1] = BLOSC2_VERSION; - output[2] = 0x00; // flags: 0x00 = compressed, 0x01 = stored raw + output[2] = 0x00; output[3] = 0; output[4..8].copy_from_slice(&(data.len() as u32).to_le_bytes()); output[8..12].copy_from_slice(&(total_len as u32).to_le_bytes()); @@ -232,6 +315,7 @@ pub fn blosc2_decompress<'a>(env: Env<'a>, data: Binary) -> Term<'a> { let result = match shuffle { BLOSC_BYTESHUFFLE => byte_unshuffle(&decompressed, typesize), + BLOSC_BITSHUFFLE => bit_unshuffle(&decompressed, typesize), _ => decompressed, }; diff --git a/test/ex_codecs/compression/blosc2_test.exs b/test/ex_codecs/compression/blosc2_test.exs index cad772c..740d860 100644 --- a/test/ex_codecs/compression/blosc2_test.exs +++ b/test/ex_codecs/compression/blosc2_test.exs @@ -23,7 +23,7 @@ defmodule ExCodecs.Compression.Blosc2Test do test "compresses data with different shuffle modes" do data = :crypto.strong_rand_bytes(4096) - for shuffle <- [:none, :byte] do + for shuffle <- [:none, :byte, :bit] do assert {:ok, compressed} = Blosc2.encode(data, shuffle: shuffle) @@ -49,10 +49,10 @@ defmodule ExCodecs.Compression.Blosc2Test do assert decompressed == data end - test "round-trips random data with typesize 4 and 8, both shuffle modes" do + test "round-trips random data with typesize 4 and 8, all shuffle modes" do data = :crypto.strong_rand_bytes(10_000) - for typesize <- [4, 8], shuffle <- [:none, :byte] do + for typesize <- [4, 8], shuffle <- [:none, :byte, :bit] do assert {:ok, compressed} = Blosc2.encode(data, cname: :lz4, clevel: 5, shuffle: shuffle, typesize: typesize) @@ -84,6 +84,13 @@ defmodule ExCodecs.Compression.Blosc2Test do assert decompressed == data end + test "round-trips sequential float64 data with bit shuffle" do + floats = for i <- 1..2048, into: <<>>, do: <> + assert {:ok, compressed} = Blosc2.encode(floats, cname: :zstd, shuffle: :bit, typesize: 8) + assert {:ok, decompressed} = Blosc2.decode(compressed, []) + assert decompressed == floats + end + test "clevel affects compression output size" do data = :crypto.strong_rand_bytes(10_000) {:ok, c1} = Blosc2.encode(data, cname: :zstd, clevel: 1, shuffle: :none) @@ -91,12 +98,14 @@ defmodule ExCodecs.Compression.Blosc2Test do assert byte_size(c9) <= byte_size(c1) end - test "no shuffle vs byte shuffle both round-trip" do + test "no shuffle vs byte shuffle vs bit shuffle all round-trip" do data = :crypto.strong_rand_bytes(8192) {:ok, c_none} = Blosc2.encode(data, shuffle: :none, typesize: 1) {:ok, c_byte} = Blosc2.encode(data, shuffle: :byte, typesize: 4) + {:ok, c_bit} = Blosc2.encode(data, shuffle: :bit, typesize: 4) assert {:ok, ^data} = Blosc2.decode(c_none, []) assert {:ok, ^data} = Blosc2.decode(c_byte, []) + assert {:ok, ^data} = Blosc2.decode(c_bit, []) end test "validates cname option" do @@ -114,9 +123,9 @@ defmodule ExCodecs.Compression.Blosc2Test do Blosc2.encode("data", shuffle: :invalid) end - test "rejects bit shuffle as unsupported" do - assert {:error, %ExCodecs.Error{reason: :invalid_options}} = - Blosc2.encode("data", shuffle: :bit) + test "accepts bit shuffle option" do + data = :crypto.strong_rand_bytes(1024) + assert {:ok, _} = Blosc2.encode(data, shuffle: :bit) end test "validates typesize option" do diff --git a/test/ex_codecs/compression/version_and_error_test.exs b/test/ex_codecs/compression/version_and_error_test.exs index 83d8aa2..77bc1a9 100644 --- a/test/ex_codecs/compression/version_and_error_test.exs +++ b/test/ex_codecs/compression/version_and_error_test.exs @@ -105,11 +105,10 @@ defmodule ExCodecs.Compression.CodecVersionTest do ExCodecs.encode(:blosc2, data, shuffle: :invalid) end - test "blosc2 rejects bit shuffle" do - data = :crypto.strong_rand_bytes(1024) + test "blosc2 accepts bit shuffle" do + floats = for i <- 1..2048, into: <<>>, do: <> - assert {:error, %Error{reason: :invalid_options}} = - ExCodecs.encode(:blosc2, data, shuffle: :bit) + assert {:ok, _} = ExCodecs.encode(:blosc2, floats, shuffle: :bit) end test "blosc2 validates typesize option" do