perf: read and write bit fields as integers

benches/bebits.rs

@@ -1,9 +1,29 @@
 //! Big-endian bit-packed headers: the shape used by real network / space
 //! protocols (CCSDS, IPv4, DVB-S2). Mirrors the CCSDS TM Transfer Frame
 //! primary header, 6 octets / 11 fields.
+//!
+//! Each shape is measured twice.
+//!
+//! The single-shot benches handle one struct per iteration. Consecutive
+//! iterations share no state, so the CPU overlaps them and the result is a
+//! throughput figure rather than the cost of one read.
+//!
+//! The `_xN` benches move N structs through one reader or writer, which is what
+//! a stream of frames actually does: each read depends on where the previous one
+//! left the cursor, so nothing overlaps. Every field is folded into a value the
+//! closure returns, so no read can be dropped. Divide by N for the per-struct
+//! cost; that is the number to quote.
 use criterion::{criterion_group, criterion_main, Criterion};
 use deku::prelude::*;
 use no_std_io::io::Cursor;
+use std::hint::black_box;
+
+/// Frames per sequential pass. 128 six-octet frames is 768 bytes, comfortably
+/// inside L1 so the measurement is field decoding rather than memory.
+const FRAMES: usize = 128;
+
+/// 1-bit fields per sequential pass: 1024 bits, i.e. 128 bytes.
+const BITS: usize = 1024;
 
 #[derive(Debug, PartialEq, DekuRead, DekuWrite)]
 #[deku(endian = "big")]
@@ -30,7 +50,8 @@ struct TmPrimaryHeader {
     fhp: u16,
 }
 
-/// Same 6 octets, byte-aligned: the deku fast path, for scale.
+/// Same 6 octets, byte-aligned: the deku fast path, for scale. Also the control
+/// for any change to the bit paths, which must leave this one alone.
 #[derive(Debug, PartialEq, DekuRead, DekuWrite)]
 #[deku(endian = "big")]
 struct SixBytes {
@@ -50,26 +71,130 @@ struct OneBitU64 {
     a: u64,
 }
 
+/// A frame stream whose bytes are not compile-time constants.
+fn stream() -> [u8; FRAMES * 6] {
+    let mut buf = [0u8; FRAMES * 6];
+    let mut x: u32 = 0x1234_5678;
+    for b in buf.iter_mut() {
+        x = x.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
+        *b = (x >> 24) as u8;
+    }
+    buf
+}
+
 fn bench(c: &mut Criterion) {
     let buf = [0x2Au8, 0xB5, 0x11, 0x22, 0xC7, 0xFF, 0x00, 0x99];
+    let stream = stream();
+
+    // One struct per iteration.
     c.bench_function("be_tm_primary_header_11_fields", |b| {
         b.iter(|| {
-            let mut r = Reader::new(Cursor::new(&buf));
+            let mut r = Reader::new(Cursor::new(black_box(&buf)));
             TmPrimaryHeader::from_reader_with_ctx(&mut r, ()).unwrap()
         })
     });
     c.bench_function("be_six_bytes_aligned", |b| {
         b.iter(|| {
-            let mut r = Reader::new(Cursor::new(&buf));
+            let mut r = Reader::new(Cursor::new(black_box(&buf)));
             SixBytes::from_reader_with_ctx(&mut r, ()).unwrap()
         })
     });
     c.bench_function("be_one_bit_in_u64", |b| {
         b.iter(|| {
-            let mut r = Reader::new(Cursor::new(&buf));
+            let mut r = Reader::new(Cursor::new(black_box(&buf)));
             OneBitU64::from_reader_with_ctx(&mut r, ()).unwrap()
         })
     });
+
+    // A stream of frames through one reader.
+    c.bench_function("be_tm_primary_header_x128", |b| {
+        b.iter(|| {
+            let mut r = Reader::new(Cursor::new(black_box(&stream)));
+            let mut acc: u64 = 0;
+            for _ in 0..FRAMES {
+                let h = TmPrimaryHeader::from_reader_with_ctx(&mut r, ()).unwrap();
+                acc ^= u64::from(h.scid)
+                    ^ u64::from(h.fhp)
+                    ^ u64::from(h.mcfc)
+                    ^ u64::from(h.vcid)
+                    ^ u64::from(h.tfvn)
+                    ^ u64::from(h.sli);
+            }
+            acc
+        })
+    });
+    c.bench_function("be_six_bytes_aligned_x128", |b| {
+        b.iter(|| {
+            let mut r = Reader::new(Cursor::new(black_box(&stream)));
+            let mut acc: u64 = 0;
+            for _ in 0..FRAMES {
+                let s = SixBytes::from_reader_with_ctx(&mut r, ()).unwrap();
+                acc ^= u64::from(s.a) ^ u64::from(s.f);
+            }
+            acc
+        })
+    });
+    c.bench_function("be_one_bit_in_u64_x1024", |b| {
+        b.iter(|| {
+            let mut r = Reader::new(Cursor::new(black_box(&stream)));
+            let mut acc: u64 = 0;
+            for _ in 0..BITS {
+                acc ^= OneBitU64::from_reader_with_ctx(&mut r, ()).unwrap().a;
+            }
+            acc
+        })
+    });
+
+    // Write side of the same shapes. Into a reused stack buffer, so the
+    // measurement is the field writes rather than an allocation.
+    let mut r = Reader::new(Cursor::new(&buf));
+    let header = TmPrimaryHeader::from_reader_with_ctx(&mut r, ()).unwrap();
+    let six = SixBytes {
+        a: 1,
+        b: 2,
+        c: 3,
+        d: 4,
+        e: 5,
+        f: 6,
+    };
+    c.bench_function("be_write_tm_primary_header_11_fields", |b| {
+        let mut out = [0u8; 16];
+        b.iter(|| {
+            let mut w = Writer::new(Cursor::new(out.as_mut_slice()));
+            black_box(&header).to_writer(&mut w, ()).unwrap();
+            w.finalize().unwrap();
+        })
+    });
+    c.bench_function("be_write_six_bytes_aligned", |b| {
+        let mut out = [0u8; 16];
+        b.iter(|| {
+            let mut w = Writer::new(Cursor::new(out.as_mut_slice()));
+            black_box(&six).to_writer(&mut w, ()).unwrap();
+            w.finalize().unwrap();
+        })
+    });
+
+    // A stream of frames through one writer.
+    c.bench_function("be_write_tm_primary_header_x128", |b| {
+        let mut out = [0u8; FRAMES * 6];
+        b.iter(|| {
+            let mut w = Writer::new(Cursor::new(out.as_mut_slice()));
+            for _ in 0..FRAMES {
+                black_box(&header).to_writer(&mut w, ()).unwrap();
+            }
+            w.finalize().unwrap();
+        })
+    });
+    c.bench_function("be_write_six_bytes_aligned_x128", |b| {
+        let mut out = [0u8; FRAMES * 6];
+        b.iter(|| {
+            let mut w = Writer::new(Cursor::new(out.as_mut_slice()));
+            for _ in 0..FRAMES {
+                black_box(&six).to_writer(&mut w, ()).unwrap();
+            }
+            w.finalize().unwrap();
+        })
+    });
 }
 criterion_group!(bebits, bench);
 criterion_main!(bebits);

src/impls/primitive.rs

@@ -83,9 +83,26 @@ impl DekuWriter<(Endian, BitSize, Order)> for u8 {
         writer: &mut Writer<W>,
         (_, size, order): (Endian, BitSize, Order),
     ) -> Result<(), DekuError> {
-        let input = self.to_le_bytes();
-
         let bit_size: usize = size.0;
+        const MAX_TYPE_BITS: usize = BitSize::of::<u8>().0;
+
+        // Fast path, ahead of the bit-slice view: for a `u8` the checks below are
+        // "does it fit" and "are there bits above `bit_size`", both of which are
+        // integer comparisons. `first_one` over a bit-slice is not.
+        if order == Order::Msb0 && writer.leftover.1 == Order::Msb0 && bit_size <= MAX_TYPE_BITS {
+            if bit_size < MAX_TYPE_BITS && (*self >> bit_size) != 0 {
+                return Err(deku_error!(
+                    DekuError::InvalidParam,
+                    "bit size of input is larger than bit requested size",
+                    "{} exceeds {}",
+                    MAX_TYPE_BITS - (self.leading_zeros() as usize),
+                    bit_size
+                ));
+            }
+            return writer.write_bits_uint_msb0(u64::from(*self), bit_size);
+        }
+
+        let input = self.to_le_bytes();
 
         let input_bits = input.view_bits::<Msb0>();
 
@@ -100,7 +117,6 @@ impl DekuWriter<(Endian, BitSize, Order)> for u8 {
         }
 
         // Check for extra bits before sending into writer
-        const MAX_TYPE_BITS: usize = BitSize::of::<u8>().0;
         if let Some(first) = input_bits.first_one() {
             let max = MAX_TYPE_BITS - bit_size;
             if max > first {
@@ -381,6 +397,12 @@ macro_rules! ImplDekuReadBits {
                         size.0
                     ));
                 }
+                // Fast path: big-endian, and `Order::default()` is Msb0. Reads the
+                // field straight into an integer, with no `BitSlice` in between.
+                if !endian.is_le() && size.0 > 0 && size.0 <= 64 {
+                    let value = reader.read_bits_uint_msb0(size.0)? as $inner;
+                    return Ok(<$typ>::from_be_bytes(value.to_be_bytes()));
+                }
                 let mut bits = ::bitvec::array::BitArray::<[u8; { MAX_TYPE_BITS / 8 }], Msb0>::new(
                     [0; { MAX_TYPE_BITS / 8 }],
                 );
@@ -407,6 +429,11 @@ macro_rules! ImplDekuReadBits {
                         size.0
                     ));
                 }
+                // Fast path: big-endian, Msb0. See the `(Endian, BitSize)` impl above.
+                if !endian.is_le() && order == Order::Msb0 && size.0 > 0 && size.0 <= 64 {
+                    let value = reader.read_bits_uint_msb0(size.0)? as $inner;
+                    return Ok(<$typ>::from_be_bytes(value.to_be_bytes()));
+                }
                 let mut bits = ::bitvec::array::BitArray::<[u8; { MAX_TYPE_BITS / 8 }], Msb0>::new(
                     [0; { MAX_TYPE_BITS / 8 }],
                 );
@@ -865,6 +892,39 @@ macro_rules! ImplDekuWrite {
                     }
                     (Endian::Big, Order::Msb0) => {
                         const MAX_TYPE_BITS: usize = BitSize::of::<$typ>().0;
+
+                        // Fast path, ahead of the bit-slice scan. Significant-bit
+                        // count from the big-endian bytes is the same test as
+                        // `first_one`: that check errors when
+                        // `MAX_TYPE_BITS - bit_size > first`, i.e. when
+                        // `MAX_TYPE_BITS - first` (the significant bits) exceeds
+                        // `bit_size`.
+                        if bit_size <= 64 && writer.leftover.1 == Order::Msb0 {
+                            let significant = match input.iter().position(|b| *b != 0) {
+                                Some(i) => {
+                                    (input.len() - i) * 8 - (input[i].leading_zeros() as usize)
+                                }
+                                None => 0,
+                            };
+                            if significant > bit_size {
+                                return Err(deku_error!(
+                                    DekuError::InvalidParam,
+                                    "bit size of input is larger than requested size",
+                                    "{} exceeds {}",
+                                    significant,
+                                    bit_size
+                                ));
+                            }
+                            // The value fits in `bit_size <= 64` bits, so the low
+                            // eight bytes carry all of it even for a 128-bit
+                            // container.
+                            let mut value: u64 = 0;
+                            for &byte in &input[input.len().saturating_sub(8)..] {
+                                value = (value << 8) | u64::from(byte);
+                            }
+                            return writer.write_bits_uint_msb0(value, bit_size);
+                        }
+
                         if let Some(first) = input_bits.first_one() {
                             let max = MAX_TYPE_BITS - bit_size;
                             if max > first {

src/lib.rs

@@ -1000,6 +1000,29 @@ where
     }
 }
 
+/// Integer view of a single-byte `Msb0` leftover, for the reader's fast path.
+///
+/// A leftover of `size` bits always occupies the *high* `size` bits of the byte,
+/// because it is built by copying a bit-slice into a zeroed array at index 0. So
+/// it is a byte and a length, and shifting it is enough: no `BitSlice` needed.
+#[cfg(feature = "bits")]
+impl BoundedBitVec<[u8; 1], crate::bitvec::Msb0> {
+    #[inline]
+    fn from_msb0_byte(byte: u8, size: usize) -> Self {
+        debug_assert!(size <= 8);
+        Self {
+            bits: BitArray::new([byte]),
+            size,
+        }
+    }
+
+    /// The leftover as `(high-aligned byte, bit count)`.
+    #[inline]
+    fn as_msb0_byte(&self) -> (u8, usize) {
+        (self.bits.as_raw_slice()[0], self.size)
+    }
+}
+
 #[cfg(test)]
 #[path = "../tests/test_common/mod.rs"]
 pub mod test_common;

src/reader.rs

@@ -376,6 +376,72 @@ impl<R: Read + Seek> Reader<R> {
         Ok(())
     }
 
+    /// Reads `amt` bits (`1..=64`) most-significant-bit first, returning them
+    /// right-aligned in a `u64`.
+    ///
+    /// This is the integer fast path for the common big-endian / `Msb0` case. It
+    /// is equivalent to `read_bits_into` followed by `load_be`, but the value
+    /// never touches a `BitSlice`: the leftover is a byte and a length, so
+    /// serving a field is a shift and a mask. `read_bits_into` instead rebuilds a
+    /// `BoundedBitVec` per call, whose bit-unaligned `copy_from_bitslice` falls
+    /// back to copying one bit at a time.
+    ///
+    /// Reads whole bytes from the stream only when the leftover cannot satisfy
+    /// `amt`, exactly as `read_bits_into` does, so it consumes no more input.
+    #[inline]
+    #[cfg(feature = "bits")]
+    pub(crate) fn read_bits_uint_msb0(&mut self, amt: usize) -> Result<u64, DekuError> {
+        debug_assert!((1..=64).contains(&amt));
+
+        // Up to 7 leftover bits plus 64 requested does not fit a u64.
+        let mut acc: u128 = 0;
+        let mut have: usize = 0;
+        match self.leftover.take() {
+            Some(Leftover::Byte(byte)) => {
+                acc = u128::from(byte);
+                have = 8;
+            }
+            Some(Leftover::Bits(bits)) => {
+                let (byte, size) = bits.as_msb0_byte();
+                if size != 0 {
+                    acc = u128::from(byte >> (8 - size));
+                    have = size;
+                }
+            }
+            None => {}
+        }
+
+        // One byte at a time, which reads exactly as much as the field needs and
+        // no more. Batching the whole field into a single variable-length
+        // `read_exact` wins on a microbenchmark over wide fields, but loses on a
+        // realistic multi-protocol pipeline, where fields are mostly narrow and
+        // the extra branch costs more than it saves.
+        while have < amt {
+            let mut buf = [0u8; 1];
+            if let Err(e) = self.inner.read_exact(&mut buf) {
+                if e.kind() == ErrorKind::UnexpectedEof {
+                    return Err(DekuError::Incomplete(NeedSize::new(amt)));
+                }
+                return Err(DekuError::Io(e.kind()));
+            }
+            acc = (acc << 8) | u128::from(buf[0]);
+            have += 8;
+        }
+
+        let rest = have - amt;
+        let value = ((acc >> rest) as u64) & (u64::MAX >> (64 - amt));
+        if rest != 0 {
+            let tail = (acc & ((1u128 << rest) - 1)) as u8;
+            self.leftover = Some(Leftover::Bits(crate::BoundedBitVec::from_msb0_byte(
+                tail << (8 - rest),
+                rest,
+            )));
+        }
+
+        self.bits_read += amt;
+        Ok(value)
+    }
+
     /// Attempt to read bits from `Reader`. If enough bits are already "Read", we just grab
     /// enough bits to satisfy `amt`, but will also "Read" more from the stream and store the
     /// leftovers if enough are not already "Read".