From 4f3d2fb09c002fcc82d6ae10d0e9645b79523d80 Mon Sep 17 00:00:00 2001 From: hed0rah <18272116+hed0rah@users.noreply.github.com> Date: Mon, 28 Sep 2026 10:30:00 -0400 Subject: [PATCH 01/23] fix(inspect, od): render a node that has no byte position The SF2 walker's preset and instrument tree has no offset, and the table, --full, --json and od formatted it as a number: inspect on any SoundFont exited with an internal error, in 1.8.6 too. Unplaced nodes print '-'. test_render_every_seed runs every render mode over every seed. --- src/acidcat/commands/inspect.py | 28 +++++++++++++++++------- src/acidcat/commands/od.py | 6 +++++- tests/test_render_every_seed.py | 38 +++++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 9 deletions(-) create mode 100644 tests/test_render_every_seed.py diff --git a/src/acidcat/commands/inspect.py b/src/acidcat/commands/inspect.py index dcea569f..c0b1ea66 100644 --- a/src/acidcat/commands/inspect.py +++ b/src/acidcat/commands/inspect.py @@ -225,6 +225,12 @@ def _render_anomalies(findings, args): print(f" {tag} {off} {f['rule']:16} {f['message']}") +def _at(offset): + """a chunk's offset for the table; a node the walk does not place + (an SF2 preset, a tree node) has none""" + return "-" if offset is None else f"0x{offset:08x}" + + def _render_table(filepath, fmt_label, chunks, file_warns, args, total=None, shown_as=None): file_size = os.path.getsize(filepath) @@ -241,8 +247,9 @@ def _render_table(filepath, fmt_label, chunks, file_warns, args, total=None, for i, c in enumerate(chunks): idx = p("dim", f"[{c.get('_idx', i):>2}]") cid = p("id", f"{c['id']:<5}") - off = p("dim", f"0x{c['offset']:08x}") - print(f" {idx} {cid} {off} {c['size']:<11,} {c['summary']}") + off = p("dim", _at(c["offset"])) + size = "-" if c["size"] is None else f"{c['size']:,}" + print(f" {idx} {cid} {off} {size:<11} {c['summary']}") if not args.quiet: for c in chunks: @@ -250,7 +257,7 @@ def _render_table(filepath, fmt_label, chunks, file_warns, args, total=None, continue print() hdr_id = p("id", c["id"].strip()) - hdr_meta = p("dim", f"@ 0x{c['offset']:08x} ({c['size']} bytes)") + hdr_meta = p("dim", f"@ {_at(c['offset'])} ({c['size'] if c['size'] is not None else '-'} bytes)") print(f"{hdr_id} {hdr_meta}") for fl in c["fields"]: note = p("dim", f" {fl['note']}") if fl["note"] else "" @@ -261,7 +268,8 @@ def _render_table(filepath, fmt_label, chunks, file_warns, args, total=None, if o is not None else " ") off_col = p("dim", off_col) val = p("val", f"{fl['value']!s:<14}") - if args.show_hex and fl["off"] is not None: + if args.show_hex and fl["off"] is not None and ( + c.get("payload_base") is not None or c["offset"] is not None): # field offsets are measured from the chunk's payload base. # RIFF/AIFF/RF64/MThd all have an 8-byte id+size header, so # that is the default; formats with a different header (FLAC @@ -329,18 +337,20 @@ def _full_chunk(chunk, filepath): payload base, the raw region bytes as hex (capped), and every field's absolute byte offset. `acidcat explore` needs nothing but this JSON.""" c = {k: v for k, v in chunk.items() if k != "_idx"} - pb = chunk.get("payload_base", chunk["offset"] + 8) + pb = chunk.get("payload_base") + if pb is None and chunk["offset"] is not None: + pb = chunk["offset"] + 8 c["payload_base"] = pb fields = [] for f in chunk["fields"]: f2 = _public_field(f) # absolute file offset, so a field maps to raw[abs - offset] - f2["abs"] = pb + f["off"] if f["off"] is not None else None + f2["abs"] = pb + f["off"] if f["off"] is not None and pb is not None else None fields.append(f2) c["fields"] = fields # only carry raw bytes for chunks that actually have positioned fields; # audio-data regions are huge and have nothing to highlight. - if any(f["off"] is not None for f in chunk["fields"]): + if chunk["offset"] is not None and any(f["off"] is not None for f in chunk["fields"]): n = min(chunk["size"], _FULL_RAW_CAP) with open(filepath, "rb") as fh: fh.seek(chunk["offset"]) @@ -649,7 +659,9 @@ def run(args): # tuned on trackers broke silently on RIFF. --full has # always emitted the absolute offsets; plain --json now # does too, at no extra cost. - pb = c.get("payload_base", c["offset"] + 8) + pb = c.get("payload_base") + if pb is None and c["offset"] is not None: + pb = c["offset"] + 8 oc["payload_base"] = pb fields = [] for f in c.get("fields", []): diff --git a/src/acidcat/commands/od.py b/src/acidcat/commands/od.py index 66ad99f6..d8537b66 100644 --- a/src/acidcat/commands/od.py +++ b/src/acidcat/commands/od.py @@ -333,7 +333,11 @@ def dim(t): print(header) for c in chunks: - base = c.get("payload_base", c["offset"] + 8) + if c.get("offset") is None: + continue # a tree node (an SF2 preset) has no bytes to dump + base = c.get("payload_base") + if base is None: + base = c["offset"] + 8 summary = c.get("summary", "") title = _c("1;37", f"{str(c['id'])!r} @ 0x{c['offset']:08x} {c['size']:,} bytes", on) print("\n" + title + (dim(" " + summary) if summary else "")) diff --git a/tests/test_render_every_seed.py b/tests/test_render_every_seed.py new file mode 100644 index 00000000..fa855dbe --- /dev/null +++ b/tests/test_render_every_seed.py @@ -0,0 +1,38 @@ +"""Every render path, on every seed, finishes without an internal error. + +1.8.6 shipped `acidcat inspect` crashing on every SoundFont: the SF2 walker +added a preset and instrument tree whose nodes have no byte position, and the +table, --full and od all formatted `offset` as a number. The walk tests passed; +nothing ran the renderers over a walk that holds an unplaced node. This does, +for every format, in every mode the CLI offers for one file.""" +import pytest + +import seeds +from acidcat.cli import main + +MODES = (["inspect"], ["inspect", "-q"], ["inspect", "--hex"], ["inspect", "--json"], + ["inspect", "--full"], ["inspect", "--pretty"], ["od"]) + + +@pytest.mark.parametrize("fmt", sorted(seeds.SEEDS)) +def test_every_mode_renders_the_seed(fmt, tmp_path, capsys): + path = tmp_path / ("seed" + seeds.suffix(fmt)) + path.write_bytes(seeds.build(fmt)) + for mode in MODES: + try: + rc = main([*mode, str(path)]) + except SystemExit as e: + rc = e.code + out = capsys.readouterr() + assert rc in (0, 1, None), f"{' '.join(mode)} exited {rc}: {out.err.strip()[-300:]}" + assert "internal error" not in out.err, f"{' '.join(mode)}: {out.err.strip()[-300:]}" + + +def test_the_seed_set_includes_an_unplaced_node(tmp_path): + """The control: without a seed whose walk holds a node with no offset, the + test above could not have caught the SoundFont crash.""" + from acidcat.core.walk import walk_file + path = tmp_path / ("probe" + seeds.suffix("sf2")) + path.write_bytes(seeds.build("sf2")) + _l, chunks, _w = walk_file(str(path)) + assert any(c.get("offset") is None for c in chunks) From 1ca72af39770acb1573793dc5f48a637435ad10a Mon Sep 17 00:00:00 2001 From: hed0rah <18272116+hed0rah@users.noreply.github.com> Date: Sat, 26 Sep 2026 12:05:54 -0400 Subject: [PATCH 02/23] fix(au, aiff): two walkers found wrong by McGill's public test files A Sun/NeXT file declaring more audio than it holds kept a data chunk past the end of the file; the chunk now owns what is there and the overrun is a size.overrun finding. An AIFF-C APPL chunk read its first data byte as a Pascal-string length without checking it fits, so a 12-byte chunk grew a 71-byte name field; the name is decoded only when it fits. --- src/acidcat/core/walk/aiff.py | 10 ++++++++-- src/acidcat/core/walk/au.py | 9 ++++++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/acidcat/core/walk/aiff.py b/src/acidcat/core/walk/aiff.py index c7fdfce3..26976397 100644 --- a/src/acidcat/core/walk/aiff.py +++ b/src/acidcat/core/walk/aiff.py @@ -277,8 +277,14 @@ def _aiff_appl(b): fields.append(_f(0x00, 4, "signature", sig)) if sig in ("pdos", "stoc") and len(b) > 4: nlen = b[4] - name = b[5:5 + nlen].decode("ascii", errors="replace") - fields.append(_f(0x04, 1 + nlen, "name", name, "pstring")) + if 5 + nlen <= len(b): + name = b[5:5 + nlen].decode("ascii", errors="replace") + fields.append(_f(0x04, 1 + nlen, "name", name, "pstring")) + else: + # the length byte claims more than the chunk holds: not a pstring + warns.append(defect("size.overrun", + "APPL '%s' data does not start with a pstring " + "that fits the chunk" % sig)) fields.append(_f(None, 0, "data", f"{len(b) - 4:,} bytes")) return f"app '{sig}', {len(b) - 4:,} bytes", fields, warns diff --git a/src/acidcat/core/walk/au.py b/src/acidcat/core/walk/au.py index 697e8925..2dde01cf 100644 --- a/src/acidcat/core/walk/au.py +++ b/src/acidcat/core/walk/au.py @@ -212,8 +212,10 @@ def inspect_au(filepath): w.append(f"{name} is not linear PCM; these bytes are a codec and " f"play as noise if fed to a PCM player") if data_offset + eff_size > file_size: - w.append(f"audio runs past the end of the file " - f"(@0x{data_offset:x} + {eff_size:,})") + w.append(defect("size.overrun", + f"audio runs past the end of the file " + f"(@0x{data_offset:x} + {eff_size:,}); " + f"{avail:,} bytes are there")) dfields = [ _f(None, 0, "encoding", name), _f(None, 0, "sample_rate", rate if rate else "unknown"), @@ -223,8 +225,9 @@ def inspect_au(filepath): dfields.append(_f(None, 0, "bits_per_sample", bits)) if secs is not None: dfields.append(_f(None, 0, "duration", f"{secs:.3f} s")) + # a cut file's audio chunk owns what is there, not what was declared chunks.append({ - "id": "data", "offset": data_offset, "size": eff_size, + "id": "data", "offset": data_offset, "size": min(eff_size, avail), "summary": (f"{name} @ {rate} Hz, {eff_size:,} B" + (f", {secs:.2f} s" if secs is not None else "")), "fields": dfields, "warnings": w, "payload_base": data_offset, From 96213957cd7d9ec065307915bcbd4abed3c97acd Mon Sep 17 00:00:00 2001 From: hed0rah <18272116+hed0rah@users.noreply.github.com> Date: Sat, 26 Sep 2026 14:21:16 -0400 Subject: [PATCH 03/23] fix(pdx): accept a slot table shorter than one bank Some writers emit only the slots they fill (26 is common), so the first sample begins before 768 and a whole-bank read takes sample data for slots. The short table is accepted only when it accounts for the file exactly: at least two live slots, all inside the file, laid end to end to the last byte (or one before it). 76 real banks in the corpus now walk; none of 333,922 non-PDX files passes. --- src/acidcat/core/formats/pdx.py | 50 +++++++++++++++++++++++++++++++-- tests/test_pdx.py | 38 +++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 3 deletions(-) diff --git a/src/acidcat/core/formats/pdx.py b/src/acidcat/core/formats/pdx.py index 70356012..dce8eb5a 100644 --- a/src/acidcat/core/formats/pdx.py +++ b/src/acidcat/core/formats/pdx.py @@ -26,6 +26,11 @@ Measured over 3,418 real banks, every table size that resolves is a multiple of 768. +Some writers emit a SHORT table: only the slots they fill, so the first +sample begins before 768 (26 slots is common). It is accepted only when the +short table accounts for the file exactly, samples end to end to the last +byte; 76 real banks do, and none of 333,922 other files does. + Slots may point at the SAME bytes. 947 duplicate slot pairs across the corpus, and every one of them is an exact duplicate -- same offset, same length -- rather than a window into another sample. A bank aliases a sample to several @@ -69,7 +74,7 @@ def parse_table(raw, filesize): """ h = {"ok": False, "why": "", "table_size": 0, "banks": 0, "slots": [], "used": 0, "packer": "", "data_start": 0, - "padded": False} + "padded": False, "short": False} if filesize < BANK + SLOT: h["why"] = "file is smaller than one %d-byte bank table" % BANK return h @@ -94,8 +99,12 @@ def parse_table(raw, filesize): if h["packer"]: h["why"] = ("the bank is packed with %s; the slot table " "belongs to the unpacked form" % h["packer"]) - else: - h["why"] = "a slot points outside the file" + return h + short = _short_table(raw, filesize) + if short: + h.update(short) + return h + h["why"] = "a slot points outside the file" return h low = min(o for o, _s in live) @@ -125,6 +134,41 @@ def parse_table(raw, filesize): return h +def _short_table(raw, filesize): + """A table shorter than one bank, or None. + + Some writers emit only as many slots as they fill: the first sample then + begins where those slots end, before 768, and reading a whole bank reads + sample data as slots. Accepted only when the short table accounts for the + file exactly: every live slot inside it, the samples laid end to end, and + the last one ending at the end of the file (or one byte before it, which + a handful of real banks do). That is what separates a bank from any other + run of bytes, since the format has no magic.""" + first = None + for i in range(0, min(len(raw), BANK) - SLOT + 1, SLOT): + o, n = struct.unpack_from(">II", raw, i) + if o or n: + first = o + break + if first is None or first % SLOT or not 2 * SLOT <= first < BANK: + return None + k = first // SLOT + rows = [struct.unpack_from(">II", raw, i * SLOT) for i in range(k)] + live = sorted(set((o, n) for o, n in rows if o or n)) + if len(live) < 2 or live[0][0] != first: + return None + if any(n == 0 or o < first or o + n > filesize for o, n in live): + return None + if any(a[0] + a[1] != b[0] for a, b in zip(live, live[1:])): + return None + end = live[-1][0] + live[-1][1] + if filesize - end not in (0, 1): + return None + return {"ok": True, "why": "", "table_size": first, "banks": 1, "slots": rows, + "used": sum(1 for o, n in rows if o or n), "data_start": first, + "short": True} + + def looks_like_pdx(raw, filesize): """Structural identification. The format has no magic, so this is it.""" return parse_table(raw, filesize)["ok"] diff --git a/tests/test_pdx.py b/tests/test_pdx.py index 58768b77..3c814a9a 100644 --- a/tests/test_pdx.py +++ b/tests/test_pdx.py @@ -264,3 +264,41 @@ def test_real_corpus_walks_completely(): assert seen >= 100, "only %d banks identified; that tests almost nothing" % seen assert covered_fully >= seen - 20, ( "%d of %d banks were not fully accounted for" % (seen - covered_fully, seen)) + + +def _short_bank(lengths, slots=None, tail=b""): + """A bank whose table holds only `slots` rows (default: one per sample), + with the samples laid end to end right after it.""" + slots = slots or len(lengths) + table_end = slots * pdxmod.SLOT + rows, at = [], table_end + for n in lengths: + rows.append((at, n)) + at += n + rows += [(0, 0)] * (slots - len(rows)) + table = b"".join(struct.pack(">II", o, n) for o, n in rows) + return table + b"".join(bytes([0x11] * n) for n in lengths) + tail + + +def test_a_table_shorter_than_one_bank_is_accepted_when_it_accounts_for_the_file(tmp_path): + """Some writers emit only the slots they fill. Seen on 76 real banks: the + first sample begins where those slots end, and the samples run end to end + to the last byte of the file.""" + raw = _short_bank([400, 300, 500], slots=26) + h = pdxmod.parse_table(raw, len(raw)) + assert h["ok"] and h["short"] and h["table_size"] == 26 * 8 + assert h["used"] == 3 and len(h["slots"]) == 26 + p = tmp_path / "short.pdx" + p.write_bytes(raw) + assert sniff.sniff(str(p)) == "pdx" + + +def test_a_short_table_with_a_gap_or_a_tail_is_not_a_bank(): + gapped = _short_bank([400, 300]) + rows = bytearray(gapped) + struct.pack_into(">I", rows, 8, 16 + 400 + 20) # second sample 20 bytes late + assert not pdxmod.parse_table(bytes(rows), len(rows))["ok"] + tailed = _short_bank([400, 300], tail=bytes(64)) # unaccounted bytes after + assert not pdxmod.parse_table(tailed, len(tailed))["ok"] + one = _short_bank([400]) # one sample proves too little + assert not pdxmod.parse_table(one, len(one))["ok"] From 006da09c1ae40fd08714b80d44684f8bfab0ae30 Mon Sep 17 00:00:00 2001 From: hed0rah <18272116+hed0rah@users.noreply.github.com> Date: Sat, 26 Sep 2026 14:25:32 -0400 Subject: [PATCH 04/23] fix(mdx): a cut module is still an MDX Identification required every offset to land inside the file, so a truncated rip was reported as unrecognised. The header alone identifies an MDX; the walker already reports each offset that dangles. 16 real modules now sniff; no non-MDX file in the corpus has a header that parses. --- src/acidcat/core/formats/mdx.py | 19 ++++++++----------- tests/test_mdx.py | 19 ++++++++++++------- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/src/acidcat/core/formats/mdx.py b/src/acidcat/core/formats/mdx.py index d585dff1..26a87f13 100644 --- a/src/acidcat/core/formats/mdx.py +++ b/src/acidcat/core/formats/mdx.py @@ -254,23 +254,20 @@ def looks_like_mdx(raw, filesize=None): Everything before the offset table is variable-length text, so the only thing that can identify an MDX is whether the arithmetic lands: a title - terminator, a NUL, and a table that resolves to a legal channel count with - every offset inside the file. - - `filesize` matters. The offsets routinely point past the few kilobytes a - sniffer reads, so checking them against len(raw) rejects almost every real - tune when handed a truncated head -- which is exactly what happened, and - the bound has to be the FILE's length rather than the buffer's. + terminator, a NUL, and a table that resolves to a legal channel count. + Offsets that run past the end mark a cut file, not a different format; + `filesize` is kept for callers and no longer consulted. """ h = parse_header(raw) if not h["ok"]: # a packed module is an MDX whose body cannot be walked, not a file # that is something else return bool(h.get("packer")) - n = len(raw) if filesize is None else filesize - if not 0 <= h["voice_abs"] <= n: - return False - return all(0 <= a <= n for a in h["mml_abs"]) + # Offsets past the end of the file do not make it something else: a cut + # rip keeps a whole header (terminator, bank name, a 9- or 16-channel + # table) and the walker reports each offset that dangles. 16 real modules + # are cut like that, and no other file in the corpus has such a header. + return True def voice_count(raw, h): diff --git a/tests/test_mdx.py b/tests/test_mdx.py index 26ff4c6f..cca9f39f 100644 --- a/tests/test_mdx.py +++ b/tests/test_mdx.py @@ -93,16 +93,21 @@ def test_identification_needs_more_than_the_shared_sniff_head(tmp_path): assert sniff.sniff(str(p)) == "mdx" -def test_offsets_are_bounded_by_the_FILE_not_the_buffer(): - """The same bug from the other side. Offsets routinely point past the few - kilobytes a sniffer reads, so checking them against len(buffer) rejects - almost every real tune.""" +def test_offsets_past_the_end_do_not_unmake_an_mdx(tmp_path): + """Offsets routinely point past the few kilobytes a sniffer reads, and a + cut rip's offsets point past the end of the file itself. Neither makes it + a different format: the header identifies it, and the walker reports each + offset that dangles. 16 real modules are cut like that.""" blob = _mdx(title="B" * 300, mml_len=4096) head = blob[:512] assert mdxmod.looks_like_mdx(head, len(blob)) is True - assert mdxmod.looks_like_mdx(head) is False, ( - "without a file size the truncated head cannot resolve, which is " - "exactly why the size has to be passed") + assert mdxmod.looks_like_mdx(head) is True + p = tmp_path / "cut.mdx" + p.write_bytes(head) + assert sniff.sniff(str(p)) == "mdx" + from acidcat.core.walk import walk_file + _label, _chunks, warns = walk_file(str(p)) + assert any("past the end" in str(w) for w in warns) @pytest.mark.parametrize("blob", [ From 85afe651b8bb65ad0803cbb1dd1a0c575132c648 Mon Sep 17 00:00:00 2001 From: hed0rah <18272116+hed0rah@users.noreply.github.com> Date: Sat, 26 Sep 2026 15:19:11 -0400 Subject: [PATCH 05/23] fix(aiff, wav): a 12-bit sample is two bytes Both formats pad a sample point to whole bytes, but the size checks took bits // 8, so 12-bit came out as one byte. Every 12-bit AIFF warned that SSND held twice the audio COMM implied, and every 12-bit PCM WAV that a correct block_align was wrong. The same round-down sat in the AIFF frames-past-EOF check, the grammar's WAV check and the TUI's play duration; all now round up. Found on two 1991 Prosonus AIFFs in the specimen library. --- src/acidcat/core/grammar/helpers.py | 4 ++-- src/acidcat/core/walk/aiff.py | 5 +++-- src/acidcat/core/walk/wav.py | 4 ++-- src/acidcat/tui_app/app.py | 2 +- tests/test_aiff.py | 12 ++++++++++++ tests/test_wave_format_tags.py | 10 ++++++++++ 6 files changed, 30 insertions(+), 7 deletions(-) diff --git a/src/acidcat/core/grammar/helpers.py b/src/acidcat/core/grammar/helpers.py index 2f4a0799..ef704556 100644 --- a/src/acidcat/core/grammar/helpers.py +++ b/src/acidcat/core/grammar/helpers.py @@ -86,8 +86,8 @@ def _wav_fmt_relations(local, ctx): rate = local.get("sample_rate") avg = local.get("avg_bytes_per_sec") warns = [] - if tag == 1 and ch and bits and align != ch * bits // 8: - warns.append(f"block_align {align} != channels*bits/8 = {ch * bits // 8}") + if tag == 1 and ch and bits and align != ch * ((bits + 7) // 8): + warns.append(f"block_align {align} != channels*ceil(bits/8) = {ch * ((bits + 7) // 8)}") if tag == 1 and rate and align and avg != rate * align: warns.append(f"avg_bytes_per_sec {avg} != sample_rate*block_align = {rate * align}") return warns diff --git a/src/acidcat/core/walk/aiff.py b/src/acidcat/core/walk/aiff.py index 26976397..6336a917 100644 --- a/src/acidcat/core/walk/aiff.py +++ b/src/acidcat/core/walk/aiff.py @@ -100,7 +100,8 @@ def _aiff_ssnd(b, ctx, size, avail=None): comp = ctx.get("compression", "NONE") uncompressed = comp in ("NONE", "none", "sowt", "twos", "raw ") if frames and ch and bits and uncompressed: - expected = frames * ch * (bits // 8) + # a sample point is padded to whole bytes: 12-bit is stored in two + expected = frames * ch * ((bits + 7) // 8) if audio_bytes >= 0 and abs(audio_bytes - expected) > max(16, expected * 0.01): warns.append( f"SSND holds {audio_bytes:,} audio bytes but COMM frames " @@ -357,7 +358,7 @@ def inspect_aiff(filepath, form_type, ctx=None): fr, ch, bits = ctx.get("frames"), ctx.get("channels"), ctx.get("bits") comp = ctx.get("compression") if fr and ch and bits and (comp is None or comp in _AIFC_UNCOMPRESSED) \ - and fr * ch * (bits // 8) > file_size: + and fr * ch * ((bits + 7) // 8) > file_size: entry["warnings"].append( f"num_sample_frames {fr:,} implies more audio than the " f"{file_size:,}-byte file holds; duration is not trustworthy" diff --git a/src/acidcat/core/walk/wav.py b/src/acidcat/core/walk/wav.py index 91c92a3f..d22ba519 100644 --- a/src/acidcat/core/walk/wav.py +++ b/src/acidcat/core/walk/wav.py @@ -58,8 +58,8 @@ def _parse_fmt(b, ctx): ctx.update({"format_tag": tag, "channels": ch, "sample_rate": rate, "block_align": align, "bits": bits}) - if tag == 1 and ch and bits and align != ch * bits // 8: - warns.append(f"block_align {align} != channels*bits/8 = {ch * bits // 8}") + if tag == 1 and ch and bits and align != ch * ((bits + 7) // 8): + warns.append(f"block_align {align} != channels*ceil(bits/8) = {ch * ((bits + 7) // 8)}") if tag == 1 and rate and align and avg != rate * align: warns.append(f"avg_bytes_per_sec {avg} != sample_rate*block_align = {rate * align}") # physically implausible but structurally valid values -- a crafted-file diff --git a/src/acidcat/tui_app/app.py b/src/acidcat/tui_app/app.py index 2cc55871..2d1c5b73 100644 --- a/src/acidcat/tui_app/app.py +++ b/src/acidcat/tui_app/app.py @@ -3352,7 +3352,7 @@ def _do_play(self, off, length): rate, ch, bits, floating = self._audio_params() self.action_stop_play(quiet=True) self._play = play.play_bytes(data, rate=rate, ch=ch, bits=bits, floating=floating) - secs = len(data) / max(1, rate * ch * (bits // 8)) + secs = len(data) / max(1, rate * ch * ((bits + 7) // 8)) self.notify(f"playing {len(data):,} bytes as {rate} Hz {ch}ch {bits}-bit " f"(~{secs:.1f}s) -- . to stop") diff --git a/tests/test_aiff.py b/tests/test_aiff.py index 6a3140fd..0a8268be 100644 --- a/tests/test_aiff.py +++ b/tests/test_aiff.py @@ -324,3 +324,15 @@ def test_a_cate_with_no_readable_label_does_not_invent_one(tmp_path): cate = _named(chunks, "cate") assert "bytes" in cate["summary"] assert not [x for x in cate["fields"] if x["name"] == "label"] + + +def test_a_12_bit_sample_point_is_stored_in_two_bytes(tmp_path): + """AIFF pads each sample point up to whole bytes, so 441 frames of 12-bit + mono is 882 bytes. Sizing it at bits // 8 flagged every 12-bit file.""" + f = tmp_path / "twelve.aif" + f.write_bytes(_form(b"AIFF", _comm_aiff(bits=12), _ssnd(882))) + chunks, _w, _c = _walk(f) + assert not _named(chunks, "SSND")["warnings"] + f.write_bytes(_form(b"AIFF", _comm_aiff(bits=12), _ssnd(441))) + chunks, _w, _c = _walk(f) + assert any("imply 882" in w for w in _named(chunks, "SSND")["warnings"]) diff --git a/tests/test_wave_format_tags.py b/tests/test_wave_format_tags.py index f86c7b81..b7b5e110 100644 --- a/tests/test_wave_format_tags.py +++ b/tests/test_wave_format_tags.py @@ -148,3 +148,13 @@ def test_no_tag_is_named_twice_or_left_blank(): reads as 'no tag' rather than 'unnamed tag'.""" assert all(isinstance(v, str) and v.strip() for v in WAVE_FORMAT_TAGS.values()) assert len(set(WAVE_FORMAT_TAGS)) == len(WAVE_FORMAT_TAGS) + + +def test_a_12_bit_pcm_block_is_two_bytes_a_sample(tmp_path): + """PCM pads a sample to whole bytes, so 12-bit stereo aligns on 4. The + check computed channels * bits // 8 = 3 and warned on every such file.""" + fmt = struct.pack(" Date: Sun, 27 Sep 2026 01:12:33 -0400 Subject: [PATCH 06/23] fix(brstm): a cut stream says it is cut The RSTM header declares the whole file's size at 0x08 and the walker never read it, so a stream in FFmpeg's test suite holding 200 KB of a declared 9.1 MB walked as whole. The size is now a field, and a file shorter than it declares is a size.overrun. --- src/acidcat/core/walk/streams.py | 8 ++++++++ tests/test_streams.py | 15 +++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/src/acidcat/core/walk/streams.py b/src/acidcat/core/walk/streams.py index 10a34daf..05016d8f 100644 --- a/src/acidcat/core/walk/streams.py +++ b/src/acidcat/core/walk/streams.py @@ -24,6 +24,8 @@ import os +import struct + from acidcat.core.primitives.notes import coverage from acidcat.core.walk.base import _f @@ -121,8 +123,10 @@ def inspect_brstm(filepath, deep=False): audio = h.get("audio_off", 0) body = max(0, size - audio) + declared = struct.unpack_from(">I", raw, 0x08)[0] fields = _stream("DSP-ADPCM", h["channels"], h["rate"], h["samples"], [ _f(0x00, 4, "magic", "RSTM"), + _f(0x08, 4, "file_size", declared, "bytes, the whole stream", enc=">I"), _f(None, 0, "blocks", format(h.get("blocks", 0), ","), "%s bytes each" % format(h.get("block_size", 0), ",")), _f(None, 0, "audioOffset", "0x%08X" % audio, @@ -132,6 +136,10 @@ def inspect_brstm(filepath, deep=False): ]) if audio > size: warns.append("the header points at audio beyond the end of the file") + elif declared > size: + warns.append( + "the header declares %s bytes and the file holds %s; the stream " + "is cut" % (format(declared, ","), format(size, ","))) return [ {"id": "header", "offset": 0, "size": min(audio, size), "summary": "Nintendo BRSTM, DSP-ADPCM, %d ch at %s Hz" diff --git a/tests/test_streams.py b/tests/test_streams.py index 3e626fd9..2e87192c 100644 --- a/tests/test_streams.py +++ b/tests/test_streams.py @@ -210,3 +210,18 @@ def test_codec_refusals_reach_extract_as_sampleerror(tmp_path): p.write_bytes(MAGIC + struct.pack(">II", 32000, 5) + bytes(0x200)) with pytest.raises(SampleError): list(iter_samples(str(p), fmt="hps")) + + +def test_a_cut_brstm_says_it_is_cut(tmp_path): + """The RSTM header declares the whole file's size at 0x08. FFmpeg's test + suite carries a stream cut to 200 KB of a declared 9.1 MB, and the walker + reported it as whole because it never read that field.""" + data = bytearray(_brstm()) + struct.pack_into(">I", data, 0x08, len(data)) + chunks, warns = streams.inspect_brstm(_write(tmp_path, "whole.brstm", bytes(data))) + assert not warns + assert [f["value"] for f in chunks[0]["fields"] if f["name"] == "file_size"] == [len(data)] + struct.pack_into(">I", data, 0x08, len(data) * 45) + _chunks, warns = streams.inspect_brstm(_write(tmp_path, "cut.brstm", bytes(data))) + assert len(warns) == 1 + assert "the stream is cut" in str(warns[0]) From b1971553d023362de1194a1a344a89498d0dc446 Mon Sep 17 00:00:00 2001 From: hed0rah <18272116+hed0rah@users.noreply.github.com> Date: Mon, 28 Sep 2026 19:47:10 -0400 Subject: [PATCH 07/23] fix(au, aiff): the backported overrun warnings are plain strings The McGill fix was written against 2.0's finding codes; 1.8 has none, so the two defect() calls would have raised NameError on the first file that reached them. --- src/acidcat/core/walk/aiff.py | 5 ++--- src/acidcat/core/walk/au.py | 7 +++---- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/acidcat/core/walk/aiff.py b/src/acidcat/core/walk/aiff.py index 6336a917..2e5c37d4 100644 --- a/src/acidcat/core/walk/aiff.py +++ b/src/acidcat/core/walk/aiff.py @@ -283,9 +283,8 @@ def _aiff_appl(b): fields.append(_f(0x04, 1 + nlen, "name", name, "pstring")) else: # the length byte claims more than the chunk holds: not a pstring - warns.append(defect("size.overrun", - "APPL '%s' data does not start with a pstring " - "that fits the chunk" % sig)) + warns.append("APPL '%s' data does not start with a pstring " + "that fits the chunk" % sig) fields.append(_f(None, 0, "data", f"{len(b) - 4:,} bytes")) return f"app '{sig}', {len(b) - 4:,} bytes", fields, warns diff --git a/src/acidcat/core/walk/au.py b/src/acidcat/core/walk/au.py index 2dde01cf..1092e70b 100644 --- a/src/acidcat/core/walk/au.py +++ b/src/acidcat/core/walk/au.py @@ -212,10 +212,9 @@ def inspect_au(filepath): w.append(f"{name} is not linear PCM; these bytes are a codec and " f"play as noise if fed to a PCM player") if data_offset + eff_size > file_size: - w.append(defect("size.overrun", - f"audio runs past the end of the file " - f"(@0x{data_offset:x} + {eff_size:,}); " - f"{avail:,} bytes are there")) + w.append(f"audio runs past the end of the file " + f"(@0x{data_offset:x} + {eff_size:,}); " + f"{avail:,} bytes are there") dfields = [ _f(None, 0, "encoding", name), _f(None, 0, "sample_rate", rate if rate else "unknown"), From 1124520520f56b04dd9fbf079492a8d267a32d08 Mon Sep 17 00:00:00 2001 From: hed0rah <18272116+hed0rah@users.noreply.github.com> Date: Sun, 27 Sep 2026 11:22:30 -0400 Subject: [PATCH 08/23] fix(sigmf): the meta half places only its own bytes Opened by its .sigmf-meta, the walk laid the data file's regions over the JSON, so a real 1,045-byte sidecar came back with three invalid chunks and every byte unaccounted for. From the meta, the walk now places one sidecar chunk; captures and annotations are listed, unplaced, with the file that holds them. The tests that asserted byte regions from the meta now walk the data half. --- src/acidcat/core/walk/sigmf.py | 21 +++++++++++++++++++++ tests/test_sigmf.py | 31 ++++++++++++++++++++++++++++--- 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/src/acidcat/core/walk/sigmf.py b/src/acidcat/core/walk/sigmf.py index 8bb9b97b..2e57a37c 100644 --- a/src/acidcat/core/walk/sigmf.py +++ b/src/acidcat/core/walk/sigmf.py @@ -325,11 +325,32 @@ def inspect_sigmf(path, deep=False): warns.append(f"listing the first {_ANNOTATION_CAP} of " f"{len(annotations)} annotations") + if meta is not None and meta is src: + return _as_sidecar(chunks, meta, data_src), warns samples, _ = _samples_chunk(data_path, data_size, geo, dt, deep) chunks.append(samples) return chunks, warns +def _as_sidecar(chunks, meta, data_src): + """The walk seen from the .sigmf-meta: its own bytes are the JSON. The + captures and annotations locate bytes of the .sigmf-data, so here they are + listed but not placed, and their pointers are not followed into this file.""" + where = os.path.basename(data_src.name) if data_src is not None else "the .sigmf-data" + for c in chunks: + if c["id"] == "global": + continue + c.update(offset=0, size=0, payload_base=0) + c["summary"] += f" (in {where})" + for f in c["fields"]: + f.pop("xref", None) + chunks.append({"id": "sidecar", "offset": 0, "size": meta.size, "payload_base": 0, + "summary": f"SigMF metadata, {meta.size:,} bytes of JSON; " + f"the samples are in {where}", + "fields": [], "warnings": []}) + return chunks + + def inspect_iq(path, deep=False): size = os.path.getsize(path) ext = os.path.splitext(path.lower())[1] diff --git a/tests/test_sigmf.py b/tests/test_sigmf.py index 734009da..3595e7ce 100644 --- a/tests/test_sigmf.py +++ b/tests/test_sigmf.py @@ -6,6 +6,7 @@ format is open and headerless, so no real capture is needed to exercise it.""" import hashlib import json +import os import struct from acidcat.core.infra.sniff import sniff @@ -37,6 +38,11 @@ def _make_pair(tmp_path, extra_global=None, captures=None, annotations=None): return str(p), data +def _data(meta): + """the byte regions live in the data file; walk that half to place them""" + return meta[:-len(".sigmf-meta")] + ".sigmf-data" + + def test_sigmf_sniffs_both_pair_members(tmp_path): meta, _ = _make_pair(tmp_path) assert sniff(meta) == "sigmf" @@ -58,7 +64,7 @@ def test_sigmf_captures_are_byte_regions(tmp_path): meta, _ = _make_pair(tmp_path, captures=[ {"core:sample_start": 0, "core:frequency": 8_428_190_000}, {"core:sample_start": 4, "core:frequency": 8_500_000_000}]) - chunks, _ = sigmf.inspect_sigmf(meta) + chunks, _ = sigmf.inspect_sigmf(_data(meta)) caps = [c for c in chunks if c["id"].startswith("capture")] assert caps[0]["offset"] == 0 and caps[0]["size"] == 16 # samples 0..4, 4 B each assert caps[1]["offset"] == 16 and caps[1]["size"] == 16 @@ -69,7 +75,7 @@ def test_sigmf_captures_are_byte_regions(tmp_path): def test_sigmf_annotation_region(tmp_path): meta, _ = _make_pair(tmp_path, annotations=[ {"core:sample_start": 2, "core:sample_count": 3, "core:label": "burst"}]) - chunks, _ = sigmf.inspect_sigmf(meta) + chunks, _ = sigmf.inspect_sigmf(_data(meta)) ann = next(c for c in chunks if c["id"].startswith("annotation")) assert ann["offset"] == 8 and ann["size"] == 12 # 3 samples * 4 B assert "burst" in ann["summary"] @@ -77,7 +83,7 @@ def test_sigmf_annotation_region(tmp_path): def test_sigmf_deep_verifies_sha_and_stats(tmp_path): meta, _ = _make_pair(tmp_path) - chunks, warns = sigmf.inspect_sigmf(meta, deep=True) + chunks, warns = sigmf.inspect_sigmf(_data(meta), deep=True) g = next(c for c in chunks if c["id"] == "global") assert next(x for x in g["fields"] if x["name"] == "sha512")["note"] == "verified" samp = next(c for c in chunks if c["id"] == "samples") @@ -155,3 +161,22 @@ def test_malformed_json_types_degrade(tmp_path): meta.write_text(js) chunks, warns = sigmf.inspect_sigmf(str(meta)) # must not raise assert chunks # always a degraded result + + +def test_the_meta_half_places_only_its_own_bytes(tmp_path): + """Opened by its .sigmf-meta, the walk laid the data file's regions over + the JSON: a real 1,045-byte sidecar came back with three invalid chunks + and every byte unaccounted for. Its own bytes are the JSON; the captures + and annotations are listed but point into the other file.""" + meta, _ = _make_pair(tmp_path, annotations=[ + {"core:sample_start": 2, "core:sample_count": 3, "core:label": "burst"}]) + chunks, warns = sigmf.inspect_sigmf(meta) + assert warns == [] + size = os.path.getsize(meta) + side = next(c for c in chunks if c["id"] == "sidecar") + assert (side["offset"], side["size"]) == (0, size) + for c in chunks: + if c["id"] != "sidecar": + assert c["size"] == 0, c["id"] + assert all("xref" not in f for f in c["fields"]), c["id"] + assert "(in cap.sigmf-data)" in next(c for c in chunks if c["id"] == "annotation[0]")["summary"] From c285a631b23ee5eaa1671cdb4c56978206bcc346 Mon Sep 17 00:00:00 2001 From: hed0rah <18272116+hed0rah@users.noreply.github.com> Date: Sun, 27 Sep 2026 11:22:31 -0400 Subject: [PATCH 09/23] fix(mp4): repair declines a multi-track file without raising The MP4 repairer located mdat before its guarded run, and locating it raised on a real two-track video, so constraints.repair broke its no-raise promise. mdat is now compared only when there is a rewrite to guard. --- src/acidcat/core/write/repairers.py | 6 ++++-- tests/test_mp4repair.py | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/acidcat/core/write/repairers.py b/src/acidcat/core/write/repairers.py index ebf9ce21..f281d8c4 100644 --- a/src/acidcat/core/write/repairers.py +++ b/src/acidcat/core/write/repairers.py @@ -175,9 +175,11 @@ def analyze(self, data, opts=None): return self._run(data, patch=False)[1] def apply(self, data, opts=None): - before = self._mdat(data) + # locate mdat only when there is a rewrite to guard: finding it first + # raised on a multi-track file _run would have reported as out of scope new_data, report = self._run(data) - if report.violations and self._mdat(new_data) != before: + if (report.violations and new_data is not data + and self._mdat(new_data) != self._mdat(data)): raise AudioGuardError("mdat payload would change") return new_data, report diff --git a/tests/test_mp4repair.py b/tests/test_mp4repair.py index ba0e18d3..d2ee7387 100644 --- a/tests/test_mp4repair.py +++ b/tests/test_mp4repair.py @@ -209,3 +209,17 @@ def test_out_of_scope_is_still_a_quiet_note_not_damage(tmp_path, capsys): p.write_bytes(_box(b"ftyp", b"M4A \x00\x00\x00\x00") + tree + _box(b"mdat", payload)) assert main(["validate", str(p)]) == 0 + + +def test_repair_declines_a_multitrack_file_without_raising(): + """`constraints.repair` promises a report, never an exception. The MP4 + repairer located mdat BEFORE its guarded run, and locating it raised on a + multi-track file, so a real two-track video escaped as Mp4RepairError.""" + from acidcat.core.write import constraints + sizes, runs, payload, start, good = _make_multichunk() + stbl = _stsz(sizes) + _stsc(runs) + _stco(good) + _stco(good) + tree = _box(b"moov", _box(b"trak", _box(b"mdia", + _box(b"minf", _box(b"stbl", stbl))))) + data = _box(b"ftyp", b"M4A \x00\x00\x00\x00") + tree + _box(b"mdat", payload) + new_data, _report = constraints.repair(data) + assert new_data == data From faccd06e98562935fb719b35f2465ae4c35871a8 Mon Sep 17 00:00:00 2001 From: hed0rah <18272116+hed0rah@users.noreply.github.com> Date: Sun, 27 Sep 2026 11:22:31 -0400 Subject: [PATCH 10/23] fix(mp4): render box timestamps without utcfromtimestamp Deprecated from Python 3.12; the warning failed the hunt sweep on a real MP4 with a creation time. Output is unchanged. --- src/acidcat/core/walk/mp4.py | 2 +- tests/test_mp4.py | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/acidcat/core/walk/mp4.py b/src/acidcat/core/walk/mp4.py index 77678483..32389bad 100644 --- a/src/acidcat/core/walk/mp4.py +++ b/src/acidcat/core/walk/mp4.py @@ -363,7 +363,7 @@ def _stamp(unix_time): raising: the field is only as trustworthy as the writer.""" import datetime try: - return datetime.datetime.utcfromtimestamp(unix_time).strftime( + return datetime.datetime.fromtimestamp(unix_time, datetime.timezone.utc).strftime( "%Y-%m-%d %H:%M:%S UTC") except (OSError, OverflowError, ValueError): return str(unix_time) diff --git a/tests/test_mp4.py b/tests/test_mp4.py index d0584364..eedf1369 100644 --- a/tests/test_mp4.py +++ b/tests/test_mp4.py @@ -397,3 +397,14 @@ def test_a_malformed_box_claims_only_the_bytes_that_remain(): assert bad[0]["declared"] == 0x47400030 assert bad[0]["size"] == len(full) - bad[0]["offset"] assert bad[0]["offset"] + bad[0]["size"] <= len(full) + + +def test_a_box_timestamp_renders_without_the_deprecated_utc_call(): + """utcfromtimestamp is deprecated from Python 3.12 and warned on every MP4 + with a creation time; a real one in the hunt corpus failed the sweep.""" + import warnings + from acidcat.core.walk.mp4 import _stamp + with warnings.catch_warnings(): + warnings.simplefilter("error") + assert _stamp(946080000) == "1999-12-25 00:00:00 UTC" + assert _stamp(2 ** 40) == str(2 ** 40) From 87557a6d8a4951f7265f0f21913a17c5a428386a Mon Sep 17 00:00:00 2001 From: hed0rah <18272116+hed0rah@users.noreply.github.com> Date: Mon, 28 Sep 2026 10:30:00 -0400 Subject: [PATCH 11/23] fix(audit): name an extract-only format and point at extract A SNES or N64 ROM sniffs as one, yet audit printed [unknown] and suggested locate. Exit 2 still says nothing was checked. --- src/acidcat/commands/audit.py | 18 +++++++++++++++--- tests/test_audit.py | 16 ++++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/src/acidcat/commands/audit.py b/src/acidcat/commands/audit.py index fabaca5a..3bf8246c 100644 --- a/src/acidcat/commands/audit.py +++ b/src/acidcat/commands/audit.py @@ -278,6 +278,17 @@ def _run_one(args): print(f"acidcat audit: {path}: {e}", file=sys.stderr) return 2 size = os.path.getsize(path) + # a format acidcat recognises but only extracts from (a console ROM) is + # not "unknown", and `extract` is the verb that gets at its audio + extract_only = None + if not scanned: + from acidcat.core.infra import sniff + from acidcat.core.walk import _EXTRACT_ONLY + what = _EXTRACT_ONLY.get(sniff.sniff(path)) + if what: + extract_only = what.split(" ", 1)[1] + label = label or extract_only + todo = "extract" if extract_only else "locate" if chosen_format(args) == "json": out = { @@ -329,8 +340,9 @@ def _run_one(args): if not scanned: print(" FORENSICS not scanned -- no walker for this format") - print(" try: acidcat locate " + - os.path.basename(path) + " (finds embedded audio regardless)") + print(f" try: acidcat {todo} " + os.path.basename(path) + + (" (recovers its samples)" if extract_only + else " (finds embedded audio regardless)")) elif not other: print(" FORENSICS nothing else flagged") else: @@ -378,6 +390,6 @@ def _run_one(args): # "clean" is a claim about checks that ran. With no walker they did not, # and the honest answer is that this verb had nothing to say -- `locate` # still finds embedded containers in a format we cannot walk. - bits.append("clean" if scanned else "not analyzable -- no walker; try acidcat locate") + bits.append("clean" if scanned else f"not analyzable -- no walker; try acidcat {todo}") print(f"\n VERDICT: {', '.join(bits) if bits else 'no structural fixes; review findings'}") return _code(scanned, vios, findings, integ) diff --git a/tests/test_audit.py b/tests/test_audit.py index 73bd2dbe..820f26ed 100644 --- a/tests/test_audit.py +++ b/tests/test_audit.py @@ -133,3 +133,19 @@ def test_audit_survives_a_data_chunk_that_overruns_the_file(tmp_path, capsys): assert out.strip(), "audit produced no report at all" assert "VERDICT" in out assert rc in (0, 1) + + +def test_a_rom_is_named_and_pointed_at_extract(tmp_path, capsys): + """A SNES ROM sniffs as one and `extract` recovers its samples, yet audit + printed it as [unknown] and suggested `locate`. It names what it recognised + and the verb that gets at the audio; exit 2 still says nothing was checked.""" + rom = bytearray(0x8000) + rom[0x7FC0 + 0x15] = 0x20 # LoROM map mode + rom[0x7FC0 + 0x1C:0x7FC0 + 0x20] = bytes((0x34, 0x12, 0xCB, 0xED)) # complement, checksum + p = tmp_path / "cart.sfc" + p.write_bytes(bytes(rom)) + rc = audit.run(SimpleNamespace(input=str(p), signal=False, output_format="table", + json=False)) + out = capsys.readouterr().out + assert "[SNES ROM]" in out and "acidcat extract" in out and "[unknown]" not in out + assert rc == 2 From f74db47755f2c7b446f4beac29cea134eb3d2b08 Mon Sep 17 00:00:00 2001 From: hed0rah <18272116+hed0rah@users.noreply.github.com> Date: Mon, 28 Sep 2026 10:30:00 -0400 Subject: [PATCH 12/23] fix(anomalies): a file that is a zip is not an appended zip Every real .xpn, .labx and .multisample was a polyglot carrying embedded media: the archive's own end record read as an appended zip, its stored members as smuggled files. An SF3's Ogg samples tripped the same rule. The appended-zip scan skips a file that starts with a local header, embedded media skips an archive's members and an SF3's Ogg, and a multisample places its central directory (zip_directory_extent). --- src/acidcat/core/forensics/anomalies.py | 12 ++++++++-- src/acidcat/core/primitives/zipio.py | 29 +++++++++++++++++++++++++ src/acidcat/core/walk/multisample.py | 7 +++++- tests/test_anomalies.py | 27 +++++++++++++++++++++++ 4 files changed, 72 insertions(+), 3 deletions(-) diff --git a/src/acidcat/core/forensics/anomalies.py b/src/acidcat/core/forensics/anomalies.py index db5f2f8c..9537c620 100644 --- a/src/acidcat/core/forensics/anomalies.py +++ b/src/acidcat/core/forensics/anomalies.py @@ -486,10 +486,13 @@ def scan(filepath, fmt_label=None, chunks=None, warns=None): # size-based check above cannot see it). Scans the last 64K+ from the end. if not any(f["rule"] == "polyglot" for f in findings): with open(filepath, "rb") as f: + is_zip = f.read(4) == b"PK\x03\x04" f.seek(max(0, size - 66000)) tail = f.read() idx = tail.rfind(b"PK\x05\x06") - if idx >= 0 and len(tail) - idx >= 22: + # a file that IS a zip (.xpn, .labx, .multisample) ends with its own + # end record; that is the archive's index, not one appended to it + if idx >= 0 and len(tail) - idx >= 22 and not is_zip: findings.append({"severity": "alert", "offset": (size - len(tail)) + idx, "rule": "polyglot", "message": "possible polyglot: appended ZIP archive " @@ -748,8 +751,13 @@ def scan(filepath, fmt_label=None, chunks=None, warns=None): # branched on the label, and it made the guard depend on the label being # non-empty: a walker that returned "" got a spurious embedded-Ogg # finding on every ordinary Ogg file. + # an archive's members are whole files by design (a .multisample zone + # is a WAV in a zip), and an SF3 stores every sample as an Ogg stream wanted = [c for c in _EMBEDDED_MEDIA - if not (c[0] == b"OggS" and fmt_id in _OGG_IDS)] + if not (c[0] == b"OggS" and fmt_id in _OGG_IDS) + and not (c[0] == b"OggS" and fmt_id == "sf2")] + if own == b"PK\x03\x04": + wanted = [] hits = _find_embedded(filepath, wanted, own) if wanted else {} for magic, _form_at, _form, name in wanted: at = hits.get(magic) diff --git a/src/acidcat/core/primitives/zipio.py b/src/acidcat/core/primitives/zipio.py index 7d3e4022..deb96e2b 100644 --- a/src/acidcat/core/primitives/zipio.py +++ b/src/acidcat/core/primitives/zipio.py @@ -33,3 +33,32 @@ def zip_data_offset(z, zi): n = int.from_bytes(hdr[26:28], "little") # file name length m = int.from_bytes(hdr[28:30], "little") # extra field length return zi.header_offset + 30 + n + m + + +_EOCD = b"PK\x05\x06" + + +def zip_directory_extent(z, size): + """(offset, length) of the central directory plus the end record, or None. + + A zip-backed format ends with its own directory; a walker that places only + the entries leaves those bytes "past the container end", and the forensic + scan then reports the file's own index as an appended archive. Located from + the end record, which states the directory's size, rather than trusting its + stated offset: data prepended to the archive shifts the one and not the other. + """ + tail_len = min(size, 22 + 0xFFFF) + try: + z.fp.seek(size - tail_len) + tail = z.fp.read(tail_len) + except OSError: + return None + at = tail.rfind(_EOCD) + if at < 0 or len(tail) - at < 22: + return None + cd_size = int.from_bytes(tail[at + 12:at + 16], "little") + eocd = size - tail_len + at + start = eocd - cd_size + if start < 0: + return None + return start, size - start diff --git a/src/acidcat/core/walk/multisample.py b/src/acidcat/core/walk/multisample.py index 806188a9..63f4ebab 100644 --- a/src/acidcat/core/walk/multisample.py +++ b/src/acidcat/core/walk/multisample.py @@ -16,7 +16,7 @@ import zipfile import zlib -from acidcat.core.primitives.zipio import zip_data_offset +from acidcat.core.primitives.zipio import zip_data_offset, zip_directory_extent from acidcat.core.walk.base import _f _ZONE_CAP = 48 # don't flood the view on big kits @@ -156,4 +156,9 @@ def inspect_multisample(filepath): chunks.append({"id": "zone", "offset": 0, "size": 0, "summary": f"... {len(samples) - _ZONE_CAP} more zone(s)", "fields": [], "warnings": [], "payload_base": 0}) + cd = zip_directory_extent(z, size) + if cd: + chunks.append({"id": "zip_directory", "offset": cd[0], "size": cd[1], + "summary": f"zip central directory, {len(names)} entries", + "fields": [], "warnings": [], "payload_base": cd[0]}) return chunks, warns diff --git a/tests/test_anomalies.py b/tests/test_anomalies.py index 19a1989c..5c2c6442 100644 --- a/tests/test_anomalies.py +++ b/tests/test_anomalies.py @@ -609,3 +609,30 @@ def test_the_four_argument_form_is_unchanged(tmp_path): path = _write(tmp_path, "c.wav", _wav(_FMT, _chunk(b"data", b"\x00" * 32))) fmt, chunks, warns = walk_file(path, deep=False) assert anomalies.scan(path, fmt, chunks, warns) == anomalies.scan(path) + + +def _zip_with_wav(tmp_path, name): + wav = _wav(_FMT, _chunk(b"data", b"\x00\x01" * 64)) + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w", zipfile.ZIP_STORED) as z: + z.writestr("multisample.xml", '') + z.writestr("a.wav", wav) + p = tmp_path / name + p.write_bytes(buf.getvalue()) + return p + + +def test_a_file_that_is_a_zip_is_not_an_appended_zip(tmp_path): + """.xpn, .labx and .multisample are zips from byte 0. Their end record is + their own index, and a stored WAV member is a whole file by design; every + real one was reported as a polyglot carrying embedded media.""" + rules = {f["rule"] for f in anomalies.scan(str(_zip_with_wav(tmp_path, "kit.multisample")))} + assert not rules & {"polyglot", "embedded_standalone_media", "trailing_data"} + + +def test_a_zip_appended_to_a_wav_is_still_a_polyglot(tmp_path): + """The control: the exemption is for a file that starts as a zip.""" + host = _wav(_FMT, _chunk(b"data", b"\x00\x01" * 64)) + p = tmp_path / "carrier.wav" + p.write_bytes(host + _zip_with_wav(tmp_path, "inner.zip").read_bytes()) + assert "polyglot" in {f["rule"] for f in anomalies.scan(str(p))} From ecbb7cc346ea2bc2ab7021ccaa25b972b7e8763f Mon Sep 17 00:00:00 2001 From: hed0rah <18272116+hed0rah@users.noreply.github.com> Date: Mon, 28 Sep 2026 10:30:00 -0400 Subject: [PATCH 13/23] fix(amxd): the marker is the device type A Max instrument says 'iiii' and was a magic mismatch. Every audio effect measured carries 'aaaa' and every instrument 'iiii'; the field is named device_type, and an unmeasured value is a notice. --- docs/formats/ableton-anatomy.html | 6 +++--- src/acidcat/core/walk/ableton.py | 20 +++++++++++++------- tests/test_ableton.py | 19 ++++++++++++++++++- 3 files changed, 34 insertions(+), 11 deletions(-) diff --git a/docs/formats/ableton-anatomy.html b/docs/formats/ableton-anatomy.html index c876e1bc..ecd1ba8c 100644 --- a/docs/formats/ableton-anatomy.html +++ b/docs/formats/ableton-anatomy.html @@ -338,12 +338,12 @@
magic"ampf"header12 byteschainid + u32 lengthpayloadMax patcher JSON
-

A Max for Live device is an Ableton Max Patch Format container: the ASCII magic ampf, a version word, and then -- before the chunk chain begins -- a four-byte marker, aaaa. Only at offset 12 does the chain start: four-byte id, little-endian length, payload. The ptch chunk carries the Max patcher, a short binary preamble followed by JSON.

+

A Max for Live device is an Ableton Max Patch Format container: the ASCII magic ampf, a version word, and then -- before the chunk chain begins -- a four-byte device type: aaaa for an audio effect, iiii for an instrument. Only at offset 12 does the chain start: four-byte id, little-endian length, payload. The ptch chunk carries the Max patcher, a short binary preamble followed by JSON.

layout
- 
0x00  "ampf"  u32:version  "aaaa"      <- 12-byte header
+      
0x00  "ampf"  u32:version  "aaaa"      <- device type; 12-byte header
 0x0C  "meta"  u32:len      ...
 0x18  "ptch"  u32:len      mx@c ... { patcher JSON }

-      
The marker is not a chunk, and mistaking it for one is loud. Reading aaaa as an id makes the following four bytes -- the ASCII meta -- decode as a length of 1,635,018,093 bytes, and the whole chain collapses to one impossible chunk. The chain starts at 12, not 8.

+      
The device type is not a chunk, and mistaking it for one is loud. Reading aaaa as an id makes the following four bytes -- the ASCII meta -- decode as a length of 1,635,018,093 bytes, and the whole chain collapses to one impossible chunk. The chain starts at 12, not 8.

       
The length field cannot be trusted. A chunk claiming more bytes than remain must stop the walk rather than read past the end. The ptch chunk ends exactly on the last byte of the file, so a chain that stops short means something is missing.
diff --git a/src/acidcat/core/walk/ableton.py b/src/acidcat/core/walk/ableton.py index d731e146..e7e4c3b8 100644 --- a/src/acidcat/core/walk/ableton.py +++ b/src/acidcat/core/walk/ableton.py @@ -417,10 +417,14 @@ def inspect_ableton_xml(filepath, fmt_id="als"): # ── Max for Live ────────────────────────────────────────────────────────── -# 'ampf' magic, a u32 version, then a 4-byte marker -- 'aaaa' in every specimen -# seen -- and only then the chunk chain. Reading the marker as a chunk id makes -# its next 4 bytes look like a 1.6 GB length, which is how this was caught. +# 'ampf' magic, a u32 version, then a 4-byte device type, and only then the +# chunk chain. Reading the type as a chunk id makes its next 4 bytes look like +# a 1.6 GB length, which is how this was caught. The type was taken for a +# constant 'aaaa' until a Max instrument said 'iiii': every audio effect +# measured carries 'aaaa' and every instrument 'iiii', each from the preset +# folder Live files that kind under. _AMXD_HEADER = 12 +_AMXD_KINDS = {b"aaaa": "audio effect", b"iiii": "instrument"} _AMXD_MAX_CHUNKS = 64 @@ -433,8 +437,10 @@ def inspect_amxd(filepath): if raw[:4] != b"ampf": warns.append("missing 'ampf' magic") marker = raw[8:12] - if marker != b"aaaa": - warns.append(f"marker at offset 8 is {marker!r}, expected b'aaaa'") + kind = _AMXD_KINDS.get(marker) + if kind is None: + warns.append(f"device type at offset 8 is {marker!r}; the measured " + f"ones are b'aaaa' (audio effect) and b'iiii' (instrument)") chunks = [{ "id": "ampf", "offset": 0, "size": min(size, _AMXD_HEADER), @@ -447,8 +453,8 @@ def inspect_amxd(filepath): _f(0x00, 4, "magic", raw[:4].decode("ascii", "replace")), _f(0x04, 4, "version", struct.unpack_from("= 8 else "?"), - _f(0x08, 4, "marker", marker.decode("ascii", "replace"), - "constant separator before the chunk chain"), + _f(0x08, 4, "device_type", marker.decode("ascii", "replace"), + kind or "not a type measured"), ], "warnings": [], "payload_base": 0, }] diff --git a/tests/test_ableton.py b/tests/test_ableton.py index f663b51f..4738c872 100644 --- a/tests/test_ableton.py +++ b/tests/test_ableton.py @@ -372,7 +372,7 @@ def test_xml_walker_reads_the_creator(tmp_path): # ── Max for Live ────────────────────────────────────────────────────────── def test_amxd_chunk_chain(tmp_path): - """The chain starts at 12, after a constant 'aaaa' marker. Treating that + """The chain starts at 12, after the 4-byte device type. Treating that marker as a chunk id makes its next 4 bytes read as a 1.6 GB length -- which is exactly how the first version of this walker failed on a real device.""" p = tmp_path / "d.amxd" @@ -385,6 +385,23 @@ def test_amxd_chunk_chain(tmp_path): assert "JSON at +4" in chunks[2]["summary"] +def test_amxd_device_type_is_named_not_flagged(tmp_path): + """The 'aaaa' was never a constant: it is the device type. A real Max + instrument says 'iiii' and was reported as a magic mismatch.""" + chain = (b"meta" + struct.pack(" Date: Mon, 28 Sep 2026 10:30:01 -0400 Subject: [PATCH 14/23] fix(vital): a skin is not a preset A .vitalskin carries synth_version too, so its theme keys came back as unknown preset keys with carrier alerts. A skin has no settings; the extension and a whole file without settings are refused. The golden sniff of a 16-byte vital head moves from vital to unknown. --- src/acidcat/core/infra/sniff.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/acidcat/core/infra/sniff.py b/src/acidcat/core/infra/sniff.py index 9b41b2cf..4d8822d6 100644 --- a/src/acidcat/core/infra/sniff.py +++ b/src/acidcat/core/infra/sniff.py @@ -280,6 +280,7 @@ def sniff_bytes(head): _VITAL_WINDOW = 64 * 1024 _VITAL_KEY = b'"synth_version"' +_VITAL_BODY = b'"settings"' def _is_vital(filepath): @@ -298,11 +299,18 @@ def _is_vital(filepath): the sniffer stricter than the parser and left `inspect` unable to reach any real preset. """ + # A .vitalskin (Vital's UI theme) also carries synth_version, but it is a + # table of colours and sizes with no `settings`: read as a preset, every key + # came back as an unknown one, "carrier" alerts included. + if input_name(filepath).lower().endswith(".vitalskin"): + return False try: with open(filepath, "rb") as fh: head = fh.read(_VITAL_WINDOW) if _VITAL_KEY in head: - return True + size = fh.seek(0, 2) + # the whole file was read and holds no preset body + return size > _VITAL_WINDOW or _VITAL_BODY in head size = fh.seek(0, 2) if size <= _VITAL_WINDOW: return False # the head already was the file From 040e083232bd9019e70aab17af82b35ef4cb3d8d Mon Sep 17 00:00:00 2001 From: hed0rah <18272116+hed0rah@users.noreply.github.com> Date: Mon, 28 Sep 2026 10:30:01 -0400 Subject: [PATCH 15/23] fix(wav): read the RF64 reservation instead of calling it damage The 28-byte JUNK after the RIFF header is the ds64 reservation. 174 of 6,000 real WAVs fill it on purpose and were reported as something overwritten in place: the ds64 sizes (stale when chunks were added later, or in a 64-byte reservation), a 28-character quote Ableton Live writes there, or the RIFF size over such a quote. Each is now named; the quote is an Ableton Live provenance tell. --- src/acidcat/core/forensics/provenance.py | 22 ++++++++ src/acidcat/core/walk/wav.py | 67 +++++++++++++++++++++++ tests/test_wav_ds64_reservation.py | 70 ++++++++++++++++++++++++ 3 files changed, 159 insertions(+) create mode 100644 tests/test_wav_ds64_reservation.py diff --git a/src/acidcat/core/forensics/provenance.py b/src/acidcat/core/forensics/provenance.py index ecb8133a..2fe981f9 100644 --- a/src/acidcat/core/forensics/provenance.py +++ b/src/acidcat/core/forensics/provenance.py @@ -210,6 +210,25 @@ def _ffmpeg_rf64_junk(chunks): return None +def _ableton_junk_quote(chunks): + """Ableton Live writes a 28-character quote ("The sleeper must awaken", + "Why r u using a hex editor?") into the RF64 reservation of the WAVs it + renders; measured in two unrelated Live projects. A later tool may write + the RIFF size over its first 8 bytes, and the tail still names Live. The + walker has decoded the text; this reads its summary.""" + first = chunks[0] if chunks else {} + if str(first.get("id", "")).strip() != "JUNK": + return None + s = first.get("summary", "") + if s.startswith("RF64 reservation (ds64) holding text"): + basis = "a quote in the 28-byte RF64 reservation" + elif "written over a text" in s: + basis = "the tail of a quote under a later tool's RF64 sizes" + else: + return None + return {"tool": "Ableton Live", "confidence": "likely", "basis": basis} + + def _structural(label, chunks, data): out = [] if "MP3" in label or "MPEG" in label: @@ -232,6 +251,9 @@ def _structural(label, chunks, data): ff = _ffmpeg_rf64_junk(chunks) if ff: out.append(ff) + live = _ableton_junk_quote(chunks) + if live: + out.append(live) return out diff --git a/src/acidcat/core/walk/wav.py b/src/acidcat/core/walk/wav.py index d22ba519..ff44d466 100644 --- a/src/acidcat/core/walk/wav.py +++ b/src/acidcat/core/walk/wav.py @@ -769,6 +769,69 @@ def _parse_padding(b, ctx): """JUNK / FLLR / PAD, through the reader every container shares.""" return parse_padding(b) + +def _printable(b): + return all(0x20 <= x < 0x7F for x in b) + + +def _parse_ds64_reservation(b, riff_size): + """A 28-byte JUNK straight after the RIFF header, or None. + + 28 bytes is exactly a ds64 payload (riffSize, dataSize and sampleCount as + u64, then a u32 table length), and EBU Tech 3306 has a writer put a JUNK of + that size first so the file can become RF64 past 4 GB by renaming it. Read + as padding, every one that is not zero was reported as something + overwritten in place, but writers fill it three ways on purpose: + + - the ds64 values themselves, already true for this file; + - 28 characters of text: Ableton Live writes a quote here ("The sleeper + must awaken", "Why r u using a hex editor?"), on the renders it makes; + - the RIFF size written over the first 8 bytes of such a text, which is a + later tool preparing the ds64 in a file Live wrote. + Anything else is left to the shared padding reader.""" + if len(b) < 28 or not b.strip(b"\x00"): + return None + r64, d64, n64 = struct.unpack_from(" Date: Mon, 28 Sep 2026 16:59:22 -0400 Subject: [PATCH 16/23] fix(wav): appended bytes past the RIFF end are not chunks The walker read every 8 bytes past the declared end as a chunk header: 2 MB of appended zeros became 262,144 empty chunks, the Document took seconds to build, and the TUI mount test ran past its 60 s budget on the slowest CI job (under coverage) five runs in a row. A chunk id is four ASCII characters, so the walk stops at the first that is not; the bytes stay past the container end, where the scan names them as trailing data. The TUI tests built their huge chunk count from those phantom chunks; they now use 65,536 real empty JUNK chunks inside the RIFF. The wav-findings golden shot loses its five phantom hex chunks and keeps the polyglot alert. --- src/acidcat/core/walk/wav.py | 20 ++++++++++---------- tests/test_anomalies.py | 14 ++++++++++++++ tests/test_tui_fundamentals.py | 22 +++++++++++----------- tests/test_wav_ds64_reservation.py | 8 ++++---- 4 files changed, 39 insertions(+), 25 deletions(-) diff --git a/src/acidcat/core/walk/wav.py b/src/acidcat/core/walk/wav.py index ff44d466..4c8e1148 100644 --- a/src/acidcat/core/walk/wav.py +++ b/src/acidcat/core/walk/wav.py @@ -807,19 +807,14 @@ def _parse_ds64_reservation(b, riff_size): return "RF64 reservation (ds64), filled in with this file's sizes", fields, [] return (f"RF64 reservation (ds64), filled in when the file was " f"{r64 + 8:,} bytes", fields, - [info("convention.noted", - f"the ds64 riff_size is {r64:,} and the RIFF header says " - f"{riff_size:,}: chunks were added after the sizes were " - f"written, so an RF64 conversion must rewrite them")]) + []) if len(b) != 28: return None if _printable(b): text = b.decode("ascii") return (f"RF64 reservation (ds64) holding text: {text.rstrip()!r}", [ _f(0x00, 28, "text", text, "Ableton Live writes a quote here"), - ], [info("convention.noted", - "the RF64 reservation holds a text, not sizes; Ableton Live " - "fills it this way")]) + ], []) if r64 == riff_size and _printable(b[8:]): text = b[8:].decode("ascii") return (f"RF64 reservation (ds64): the RIFF size written over a text " @@ -827,9 +822,7 @@ def _parse_ds64_reservation(b, riff_size): _f(0x00, 8, "riff_size", r64, enc="= riff_size + 8 + and str(cid).startswith("hex:")): + # appended bytes, not chunks: a chunk id is four ASCII + # characters. Reading on turned 2 MB of appended zeros into + # 262,144 empty "chunks"; they stay past the container end, + # where the forensic scan reports them as trailing data. + break if size > avail and not (cid == "data" and size in _STREAM_SENTINELS): file_warns.append( f"chunk {cid!r} at 0x{offset:08x} claims {size:,} bytes " diff --git a/tests/test_anomalies.py b/tests/test_anomalies.py index 5c2c6442..06730a2a 100644 --- a/tests/test_anomalies.py +++ b/tests/test_anomalies.py @@ -636,3 +636,17 @@ def test_a_zip_appended_to_a_wav_is_still_a_polyglot(tmp_path): p = tmp_path / "carrier.wav" p.write_bytes(host + _zip_with_wav(tmp_path, "inner.zip").read_bytes()) assert "polyglot" in {f["rule"] for f in anomalies.scan(str(p))} + + +def test_appended_zeros_are_trailing_data_not_chunks(tmp_path): + """The walker read every 8 bytes past the RIFF end as a chunk: 2 MB of + appended zeros became 262,144 empty chunks, and building the Document took + seconds. A chunk id is four ASCII characters, so the walk stops at the + first one that is not; the bytes stay past the container end, where the + scan still names them.""" + host = _wav(_FMT, _chunk(b"data", b"\x00\x01" * 64)) + p = tmp_path / "padded.wav" + p.write_bytes(host + bytes(2 * 1024 * 1024)) + _label, chunks, _w = walk_file(str(p)) + assert [c["id"].strip() for c in chunks] == ["fmt", "data"] + assert "trailing_data" in {f["rule"] for f in anomalies.scan(str(p))} diff --git a/tests/test_tui_fundamentals.py b/tests/test_tui_fundamentals.py index 8333f757..c0298af5 100644 --- a/tests/test_tui_fundamentals.py +++ b/tests/test_tui_fundamentals.py @@ -306,21 +306,21 @@ def test_the_footer_only_advertises_keys_that_work(keysweep): # ── mounting must not scale with a hostile chunk count ───────────── -def _null_tailed_wav(path, tail_bytes): - """A valid WAV with a run of nulls appended. +def _many_chunk_wav(path, n_chunks): + """A valid WAV holding n_chunks empty JUNK chunks inside its RIFF. - The deep walker reads each 8-byte run of zeros as a zero-size chunk, so - this is the cheapest way to make a file with an enormous chunk count. It is - not contrived: a truncated or zero-padded file from a failed transfer looks - exactly like this, and opening damaged files is the tool's purpose. + The cheapest real file with an enormous chunk count. It was built from + nulls appended past the RIFF end, which the walker used to read as + zero-size chunks; appended bytes are trailing data, not chunks, so the walk + now stops there, and the count has to come from chunks the RIFF declares. """ import struct pcm = b"\x00\x01" * 200 body = (b"WAVE" + b"fmt " + struct.pack(" Date: Mon, 28 Sep 2026 19:51:30 -0400 Subject: [PATCH 17/23] fix(sigmf, vital): the backported fixes use 1.8's paths Both were written against 2.0's Source layer: the SigMF sidecar view read a Source object and the vitalskin check called input_name, neither of which 1.8 has, so opening a .sigmf-meta or sniffing any Vital preset raised NameError. The RF64-reservation notes are dropped, 1.8 having no info kind; the summary line says what the reservation holds. --- src/acidcat/core/infra/sniff.py | 2 +- src/acidcat/core/walk/sigmf.py | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/acidcat/core/infra/sniff.py b/src/acidcat/core/infra/sniff.py index 4d8822d6..ba7d64ad 100644 --- a/src/acidcat/core/infra/sniff.py +++ b/src/acidcat/core/infra/sniff.py @@ -302,7 +302,7 @@ def _is_vital(filepath): # A .vitalskin (Vital's UI theme) also carries synth_version, but it is a # table of colours and sizes with no `settings`: read as a preset, every key # came back as an unknown one, "carrier" alerts included. - if input_name(filepath).lower().endswith(".vitalskin"): + if str(filepath).lower().endswith(".vitalskin"): return False try: with open(filepath, "rb") as fh: diff --git a/src/acidcat/core/walk/sigmf.py b/src/acidcat/core/walk/sigmf.py index 2e57a37c..ab66e53e 100644 --- a/src/acidcat/core/walk/sigmf.py +++ b/src/acidcat/core/walk/sigmf.py @@ -325,18 +325,19 @@ def inspect_sigmf(path, deep=False): warns.append(f"listing the first {_ANNOTATION_CAP} of " f"{len(annotations)} annotations") - if meta is not None and meta is src: - return _as_sidecar(chunks, meta, data_src), warns + if str(path).lower().endswith(".sigmf-meta") and os.path.isfile(meta_path): + return _as_sidecar(chunks, meta_path, data_path), warns samples, _ = _samples_chunk(data_path, data_size, geo, dt, deep) chunks.append(samples) return chunks, warns -def _as_sidecar(chunks, meta, data_src): +def _as_sidecar(chunks, meta_path, data_path): """The walk seen from the .sigmf-meta: its own bytes are the JSON. The captures and annotations locate bytes of the .sigmf-data, so here they are listed but not placed, and their pointers are not followed into this file.""" - where = os.path.basename(data_src.name) if data_src is not None else "the .sigmf-data" + where = os.path.basename(data_path) if os.path.isfile(data_path) else "the .sigmf-data" + meta_size = os.path.getsize(meta_path) for c in chunks: if c["id"] == "global": continue @@ -344,8 +345,8 @@ def _as_sidecar(chunks, meta, data_src): c["summary"] += f" (in {where})" for f in c["fields"]: f.pop("xref", None) - chunks.append({"id": "sidecar", "offset": 0, "size": meta.size, "payload_base": 0, - "summary": f"SigMF metadata, {meta.size:,} bytes of JSON; " + chunks.append({"id": "sidecar", "offset": 0, "size": meta_size, "payload_base": 0, + "summary": f"SigMF metadata, {meta_size:,} bytes of JSON; " f"the samples are in {where}", "fields": [], "warnings": []}) return chunks From 3dec2d906ec83614c42e555a92d130bfc97e6e0f Mon Sep 17 00:00:00 2001 From: hed0rah <18272116+hed0rah@users.noreply.github.com> Date: Mon, 28 Sep 2026 20:32:09 -0400 Subject: [PATCH 18/23] Release 1.8.7 --- ARCHITECTURE.md | 2 +- CHANGELOG.md | 43 +++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 2 +- src/acidcat/__init__.py | 2 +- 4 files changed, 46 insertions(+), 3 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index a02f50f0..c9bd076c 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -6,7 +6,7 @@ see exactly what a file is, flag anomalies, and edit or repair its structure. Closer to readelf / 010 Editor / radare2's format layer than to exiftool, with some optional audio analysis (BPM/key via librosa). -v1.8.6 · ~56k source LOC · ~52k test LOC · one hard dependency (`mutagen`); +v1.8.7 · ~56k source LOC · ~52k test LOC · one hard dependency (`mutagen`); everything heavier is an optional, lazily imported extra, so `import acidcat` pulls only the stdlib core. diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c639605..fb26d7a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,49 @@ All notable changes to acidcat. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); the project will adopt [Semantic Versioning](https://semver.org/spec/v2.0.0.html) at 1.0. +## [1.8.7] - 2026-09-28 + +Bug fixes only, found by running acidcat over a curated library of real files +from every format it reads. No new features and no changes in behaviour +beyond the fixes. + +### Fixed + +- **`acidcat inspect` crashed on every SoundFont**, and so did `inspect + --full`, `--json` and `od`: the preset and instrument tree has no byte + position, and the renderers formatted its offset as a number. A test now + runs every render mode over every seed format. +- **A WAV's RF64 reservation was reported as damage.** The 28-byte JUNK a + writer puts first is filled on purpose: with the ds64 sizes (sometimes + stale, the file having grown), with a quote Ableton Live writes there, or + with the RIFF size over such a quote. It is now read as the reservation, + and the quote names Ableton Live as the writer in `audit`. +- **Bytes appended past a WAV's RIFF end were read as chunks.** 2 MB of + appended zeros became 262,144 empty chunks and opening the file took + seconds. The walk now stops at the first id that is not four ASCII + characters; the bytes are still reported as trailing data. +- **Zip-based formats were reported as polyglots** (.xpn, .labx, + .multisample): the archive's own end record read as an appended zip, its + members as smuggled files. An SF3's Ogg samples were flagged the same way. +- **A Max for Live instrument was a "magic mismatch".** The four bytes at 8 + are the device type: `aaaa` for an audio effect, `iiii` for an instrument. +- **A `.vitalskin` was read as a Vital preset**, every theme key flagged as + an unknown one. +- **A cut BRSTM was reported as whole.** The header's file size at 0x08 is + now read, and a file shorter than it declares says the stream is cut. +- **A SigMF recording opened by its `.sigmf-meta` was reported as broken.** +- **Every 12-bit AIFF and WAV was reported as damaged**: the size checks + rounded 12 bits down to one byte per sample. +- **Two walkers found wrong by McGill's public test files**: an AU whose + data runs past the end of the file, and an AIFF APPL chunk without a + fitting name. +- **76 X68000 PDX banks with a short slot table, and 16 cut MDX modules, + were not recognised.** +- **`constraints.repair` raised on a multi-track MP4** instead of declining + it, and MP4 timestamps used a call Python 3.12 deprecates. +- **`audit` called a console ROM `[unknown]`**; it now names the ROM and + suggests `acidcat extract`. + ## [1.8.6] - 2026-09-25 ### Fixed diff --git a/pyproject.toml b/pyproject.toml index 4bdb3e28..be7bbce7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "acidcat" -version = "1.8.6" +version = "1.8.7" description = "Binary analysis and reverse-engineering for audio file formats and music-production hardware" readme = {file = "README.md", content-type = "text/markdown"} license = {text = "MIT"} diff --git a/src/acidcat/__init__.py b/src/acidcat/__init__.py index bc205830..494caeea 100644 --- a/src/acidcat/__init__.py +++ b/src/acidcat/__init__.py @@ -46,7 +46,7 @@ See docs/format_internals.md for the formats acidcat walks. """ -__version__ = "1.8.6" +__version__ = "1.8.7" # dissection namespaces from acidcat.core import probe # noqa: E402,F401 From 2645671371b20f32e97088cd6a37123194edfb53 Mon Sep 17 00:00:00 2001 From: hed0rah <18272116+hed0rah@users.noreply.github.com> Date: Sat, 26 Sep 2026 12:32:06 -0400 Subject: [PATCH 19/23] test: a repair-guard refusal is an answer, not a crash AudioGuardError is how repair declines a file whose audio it cannot locate; the repair verb reports it as nothing written. The corpus sweep counted it as a crash because no real file had tripped the guard until a malformed AIFC from McGill's public set (its COMM chunk is four bytes short). --- tests/test_secondary_path_hunt.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/test_secondary_path_hunt.py b/tests/test_secondary_path_hunt.py index 64e15c1a..cb5c6f8c 100644 --- a/tests/test_secondary_path_hunt.py +++ b/tests/test_secondary_path_hunt.py @@ -103,6 +103,7 @@ def test_no_real_file_crashes_the_repair_path(): degrade internally, never raise on a real file. Output is discarded; the file is never modified.""" from acidcat.core.write import constraints + from acidcat.core.write.repairers import AudioGuardError def run(path, fid): try: @@ -111,7 +112,13 @@ def run(path, fid): except OSError: return constraints.analyze(data) - constraints.repair(data) + try: + constraints.repair(data) + except AudioGuardError: + # the guard refusing to write is the designed answer for a file + # whose audio it cannot locate (the repair verb reports it as + # "nothing written"); first seen on a malformed AIFC from McGill + pass tried, crashes = _sweep(_REPAIRABLE, run) if not tried: From 8cce5be50aa00667533e4f0d6001df154cef7ac4 Mon Sep 17 00:00:00 2001 From: hed0rah <18272116+hed0rah@users.noreply.github.com> Date: Sun, 27 Sep 2026 01:12:33 -0400 Subject: [PATCH 20/23] test(geometry): a format walk_file refuses is not a crash The hunt sweep counted Unsupported as a walker raising, so the first SNES ROM in the corpus failed it. A ROM is recognised and deliberately not walked; the refusal is an answer. --- tests/test_chunk_geometry.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_chunk_geometry.py b/tests/test_chunk_geometry.py index bead6ece..f1d341b2 100644 --- a/tests/test_chunk_geometry.py +++ b/tests/test_chunk_geometry.py @@ -50,7 +50,7 @@ import pytest from acidcat.core.infra import geometry -from acidcat.core.walk import walk_file +from acidcat.core.walk import Unsupported, walk_file _MAX = 8_000_000 @@ -170,6 +170,10 @@ def _hunt(): continue try: label, chunks, warns = walk_file(path, deep=False) + except Unsupported: + # a format recognised but deliberately not walked (a ROM + # says to use `extract`): a refusal is an answer, not a crash + continue except Exception as exc: # noqa: BLE001 (recorded, asserted below) HUNT_RAISED.append((path, repr(exc))) continue From aff3c6dafc6a71f0a92ad6e636ccde3708e17adf Mon Sep 17 00:00:00 2001 From: hed0rah <18272116+hed0rah@users.noreply.github.com> Date: Mon, 28 Sep 2026 21:01:19 -0400 Subject: [PATCH 21/23] fix(check): a valid RF64 is not a broken RIFF; a lost audio chunk is not repairable RF64 keeps its sizes in ds64 and writes 0xFFFFFFFF in the 32-bit fields; the structure model read them literally, so every valid RF64 (one libsndfile wrote, here) failed check with the data chunk overrunning the file and an 88-byte RIFF size proposed. check declines RF64 with a note, as edit does. A real CCRMA AIFC declares COMM as 18 bytes but writes the AIFC layout, so SSND is never reached; check proposed shrinking FORM and printed 'fix with check --fix', which repair then refused. analyze now marks nothing repairable when apply would refuse, and says why. --- src/acidcat/core/write/repairers.py | 33 +++++++++++++++- tests/test_rf64_check.py | 59 +++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 tests/test_rf64_check.py diff --git a/src/acidcat/core/write/repairers.py b/src/acidcat/core/write/repairers.py index f281d8c4..90404422 100644 --- a/src/acidcat/core/write/repairers.py +++ b/src/acidcat/core/write/repairers.py @@ -75,6 +75,18 @@ def _orphan_violation(node, n_bytes): f"Recomputing the size here would orphan them.")) +def _lost_audio_violation(node): + want = _IFF_AUDIO.get(node.form_type, b"data").decode("latin-1") + return Violation( + SIZE, f"{node.form_type.decode('latin-1', 'replace')}/{want}", + "declared_size", None, None, + witness="", # NOT repairable: the walk lost the audio + detail=(f"the {want} chunk is in the file but the chunk walk never " + f"reaches it: a size before it is wrong, so every chunk after " + f"that one is misread. Recomputing the container size would " + f"cut the audio off.")) + + def _iff_violation(change): """Map a structure.recompute change to a Violation. A top-level (master) size is witnessed by end-of-file; a nested size by its container's parsed @@ -90,17 +102,32 @@ def _iff_violation(change): class IffRepairer(Repairer): label = "IFF" + # RF64 keeps its real sizes in the ds64 chunk and writes 0xFFFFFFFF in the + # 32-bit fields. The structure model reads those fields literally, so every + # valid RF64 failed validation ("data overruns the file") and repair + # proposed an 88-byte RIFF size. Declined, as the tag editor declines it. + _DS64_NOTE = ("RF64 keeps its sizes in the ds64 chunk; the 32-bit size " + "fields are placeholders, and checking them is not modelled") def applies(self, data): return structure.is_iff(data) + def _is_rf64(self, data): + return bytes(data[:4]) == b"RF64" + def _report(self, data, opts): node = structure.parse(data) orphan = _orphaned_audio(node) changes = structure.recompute(node, normalize_pad=not (opts or {}).get("keep_pad")) label = node.form_type.decode("latin-1", "replace") violations = [_iff_violation(c) for c in changes] - if orphan: + want = _IFF_AUDIO.get(node.form_type) + if want and _iff_audio(node) is None and want in bytes(data): + # apply() refuses exactly this file (audio present, not in the + # tree); analyze must not advertise a fix it will refuse + violations = [replace(v, witness="") for v in violations] + violations.insert(0, _lost_audio_violation(node)) + elif orphan: # The master-size change is the destructive one, so it must stop # advertising itself as repairable -- otherwise validate and audit # both print "fix with: acidcat repair" for a file repair refuses, @@ -111,10 +138,14 @@ def _report(self, data, opts): return node, violations, label, orphan def analyze(self, data, opts=None): + if self._is_rf64(data): + return Report("WAVE", note=self._DS64_NOTE) _node, violations, label, _orphan = self._report(data, opts) return Report(label, violations) def apply(self, data, opts=None): + if self._is_rf64(data): + return data, Report("WAVE", note=self._DS64_NOTE) node, violations, label, orphan = self._report(data, opts) if orphan: want = _IFF_AUDIO.get(node.form_type, b"data").decode("latin-1") diff --git a/tests/test_rf64_check.py b/tests/test_rf64_check.py new file mode 100644 index 00000000..1b45e2ae --- /dev/null +++ b/tests/test_rf64_check.py @@ -0,0 +1,59 @@ +"""A valid RF64 is not a broken RIFF. + +RF64 writes 0xFFFFFFFF in the 32-bit RIFF and data size fields and keeps the +real sizes in the ds64 chunk. The IFF structure model read those fields +literally, so every valid RF64 failed validation with "the data chunk overruns +the file" and repair proposed an 88-byte RIFF size. Found on a file libsndfile +wrote. The checker now declines RF64 with a note, as the tag editor does.""" +import struct + +from acidcat.core.write import constraints + +SENTINEL = 0xFFFFFFFF + + +def _rf64(n_audio=64): + fmt = struct.pack("hIh", 1, 16, 16) + bytes(10) + b"NONE" + b"\x0enot compressed" + b"\x00" + ssnd = struct.pack(">II", 0, 0) + bytes(32) + body = (b"AIFC" + b"COMM" + struct.pack(">I", 18) + comm + + b"SSND" + struct.pack(">I", len(ssnd)) + ssnd) + data = b"FORM" + struct.pack(">I", len(body)) + body + report = constraints.analyze(data) + assert report.violations + assert not any(v.repairable for v in report.violations) + assert "never reaches it" in report.violations[0].detail From 69d9834f229d4ee85628f94f55ca0dd33e7e0ef3 Mon Sep 17 00:00:00 2001 From: hed0rah <18272116+hed0rah@users.noreply.github.com> Date: Mon, 28 Sep 2026 21:01:57 -0400 Subject: [PATCH 22/23] docs(changelog): 1.8.7 gains the RF64 and lost-audio check fixes --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb26d7a0..e27feebd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,14 @@ beyond the fixes. --full`, `--json` and `od`: the preset and instrument tree has no byte position, and the renderers formatted its offset as a number. A test now runs every render mode over every seed format. +- **Every valid RF64 failed `validate`.** RF64 writes 0xFFFFFFFF in its + 32-bit size fields and keeps the real sizes in ds64; the checker read the + placeholders literally, reported the data chunk as overrunning the file, + and proposed an 88-byte RIFF size (repair refused to write it). RF64 is now + declined with a note, as `write` already declines it. +- **`validate` advertised a fix `repair` would refuse** on a file whose audio + chunk the walk cannot reach (a wrong size earlier in the file). It now + explains the misread and marks nothing repairable. - **A WAV's RF64 reservation was reported as damage.** The 28-byte JUNK a writer puts first is filled on purpose: with the ds64 sizes (sometimes stale, the file having grown), with a quote Ableton Live writes there, or From 5391e9b5e3f8ccb779b3939976d7994a965731ea Mon Sep 17 00:00:00 2001 From: hed0rah <18272116+hed0rah@users.noreply.github.com> Date: Mon, 28 Sep 2026 21:18:07 -0400 Subject: [PATCH 23/23] fix(check): an overrunning audio chunk keeps its own explanation The lost-audio branch ran first, and an overrunning data chunk also sits outside the parse tree, so a crash-truncated WAV lost its 'outside the container' diagnosis. The orphan case is checked first again. --- src/acidcat/core/write/repairers.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/acidcat/core/write/repairers.py b/src/acidcat/core/write/repairers.py index 90404422..73856817 100644 --- a/src/acidcat/core/write/repairers.py +++ b/src/acidcat/core/write/repairers.py @@ -122,12 +122,7 @@ def _report(self, data, opts): label = node.form_type.decode("latin-1", "replace") violations = [_iff_violation(c) for c in changes] want = _IFF_AUDIO.get(node.form_type) - if want and _iff_audio(node) is None and want in bytes(data): - # apply() refuses exactly this file (audio present, not in the - # tree); analyze must not advertise a fix it will refuse - violations = [replace(v, witness="") for v in violations] - violations.insert(0, _lost_audio_violation(node)) - elif orphan: + if orphan: # The master-size change is the destructive one, so it must stop # advertising itself as repairable -- otherwise validate and audit # both print "fix with: acidcat repair" for a file repair refuses, @@ -135,6 +130,13 @@ def _report(self, data, opts): # violation, reported first, explains why. violations = [replace(v, witness="") for v in violations] violations.insert(0, _orphan_violation(node, orphan)) + elif want and _iff_audio(node) is None and want in bytes(data): + # apply() refuses exactly this file (audio present, not in the + # tree); analyze must not advertise a fix it will refuse. Checked + # after the orphan case, whose overrunning chunk also sits outside + # the tree and has the more specific explanation. + violations = [replace(v, witness="") for v in violations] + violations.insert(0, _lost_audio_violation(node)) return node, violations, label, orphan def analyze(self, data, opts=None):