Skip to content

docs: update readme, first_diff.py script #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
zestydevy merged 1 commit into from
May 3, 2026
Merged

docs: update readme, first_diff.py script #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,5 @@ tools/ido_recomp
*_auto.txt
*.s
.DS_Store
*.ld
*.ld
.venv/
12 changes: 11 additions & 1 deletion README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,20 @@ Required base ROM file:

Convert a Mario Artist Paint Studio NDD dump to D64 with [leo64dd_python](https://github.com/LuigiBlood/leo64dd_python).
The RAM area should be only `0xFF`.

You can create the dmpj.d64 file by taking a .ndd file (md5 `8485643e5830cd67ed4c0a5fd49e2491`) and running `python3 leo64ddfile.py d64 <path-to-ndd-file> <path-to-repo>/dmpj.d64`

## Quick Start

This repository uses Git submodules. Make sure to either clone the repository with the `--recurse-submodules` flag or to run `git submodule update --init --recursive` after the initial clone.

When pulling updates, you can update all submodules with `git submodule update --recursive`.

If dependencies are already installed and `dmpj.d64` is in the repo root:

```sh
python3 -m venv .venv
source .venv/bin/activate
python3 -m pip install -r requirements.txt
python3 -m pip install -r tools/splat/requirements.txt
make setup
make
Expand Down Expand Up @@ -96,6 +103,9 @@ brew install \
Install the Python dependencies:

```sh
python3 -m venv .venv
source .venv/bin/activate
python3 -m pip install -r requirements.txt
python3 -m pip install -r tools/splat/requirements.txt
```

Expand Down
1 change: 1 addition & 0 deletions requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
mapfile-parser>=2.4.0,<3.0.0
305 changes: 53 additions & 252 deletions tools/first_diff.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,255 +1,56 @@
#!/usr/bin/env python3

import os.path
# SPDX-FileCopyrightText: © 2022 AngheloAlf
# SPDX-License-Identifier: MIT

from __future__ import annotations

import argparse
from subprocess import check_call

parser = argparse.ArgumentParser(
description="Find the first difference(s) between the built ROM and the base ROM."
)
parser.add_argument(
"-c",
"--count",
type=int,
default=5,
help="find up to this many instruction difference(s)",
)
parser.add_argument(
"-d",
"--diff",
dest="diff_args",
nargs="?",
action="store",
default=False,
const="prompt",
help="run diff.py on the result with the provided arguments"
)
parser.add_argument(
"-m", "--make", help="run make before finding difference(s)", action="store_true"
)
args = parser.parse_args()

diff_count = args.count

if args.make:
check_call(["make", "build/dmpj.d64"])

baseimg = f"dmpj.d64"
basemap = f"expected/build/dmpj.map"

myimg = f"build/dmpj.d64"
mymap = f"build/dmpj.map"

if not os.path.isfile(baseimg):
print(f"{baseimg} must exist.")
exit(1)
if not os.path.isfile(myimg) or not os.path.isfile(mymap):
print(f"{myimg} and {mymap} must exist.")
exit(1)

mybin = open(myimg, "rb").read()
basebin = open(baseimg, "rb").read()

if len(mybin) != len(basebin):
print("Modified ROM has different size...")
exit(1)

if mybin == basebin:
print("No differences!")
exit(0)


def search_rom_address(target_addr):
ram_offset = None
prev_ram = 0
prev_rom = 0
prev_sym = "<start of rom>"
cur_file = "<no file>"
prev_file = cur_file
prev_line = ""
with open(mymap) as f:
for line in f:
if "load address" in line:
# Ignore .bss sections since we're looking for a ROM address
if ".bss" in line or ".bss" in prev_line:
ram_offset = None
continue
ram = int(line[16 : 16 + 18], 0)
rom = int(line[59 : 59 + 18], 0)
ram_offset = ram - rom
continue

prev_line = line

if (
ram_offset is None
or "=" in line
or "*fill*" in line
or " 0x" not in line
):
continue

ram = int(line[16 : 16 + 18], 0)
rom = ram - ram_offset
sym = line.split()[-1]

if "0x" in sym:
ram_offset = None
continue
if "/" in sym:
cur_file = sym
continue

if rom > target_addr:
return f"{prev_sym} (RAM 0x{prev_ram:X}, ROM 0x{prev_rom:X}, {prev_file})"

prev_ram = ram
prev_rom = rom
prev_sym = sym
prev_file = cur_file

return "at end of rom?"


def parse_map(map_fname):
ram_offset = None
cur_file = "<no file>"
syms = {}
prev_sym = None
prev_line = ""
with open(map_fname) as f:
for line in f:
if "load address" in line:
ram = int(line[16 : 16 + 18], 0)
rom = int(line[59 : 59 + 18], 0)
ram_offset = ram - rom
continue

prev_line = line

if (
ram_offset is None
or "=" in line
or "*fill*" in line
or " 0x" not in line
):
continue

ram = int(line[16 : 16 + 18], 0)
rom = ram - ram_offset
sym = line.split()[-1]

if "0x" in sym:
ram_offset = None
continue
elif "/" in sym:
cur_file = sym
continue

syms[sym] = (rom, cur_file, prev_sym, ram)
prev_sym = sym

return syms


def map_diff():
map1 = parse_map(mymap)
map2 = parse_map(basemap)
min_ram = None
found = None
for sym, addr in map1.items():
if sym not in map2:
continue
if addr[0] != map2[sym][0]:
if min_ram is None or addr[0] < min_ram:
min_ram = addr[0]
found = (sym, addr[1], addr[2])
if min_ram is None:
return False
else:
print(
f"Map appears to have shifted just before {found[0]} ({found[1]}) -- in {found[2]}?"
)
if found[2] is not None and found[2] not in map2:
print(
f"(Base map file {basemap} out of date due to new or renamed symbols, so result may be imprecise.)"
)
return True


def hexbytes(bs):
return ":".join("{:02X}".format(c) for c in bs)


found_instr_diff = []
map_search_diff = []
diffs = 0
shift_cap = 1000
for i in range(24, len(mybin), 4):
# (mybin[i:i+4] != basebin[i:i+4], but that's slightly slower in CPython...)
if diffs <= shift_cap and (
mybin[i] != basebin[i]
or mybin[i + 1] != basebin[i + 1]
or mybin[i + 2] != basebin[i + 2]
or mybin[i + 3] != basebin[i + 3]
):
if diffs == 0:
print(f"First difference at ROM addr 0x{i:X}, {search_rom_address(i)}")
print(
f"Bytes: {hexbytes(mybin[i : i + 4])} vs {hexbytes(basebin[i : i + 4])}"
)
diffs += 1
if (
len(found_instr_diff) < diff_count
and mybin[i] >> 2 != basebin[i] >> 2
and not search_rom_address(i) in map_search_diff
):
found_instr_diff.append(i)
map_search_diff.append(search_rom_address(i))

if diffs == 0:
print("No differences but ROMs differ?")
exit()

if len(found_instr_diff) > 0:
for i in found_instr_diff:
print(f"Instruction difference at ROM addr 0x{i:X}, {search_rom_address(i)}")
print(
f"Bytes: {hexbytes(mybin[i : i + 4])} vs {hexbytes(basebin[i : i + 4])}"
)
print()

definite_shift = diffs > shift_cap
if definite_shift:
print(f"Over {shift_cap} differing words, must be a shifted ROM.")
else:
print(f"{diffs} differing word(s).")

if diffs > 100:
if not os.path.isfile(basemap):
print(
f"To find ROM shifts, copy a clean .map file to {basemap} and rerun this script."
)
elif not map_diff():
print(f"No ROM shift{' (!?)' if definite_shift else ''}")

if args.diff_args:
if len(found_instr_diff) < 1:
print(f"No instruction difference to run diff.py on")
exit()

diff_sym = search_rom_address(found_instr_diff[0]).split()[0]
if args.diff_args == "prompt":
diff_args = input("Call diff.py with which arguments? ") or "--"
else:
diff_args = args.diff_args
if diff_args[0] != "-":
diff_args = "-" + diff_args
check_call(
[
"python3",
"diff.py",
diff_args,
diff_sym,
]
)
import os
import mapfile_parser
from pathlib import Path
import rabbitizer

SCRIPT_DIR = Path(os.path.dirname(os.path.realpath(__file__)))

def decodeInstruction(bytesDiff: bytes, mapFile: mapfile_parser.MapFile) -> str:
word = (bytesDiff[0] << 24) | (bytesDiff[1] << 16) | (bytesDiff[2] << 8) | (bytesDiff[3] << 0)
instr = rabbitizer.Instruction(word)
immOverride = None

if instr.isJumpWithAddress():
# Instruction is a function call (jal)

# Get the embedded address of the function call
symAddress = instr.getInstrIndexAsVram()

# Search for the address in the mapfile
symInfo = mapFile.findSymbolByVramOrVrom(symAddress)
if symInfo is not None:
# Use the symbol from the mapfile instead of a raw value
immOverride = symInfo.symbol.name

return instr.disassemble(immOverride=immOverride, extraLJust=-20)

def firstDiffMain():
parser = argparse.ArgumentParser(description="Find the first difference(s) between the built ROM and the base ROM.")

parser.add_argument("-c", "--count", type=int, default=5, help="find up to this many instruction difference(s)")
parser.add_argument("-a", "--add-colons", action='store_true', help="Add colon between bytes" )

args = parser.parse_args()

os.chdir(SCRIPT_DIR.joinpath("..").resolve())

buildFolder = Path("build")

BUILTROM = buildFolder / "dmpj.d64"
BUILTMAP = buildFolder / "dmpj.map"

EXPECTEDROM = "expected" / BUILTROM
EXPECTEDMAP = "expected" / BUILTMAP

mapfile_parser.frontends.first_diff.doFirstDiff(BUILTMAP, EXPECTEDMAP, BUILTROM, EXPECTEDROM, args.count, mismatchSize=True, addColons=args.add_colons, bytesConverterCallback=decodeInstruction)

if __name__ == "__main__":
firstDiffMain()