merge

.gemini/settings.json

@@ -0,0 +1,5 @@
+{
+  "context": {
+    "fileName": "AGENTS.md"
+  }
+}

.gitignore

@@ -0,0 +1,6 @@
+*.part
+*.bak
+*.pyc
+*.log
+__MACOSX/
+/wiscsee/leaftl_scripts/leaftl_traces/

.mise.toml

@@ -0,0 +1,6 @@
+[tools]
+python = "pypy3.11-7.3.20"
+uv = "latest"
+
+[env]
+_.python.venv = ".venv"

AGENTS.md

@@ -0,0 +1 @@
+用 rg 别用 grep

bench_selected.py

@@ -0,0 +1,170 @@
+#!/usr/bin/env python3
+import os
+import sys
+import subprocess
+import re
+import glob
+import json
+
+
+def run_and_stream(cmd, env):
+    """Runs a command and streams output to stdout while capturing for parsing."""
+    full_output = []
+    # Make sure we use the same python interpreter for child processes
+    if cmd[0] == sys.executable:
+        pass  # already set
+
+    try:
+        process = subprocess.Popen(
+            cmd,
+            env=env,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            text=True,
+            bufsize=1,
+        )
+
+        for line in process.stdout:
+            print(line, end="", flush=True)
+            full_output.append(line)
+
+        process.wait()
+        return "".join(full_output)
+    except Exception as e:
+        print(f"\nError running command: {e}")
+        return None
+
+
+def main():
+    # Set up directory
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+    os.chdir(script_dir)
+
+    print(f"Python Integrity Check: {sys.executable}")
+    print(f"Version: {sys.version}")
+
+    # Add wiscsee to PYTHONPATH
+    env = os.environ.copy()
+    wiscsee_path = os.path.join(script_dir, "wiscsee")
+    if "PYTHONPATH" in env:
+        env["PYTHONPATH"] = env["PYTHONPATH"] + os.pathsep + wiscsee_path
+    else:
+        env["PYTHONPATH"] = wiscsee_path
+
+    # Traces directory
+    data_dir = os.path.join(script_dir, "data")
+    run_ftl_script = os.path.join("wiscsee", "leaftl_scripts", "run_ftl")
+
+    # Results directory
+    json_dir = os.path.join(script_dir, "json")
+    if not os.path.exists(json_dir):
+        os.makedirs(json_dir)
+
+    # Scan for all .bin files generated by init.js
+    trace_files = sorted(glob.glob(os.path.join(data_dir, "*.bin")))
+
+    if not trace_files:
+        print(f"No .bin files found in {data_dir}")
+        return
+
+    print("\nStarting Benchmarks...\n")
+    print("Trace, DFTL (MB), SFTL (MB), LeaFTL (MB)")
+
+    results = []
+
+    for trace_path in trace_files:
+        trace_name = os.path.basename(trace_path)
+        trace_base = os.path.splitext(trace_name)[0]
+        result_file = os.path.join(json_dir, f"{trace_base}.json")
+
+        if os.path.exists(result_file):
+            print(
+                f"\n>>> Skipping {trace_name} (result already exists in {result_file})"
+            )
+            continue
+
+        print(f"\n>>> Processing {trace_name}")
+
+        # 1. Run LearnedFTL (provides LeaFTL and DFTL sizes)
+        cmd_lea = [
+            sys.executable,
+            run_ftl_script,
+            "-f",
+            "learnedftl",
+            "-t",
+            trace_path,
+            "-sl",
+            "0",
+        ]
+        output_lea = run_and_stream(cmd_lea, env)
+
+        leaftl_mb = 0.0
+        dftl_mb = 0.0
+        sftl_mb = 0.0
+
+        if output_lea:
+            # Match both normal and partial stats on crash
+            lea_match = re.search(
+                r"estimated learnedftl memory footprint: (\d+) B", output_lea
+            )
+            if lea_match:
+                leaftl_mb = float(lea_match.group(1)) / (1024 * 1024)
+
+            dftl_match = re.search(
+                r"estimated dftl memory footprint: (\d+) B", output_lea
+            )
+            if dftl_match:
+                dftl_mb = float(dftl_match.group(1)) / (1024 * 1024)
+
+        # 2. Run SFTL (provides SFTL size)
+        cmd_sftl = [
+            sys.executable,
+            run_ftl_script,
+            "-f",
+            "sftl",
+            "-t",
+            trace_path,
+            "-sl",
+            "0",
+        ]
+        output_sftl = run_and_stream(cmd_sftl, env)
+
+        if output_sftl:
+            sftl_match = re.search(
+                r"estimated learnedftl memory footprint: (\d+) B", output_sftl
+            )
+            if sftl_match:
+                sftl_mb = float(sftl_match.group(1)) / (1024 * 1024)
+
+        ratio = dftl_mb / leaftl_mb if leaftl_mb > 0 else 0
+        summary_line = f"SUMMARY: {trace_name}, DFTL: {dftl_mb:.4f} MB, SFTL: {sftl_mb:.4f} MB, LeaFTL: {leaftl_mb:.4f} MB, Ratio(DFTL/LeaFTL): {ratio:.2f}x"
+        print(summary_line)
+
+        res_obj = {
+            "trace": trace_name,
+            "dftl_mb": round(dftl_mb, 4),
+            "sftl_mb": round(sftl_mb, 4),
+            "leaftl_mb": round(leaftl_mb, 4),
+            "ratio": round(ratio, 2),
+        }
+        results.append(res_obj)
+
+        # Save individual result immediately
+        with open(result_file, "w") as f:
+            json.dump(res_obj, f, indent=4)
+
+        sys.stdout.flush()
+
+    # Save to JSON
+    json_path = "bench_results.json"
+    with open(json_path, "w") as f:
+        json.dump(results, f, indent=4)
+
+    print(f"\nFinal results saved to {json_path}")
+
+
+if __name__ == "__main__":
+    main()
+
+if __name__ == "__main__":
+    main()

data

@@ -0,0 +1 @@
+../jdb_ftl/data

full_fast_replay.py

@@ -0,0 +1,147 @@
+#!/usr/bin/env python3
+import os
+import sys
+import struct
+import json
+import glob
+import time
+from collections import defaultdict
+import numpy as np
+
+# Add current dir to path to find wiscsee
+current_dir = os.path.dirname(os.path.abspath(__file__))
+wiscsee_path = os.path.join(current_dir, "wiscsee")
+sys.path.append(wiscsee_path)
+
+from wiscsim.learnedftl import FlashMetadata
+import config
+
+class FastConfig(config.ConfigNCQFTL):
+    def __init__(self, internal_type="learnedftl"):
+        super().__init__()
+        self["flash_config"]["page_size"] = 4096
+        self["flash_config"]["n_pages_per_block"] = 128
+        self["flash_config"]["n_blocks_per_plane"] = 1000000 
+        self["gamma"] = 0.0001
+        self["mapping_cache_bytes"] = 2**40 # Large cache for analysis
+        self["internal_ftl_type"] = internal_type
+        self["ftl_type"] = "learnedftl"
+
+def fast_replay(trace_path):
+    trace_file_name = os.path.basename(trace_path)
+    print(f"\n[Replay] {trace_file_name}")
+
+    # 1. Initialize metadata for both FTL types
+    # DFTL size is calculated from unique LPNs
+    meta_lea = FlashMetadata(FastConfig("learnedftl"), defaultdict(float))
+    meta_sftl = FlashMetadata(FastConfig("sftl"), defaultdict(float))
+
+    # Binary op codes
+    OP_GET, OP_SET, OP_MGET, OP_MSET, OP_RM, OP_MRM = 0x00, 0x01, 0x02, 0x03, 0x04, 0x05
+
+    total_ops = 0
+    write_pages = 0
+    start_time = time.time()
+    last_print = start_time
+
+    MB_UNIT = 1024 * 1024
+
+    try:
+        with open(trace_path, "rb") as f:
+            while True:
+                op_buf = f.read(1)
+                if not op_buf: break
+                op = struct.unpack("B", op_buf)[0]
+
+                lba_buf = f.read(4)
+                if not lba_buf: break
+                lba = struct.unpack("<I", lba_buf)[0]
+
+                count = 1
+                is_write = False
+
+                if op == OP_SET:
+                    f.read(4) # skip PBA
+                    is_write = True
+                elif op == OP_MGET or op == OP_MRM:
+                    cnt_buf = f.read(4)
+                    if not cnt_buf: break
+                    count = struct.unpack("<I", cnt_buf)[0]
+                elif op == OP_MSET:
+                    cnt_buf = f.read(4)
+                    if not cnt_buf: break
+                    count = struct.unpack("<I", cnt_buf)[0]
+                    f.read(4) # skip PBA
+                    is_write = True
+
+                total_ops += 1
+                lpns = list(range(lba, lba + count))
+
+                if is_write:
+                    # Parallel Update
+                    meta_lea.update(lpns)
+                    meta_sftl.update(lpns)
+                    write_pages += count
+                else:
+                    # Parallel Lookup (only for access distribution simulation)
+                    ref = meta_lea.reference_mapping_table
+                    for lpn in lpns:
+                        if ref.get(lpn) is not None:
+                            meta_lea.lpn_to_ppn(lpn)
+                            meta_sftl.lpn_to_ppn(lpn)
+
+                # Progress reporting
+                now = time.time()
+                if now - last_print > 3.0:
+                    pct = (total_ops / 600000) * 100 # Rough progress for large traces
+                    print(f"  Processed {total_ops} ops ({write_pages} total pages written)...", end="\r")
+                    last_print = now
+
+    except Exception as e:
+        print(f"\n  Error: {e}")
+
+    # 2. Final memory calculations
+    lea_bytes = meta_lea.mapping_table.memory
+    sftl_bytes = meta_sftl.mapping_table.memory
+    unique_lpns = len(meta_lea.reference_mapping_table.mapping_table)
+    dftl_bytes = unique_lpns * 8
+
+    duration = time.time() - start_time
+    print(f"\n  Finished in {duration:.2f}s. Mapping Sizes (MB):")
+    print(f"    DFTL:    {dftl_bytes/MB_UNIT:.4f}")
+    print(f"    SFTL:    {sftl_bytes/MB_UNIT:.4f}")
+    print(f"    LeaFTL:  {lea_bytes/MB_UNIT:.4f}")
+
+    return {
+        "trace": trace_file_name,
+        "dftl_mb": round(dftl_bytes / MB_UNIT, 4),
+        "sftl_mb": round(sftl_bytes / MB_UNIT, 4),
+        "leaftl_mb": round(lea_bytes / MB_UNIT, 4)
+    }
+
+def main():
+    data_dir = os.path.join(current_dir, "data")
+    json_dir = os.path.join(current_dir, "json_fast_replay")
+    if not os.path.exists(json_dir): os.makedirs(json_dir)
+
+    traces = sorted(glob.glob(os.path.join(data_dir, "*.bin")))
+
+    results = []
+    for trace in traces:
+        res = fast_replay(trace)
+        results.append(res)
+
+        # Save to JSON
+        trace_base = os.path.splitext(os.path.basename(trace))[0]
+        with open(os.path.join(json_dir, f"{trace_base}.json"), "w") as f:
+            json.dump(res, f, indent=4)
+
+    print("\n" + "="*60)
+    print(f"{'Trace':<15} | {'DFTL (MB)':<10} | {'SFTL (MB)':<10} | {'LeaFTL (MB)':<10}")
+    print("-" * 60)
+    for r in results:
+        print(f"{r['trace']:<15} | {r['dftl_mb']:<10.4f} | {r['sftl_mb']:<10.4f} | {r['leaftl_mb']:<10.4f}")
+    print("="*60)
+
+if __name__ == "__main__":
+    main()