From 0a58007d20da4c1845e9b665f7c33fd894e56518 Mon Sep 17 00:00:00 2001 From: "js0.site" Date: Sat, 24 Jan 2026 05:28:48 +0800 Subject: [PATCH 1/9] Migrate codebase from Python 2 to Python 3 - Apply Python 3 syntax compatibility changes across all modules - Update print statements and exception handling syntax - Modernize string formatting and iteration patterns - Add project configuration files (.mise.toml, .gitignore) - Add development scripts (init.sh, ruff.sh) - Add AGENTS.md for agent configuration --- .gemini/settings.json | 5 + .gitignore | 6 + .mise.toml | 6 + AGENTS.md | 1 + bench_selected.py | 121 ++ data | 1 + init.js | 223 ++ init.sh | 36 + ruff.sh | 8 + wiscsee/commons.py | 61 +- wiscsee/config.py | 632 +++--- wiscsee/config_helper/experiment.py | 460 +++-- wiscsee/config_helper/rule_parameter.py | 377 ++-- wiscsee/leaftl_scripts/run_ftl | 4 +- wiscsee/pyreuse/apputils/fio.py | 59 +- .../pyreuse/apputils/parseleveldboutput.py | 54 +- wiscsee/pyreuse/fsutils/__init__.py | 2 +- wiscsee/pyreuse/fsutils/ext4dumpextents.py | 39 +- wiscsee/pyreuse/fsutils/formatfs.py | 25 +- wiscsee/pyreuse/general/maketree.py | 17 +- wiscsee/pyreuse/general/zipf.py | 10 +- wiscsee/pyreuse/helpers.py | 70 +- wiscsee/pyreuse/macros.py | 4 +- wiscsee/pyreuse/sysutils/blockclassifiers.py | 39 +- wiscsee/pyreuse/sysutils/blocktrace.py | 253 ++- wiscsee/pyreuse/sysutils/cgroup.py | 21 +- wiscsee/pyreuse/sysutils/dumpe2fsparser.py | 66 +- wiscsee/pyreuse/sysutils/filefragparser.py | 51 +- wiscsee/pyreuse/sysutils/ftrace.py | 34 +- wiscsee/pyreuse/sysutils/straceParser.py | 318 +-- wiscsee/tests/__init__.py | 16 +- wiscsee/tests/config_test.py | 32 +- wiscsee/tests/flashcontroller_test.py | 208 +- wiscsee/tests/lrulist_test.py | 43 +- wiscsee/tests/misc_test.py | 130 +- wiscsee/tests/recorder_test.py | 36 +- wiscsee/tests/simulator_test.py | 9 +- wiscsee/tests/test_bitmap.py | 32 +- wiscsee/tests/test_blockpool.py | 113 +- wiscsee/tests/test_demo.py | 281 +-- wiscsee/tests/test_devblockpool.py | 359 ++-- wiscsee/tests/test_dftldes.py | 1613 ++++++++------- wiscsee/tests/test_dftlext.py | 248 ++- wiscsee/tests/test_nkftl.py | 1812 +++++++++++------ wiscsee/tests/test_tagblockpool.py | 40 +- wiscsee/tests/test_workflow.py | 79 +- wiscsee/utilities/utils.py | 204 +- wiscsee/wiscsim/__init__.py | 6 +- wiscsee/wiscsim/bitmap.py | 23 +- wiscsee/wiscsim/blkpool.py | 66 +- wiscsee/wiscsim/controller.py | 304 +-- wiscsee/wiscsim/crb_standalone.py | 25 +- wiscsee/wiscsim/datacache.py | 224 +- wiscsee/wiscsim/devblockpool.py | 94 +- wiscsee/wiscsim/dftldes.py | 935 +++++---- wiscsee/wiscsim/dftlext.py | 605 +++--- wiscsee/wiscsim/flash.py | 34 +- wiscsee/wiscsim/ftlbuilder.py | 32 +- wiscsee/wiscsim/ftlcounter.py | 328 +-- wiscsee/wiscsim/ftlsim_commons.py | 52 +- wiscsee/wiscsim/gc_analysis.py | 62 +- wiscsee/wiscsim/host.py | 16 +- wiscsee/wiscsim/hostevent.py | 96 +- wiscsee/wiscsim/learnedftl.py | 822 +++++--- wiscsee/wiscsim/lrulist.py | 130 +- wiscsee/wiscsim/nkftl2.py | 808 ++++---- wiscsee/wiscsim/recorder.py | 114 +- wiscsee/wiscsim/sftl-standalone.py | 129 +- wiscsee/wiscsim/sftl.py | 80 +- wiscsee/wiscsim/simulator.py | 165 +- wiscsee/wiscsim/ssdframework.py | 348 ++-- wiscsee/wiscsim/tagblockpool.py | 36 +- wiscsee/wiscsim/utils.py | 14 +- wiscsee/wiscsim/warmup.py | 378 ++-- wiscsee/wiscsim/workload_parser.py | 227 ++- wiscsee/workflow.py | 26 +- wiscsee/workrunner/__init__.py | 4 +- wiscsee/workrunner/cpuhandler.py | 51 +- wiscsee/workrunner/filesystem.py | 111 +- wiscsee/workrunner/fshelper.py | 157 +- wiscsee/workrunner/lbaworkloadgenerator.py | 407 ++-- wiscsee/workrunner/multiwriters.py | 86 +- wiscsee/workrunner/nonblockingreader.py | 21 +- wiscsee/workrunner/wlrunner.py | 293 ++- wiscsee/workrunner/workload.py | 49 +- 85 files changed, 8978 insertions(+), 6638 deletions(-) create mode 100644 .gemini/settings.json create mode 100644 .gitignore create mode 100644 .mise.toml create mode 100644 AGENTS.md create mode 100755 bench_selected.py create mode 120000 data create mode 100755 init.js create mode 100755 init.sh create mode 100755 ruff.sh diff --git a/.gemini/settings.json b/.gemini/settings.json new file mode 100644 index 0000000..38396c7 --- /dev/null +++ b/.gemini/settings.json @@ -0,0 +1,5 @@ +{ + "context": { + "fileName": "AGENTS.md" + } +} \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0424aed --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +*.part +*.bak +*.pyc +*.log +__MACOSX/ +/wiscsee/leaftl_scripts/leaftl_traces/ diff --git a/.mise.toml b/.mise.toml new file mode 100644 index 0000000..cf508bb --- /dev/null +++ b/.mise.toml @@ -0,0 +1,6 @@ +[tools] +python = "pypy3.11-7.3.20" +uv = "latest" + +[env] +_.python.venv = ".venv" diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..5982691 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1 @@ +用 rg 别用 grep \ No newline at end of file diff --git a/bench_selected.py b/bench_selected.py new file mode 100755 index 0000000..bfb0b16 --- /dev/null +++ b/bench_selected.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +import os +import sys +import subprocess +import re +import glob +import json + +def run_and_stream(cmd, env): + """Runs a command and streams output to stdout while capturing for parsing.""" + full_output = [] + # Make sure we use the same python interpreter for child processes + if cmd[0] == sys.executable: + pass # already set + + try: + process = subprocess.Popen( + cmd, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1 + ) + + for line in process.stdout: + print(line, end='', flush=True) + full_output.append(line) + + process.wait() + return "".join(full_output) + except Exception as e: + print(f"\nError running command: {e}") + return None + +def main(): + # Set up directory + script_dir = os.path.dirname(os.path.abspath(__file__)) + os.chdir(script_dir) + + print(f"Python Integrity Check: {sys.executable}") + print(f"Version: {sys.version}") + + # Add wiscsee to PYTHONPATH + env = os.environ.copy() + wiscsee_path = os.path.join(script_dir, "wiscsee") + if "PYTHONPATH" in env: + env["PYTHONPATH"] = env["PYTHONPATH"] + os.pathsep + wiscsee_path + else: + env["PYTHONPATH"] = wiscsee_path + + # Traces directory + data_dir = os.path.join(script_dir, "data") + run_ftl_script = os.path.join("wiscsee", "leaftl_scripts", "run_ftl") + + # Scan for all .bin files generated by init.js + trace_files = sorted(glob.glob(os.path.join(data_dir, "*.bin"))) + + if not trace_files: + print(f"No .bin files found in {data_dir}") + return + + print("\nStarting Benchmarks...\n") + print("Trace, DFTL (MB), SFTL (MB), LeaFTL (MB)") + + results = [] + + for trace_path in trace_files: + trace_name = os.path.basename(trace_path) + print(f"\n>>> Processing {trace_name}") + + # 1. Run LearnedFTL (provides LeaFTL and DFTL sizes) + cmd_lea = [sys.executable, run_ftl_script, "-f", "learnedftl", "-t", trace_path, "-sl", "0"] + output_lea = run_and_stream(cmd_lea, env) + + leaftl_mb = 0.0 + dftl_mb = 0.0 + sftl_mb = 0.0 + + if output_lea: + lea_match = re.search(r"estimated learnedftl memory footprint: (\d+) B", output_lea) + if lea_match: + leaftl_mb = float(lea_match.group(1)) / (1024 * 1024) + + dftl_match = re.search(r"estimated dftl memory footprint: (\d+) B", output_lea) + if dftl_match: + dftl_mb = float(dftl_match.group(1)) / (1024 * 1024) + + # 2. Run SFTL (provides SFTL size) + cmd_sftl = [sys.executable, run_ftl_script, "-f", "sftl", "-t", trace_path, "-sl", "0"] + output_sftl = run_and_stream(cmd_sftl, env) + + if output_sftl: + sftl_match = re.search(r"estimated learnedftl memory footprint: (\d+) B", output_sftl) + if sftl_match: + sftl_mb = float(sftl_match.group(1)) / (1024 * 1024) + + summary_line = f"SUMMARY: {trace_name}, {dftl_mb:.4f}, {sftl_mb:.4f}, {leaftl_mb:.4f}" + print(summary_line) + + results.append({ + "trace": trace_name, + "dftl_mb": round(dftl_mb, 4), + "sftl_mb": round(sftl_mb, 4), + "leaftl_mb": round(leaftl_mb, 4) + }) + + sys.stdout.flush() + + # Save to JSON + json_path = "bench_results.json" + with open(json_path, "w") as f: + json.dump(results, f, indent=4) + + print(f"\nFinal results saved to {json_path}") + +if __name__ == "__main__": + main() + +if __name__ == "__main__": + main() diff --git a/data b/data new file mode 120000 index 0000000..85425ce --- /dev/null +++ b/data @@ -0,0 +1 @@ +../jdb_ftl/data \ No newline at end of file diff --git a/init.js b/init.js new file mode 100755 index 0000000..86cde8d --- /dev/null +++ b/init.js @@ -0,0 +1,223 @@ +#!/usr/bin/env bun + +import { + openSync, + readSync, + writeSync, + closeSync, + existsSync, + mkdirSync, + copyFileSync, + unlinkSync, + rmSync, +} from "node:fs"; +import { join, basename } from "node:path"; +import { readdir } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { $ } from "zx"; +import { SingleBar, Presets } from "cli-progress"; + +$.verbose = false; + +const DIR = import.meta.dirname, + DATA = join(DIR, "data"), + TMP = tmpdir(), + DL_DIR = join(TMP, "jdb_ftl_dl"), + EXTRACT = join(DATA, "msrc_extracted"), + TAR = join(DATA, "msrc.tar.gz"), + TMP_GEN = join(TMP, "jdb_ftl"), + URL = "https://github.com/js0-site/jdb_pgm/releases/download/v0.1.0", + PARTS = ["MSRC-trace-003.tar.gz.part-aa", "MSRC-trace-003.tar.gz.part-ab"], + OP_GET = 0x00, + OP_SET = 0x01, + OP_MGET = 0x02, + OP_MSET = 0x03, + OP_RM = 0x04, + OP_MRM = 0x05, + OUT_BUF_SIZE = 1024 * 1024; + +const io = { + mkdir: (p) => { + if (!existsSync(p)) mkdirSync(p, { recursive: true }); + }, + rm: (p) => { + if (existsSync(p)) rmSync(p, { recursive: true, force: true }); + }, + mv: (src, dest) => { + if (!existsSync(src)) return; + copyFileSync(src, dest); + unlinkSync(src); + }, +}; + +const download = async (url, dest) => { + console.log(`[NET] ${url} -> ${dest}`); + const res = await fetch(url); + const buf = await res.arrayBuffer(); + writeSync(openSync(dest, "w"), new Uint8Array(buf)); +}; + +const serialize_op = (op, lba, count, pba, out_buf, offset) => { + const view = new DataView(out_buf.buffer, out_buf.byteOffset + offset); + view.setUint8(0, op); + view.setUint32(1, lba, true); + switch (op) { + case OP_GET: + case OP_RM: + return 5; + case OP_SET: + view.setUint32(5, pba, true); + return 9; + case OP_MGET: + case OP_MRM: + view.setUint32(5, count, true); + return 9; + case OP_MSET: + view.setUint32(5, count, true); + view.setUint32(9, pba, true); + return 13; + default: + return 0; + } +}; + +const process_file = async (file_path, output_name) => { + const name = output_name || basename(file_path, ".revised"), + final_path = join(DATA, `${name}.bin`), + config_path = join(DATA, `${name}.json`), + tmp_file = join(TMP_GEN, `${name}.bin`); + + if (existsSync(final_path) && existsSync(config_path)) { + console.log(`[SKIP] ${name}`); + return; + } + + console.log(`[GEN] ${name}`); + const fd_out = openSync(tmp_file, "w"), + bar = new SingleBar( + { format: `${name} [{bar}] {percentage}% | {value}/{total}` }, + Presets.shades_classic, + ), + CHUNK = 128 * 1024, + read_buf = new Uint8Array(CHUNK), + out_buf = new Uint8Array(OUT_BUF_SIZE), + dec = new TextDecoder("utf-8"), + shadow_map = new Map(); + + let count = 0, + max_lba = 0n, + pba_counter = 1n, + out_off = 0; + + bar.start(Infinity, 0); + + const flush = () => { + if (out_off > 0) { + writeSync(fd_out, out_buf.subarray(0, out_off)); + out_off = 0; + } + }; + + const write_op = (op, lba, cnt, pba) => { + if (out_off + 16 > OUT_BUF_SIZE) flush(); + out_off += serialize_op(op, lba, cnt, pba, out_buf, out_off); + count++; + if (count % 10000 === 0) bar.update(count); + }; + + const fd_in = openSync(file_path, "r"); + let rest = ""; + while (true) { + const n = readSync(fd_in, read_buf, 0, CHUNK, null); + if (n === 0) break; + const str = rest + dec.decode(read_buf.subarray(0, n), { stream: true }); + const lines = str.split("\n"); + rest = lines.pop(); + for (const line of lines) { + const l = line.trim(); + if (!l) continue; + const parts = l.split(/\s+/); + if (parts.length < 3) continue; + const op_str = parts[1]; + if (!["RS", "WS", "R", "W"].includes(op_str)) continue; + + const start_sector = BigInt(parts[2]), + sector_count = parts.length >= 4 ? BigInt(parts[3]) : 8n, + is_write = op_str.includes("W"), + start_lba = start_sector / 8n, + end_sector = start_sector + sector_count - 1n, + end_lba = end_sector / 8n, + page_count = end_lba - start_lba + 1n; + + if (page_count <= 0n) continue; + if (end_lba > max_lba) max_lba = end_lba; + + let type, + pba = 0; + if (is_write) { + if (page_count === 1n) { + type = OP_SET; + pba = Number(++pba_counter); + shadow_map.set(start_lba, pba); + } else { + type = OP_MSET; + pba = Number(pba_counter + 1n); + pba_counter += page_count; + for (let i = 0n; i < page_count; i++) + shadow_map.set(start_lba + i, pba + Number(i)); + } + } else { + type = page_count === 1n ? OP_GET : OP_MGET; + pba = 0; + } + write_op(type, Number(start_lba), Number(page_count), pba); + } + } + closeSync(fd_in); + flush(); + closeSync(fd_out); + bar.update(count); + bar.stop(); + + if (count > 0) { + io.mv(tmp_file, final_path); + writeSync( + openSync(config_path, "w"), + JSON.stringify({ max_lba: max_lba.toString() }), + ); + } +}; + +const main = async () => { + console.log(">>> [INIT]"); + io.mkdir(DATA); + io.mkdir(TMP_GEN); + + if (!existsSync(EXTRACT)) { + if (!existsSync(TAR)) { + io.mkdir(DL_DIR); + await Promise.all( + PARTS.map((p) => download(`${URL}/${p}`, join(DL_DIR, p))), + ); + await $`cat ${PARTS.map((p) => join(DL_DIR, p))} > ${TAR}`; + } + io.mkdir(EXTRACT); + await $`tar -xzf ${TAR} -C ${EXTRACT}`; + } + + const mapping = ["hm_1", "src2_0", "prxy_0", "prn_0", "usr_0"]; + + const files = (await readdir(join(EXTRACT, "final-trace"))) + .filter( + (f) => + f.endsWith(".revised") && + mapping.includes(basename(f, ".revised")), + ) + .map((f) => join(EXTRACT, "final-trace", f)); + + for (const f of files.sort()) await process_file(f); + console.log(">>> [SUCCESS]"); +}; + +await main(); +export default main; diff --git a/init.sh b/init.sh new file mode 100755 index 0000000..57be150 --- /dev/null +++ b/init.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +set -e +DIR=$(realpath $0) && DIR=${DIR%/*} +cd $DIR +set -x + +# Create .mise.toml configuration +cat > .mise.toml < sector and count """ - sector_size = self['sector_size'] + sector_size = self["sector_size"] assert offset % sector_size == 0 - sector = offset / sector_size + sector = old_div(offset, sector_size) assert size % sector_size == 0 - count = size / sector_size + count = old_div(size, sector_size) return sector, count @@ -194,186 +218,169 @@ def get_default_config(self): confdic = { ############### Global ######### - "result_dir" : None, - "workload_src" : WLRUNNER, + "result_dir": None, + "workload_src": WLRUNNER, # "workload_src" : LBAGENERATOR, - "expname" : "default-expname", - "time" : None, - "subexpname" : "default-subexp", + "expname": "default-expname", + "time": None, + "subexpname": "default-subexp", # directmap, blockmap, pagemap, hybridmap, dftl2, tpftl, nkftl - "ftl_type" : "nkftl2", - "sector_size" : 512, - "sort_block_trace" : True, + "ftl_type": "nkftl2", + "sector_size": 512, + "sort_block_trace": True, "trace_issue_and_complete": False, - ############## For wiscsim ###### - "enable_simulation" : True, - "flash_page_size" : 4096, - "flash_npage_per_block" : 4, - "flash_num_blocks" : 64, + "enable_simulation": True, + "flash_page_size": 4096, + "flash_npage_per_block": 4, + "flash_num_blocks": 64, # "enable_e2e_test" : False, - "simulation_processor" : 'e2e', # regular, extent - "stripe_size" : 4, # unit: page + "simulation_processor": "e2e", # regular, extent + "stripe_size": 4, # unit: page "max_victim_valid_ratio": 0.9, - "n_gc_procs" : 1, - - "do_gc_after_workload" : True, - - 'snapshot_valid_ratios' : False, - 'snapshot_erasure_count_dist': False, - 'snapshot_interval': None, - - 'wear_leveling_check_interval': 20*SEC, - 'do_wear_leveling' : False, - 'wear_leveling_factor' : 2, - 'wear_leveling_diff' : 10, - + "n_gc_procs": 1, + "do_gc_after_workload": True, + "snapshot_valid_ratios": False, + "snapshot_erasure_count_dist": False, + "snapshot_interval": None, + "wear_leveling_check_interval": 20 * SEC, + "do_wear_leveling": False, + "wear_leveling_factor": 2, + "wear_leveling_diff": 10, ############## hybridmap ############ - "high_log_block_ratio" : 0.4, - "high_data_block_ratio" : 0.4, - "hybridmapftl": { - "low_log_block_ratio": 0.32 - }, - + "high_log_block_ratio": 0.4, + "high_data_block_ratio": 0.4, + "hybridmapftl": {"low_log_block_ratio": 0.32}, ############## recorder ############# - "verbose_level" : -1, - "output_target" : "file", + "verbose_level": -1, + "output_target": "file", "print_when_finished": False, # "output_target" : "stdout", "record_bad_victim_block": False, - ############## For workrunner ######## - "linux_ncq_depth" : 128, - "linux_io_scheduler" : 'noop', - "dev_size_mb" : None, - "tmpfs_mount_point" : "/mnt/tmpfs", - - "device_path" : "/dev/loop0", # or sth. like /dev/sdc1 - "dev_padding" : 8*MB, - - "enable_blktrace" : False, - "dump_ext4_after_workload" : True, - - "event_file_column_names" : ['pid', 'action', 'operation', 'offset', 'size', - 'timestamp', 'pre_wait_time', 'sync'], - - "fs_mount_point" : "/mnt/fsonloop", - "mnt_opts" : { - "ext4": { 'discard': MOpt(opt_name = "discard", - value = "discard", - include_name = False), - 'data': MOpt(opt_name = "data", - value = "ordered", - include_name = True) }, - "btrfs": { "discard": MOpt(opt_name = "discard", - value = "discard", - include_name = False), - "ssd": MOpt(opt_name = 'ssd', - value = 'ssd', - include_name = False), - "autodefrag": MOpt(opt_name = 'autodefrag', - value = 'autodefrag', - include_name = False) }, - "xfs": {'discard': MOpt(opt_name = 'discard', - value = 'discard', - include_name = False)}, - "f2fs": {'discard': MOpt(opt_name = 'discard', - value = 'discard', - include_name = False)} + "linux_ncq_depth": 128, + "linux_io_scheduler": "noop", + "dev_size_mb": None, + "tmpfs_mount_point": "/mnt/tmpfs", + "device_path": "/dev/loop0", # or sth. like /dev/sdc1 + "dev_padding": 8 * MB, + "enable_blktrace": False, + "dump_ext4_after_workload": True, + "event_file_column_names": [ + "pid", + "action", + "operation", + "offset", + "size", + "timestamp", + "pre_wait_time", + "sync", + ], + "fs_mount_point": "/mnt/fsonloop", + "mnt_opts": { + "ext4": { + "discard": MOpt( + opt_name="discard", value="discard", include_name=False + ), + "data": MOpt(opt_name="data", value="ordered", include_name=True), + }, + "btrfs": { + "discard": MOpt( + opt_name="discard", value="discard", include_name=False + ), + "ssd": MOpt(opt_name="ssd", value="ssd", include_name=False), + "autodefrag": MOpt( + opt_name="autodefrag", value="autodefrag", include_name=False + ), + }, + "xfs": { + "discard": MOpt( + opt_name="discard", value="discard", include_name=False + ) + }, + "f2fs": { + "discard": MOpt( + opt_name="discard", value="discard", include_name=False + ) + }, }, # "common_mnt_opts" : ["discard", "nodatacow"], - "filesystem" : None, - + "filesystem": None, ############## FS ################## - "ext4" : { - "make_opts": {'-O':['^uninit_bg'], '-b':[4096]} - }, - "f2fs" : {"make_opts": {}, 'sysfs':{}}, - "btrfs" : {"make_opts": {}}, - - + "ext4": {"make_opts": {"-O": ["^uninit_bg"], "-b": [4096]}}, + "f2fs": {"make_opts": {}, "sysfs": {}}, + "btrfs": {"make_opts": {}}, ############## workload.py workload to age FS ### # This is run after mounting the file and before real workload # Having this specific aging workload is because we don't want # its performance statistics to be recorded. - "age_workload_class" : "NoOp", - + "age_workload_class": "NoOp", # the following config should match the age_workload_class you use - "aging_config_key" :None, - "aging_config" :{ + "aging_config_key": None, + "aging_config": { "generating_func": "self.generate_random_workload", # "chunk_count": 100*2**20/(8*1024), - "chunk_count": 4 * 2**20 / (512 * 1024), - "chunk_size" : 512 * 1024, - "iterations" : 1, - "filename" : "aging.file", - "n_col" : 5 # only for hotcold workload + "chunk_count": old_div(4 * 2**20, (512 * 1024)), + "chunk_size": 512 * 1024, + "iterations": 1, + "filename": "aging.file", + "n_col": 5, # only for hotcold workload }, - - ############## workload.py on top of FS ######### # "workload_class" : "Simple", - "workload_class" : "Synthetic", - "workload_conf_key" : "workload_conf", - "workload_conf" :{ + "workload_class": "Synthetic", + "workload_conf_key": "workload_conf", + "workload_conf": { # "generating_func": "self.generate_hotcold_workload", # "generating_func": "self.generate_sequential_workload", # "generating_func": "self.generate_backward_workload", "generating_func": "self.generate_random_workload", # "chunk_count": 100*2**20/(8*1024), - "chunk_count": 4 * 2**20 / (512 * 1024), - "chunk_size" : 512 * 1024, - "iterations" : 1, - "n_col" : 5, # only for hotcold workload - "filename" : "test.file" + "chunk_count": old_div(4 * 2**20, (512 * 1024)), + "chunk_size": 512 * 1024, + "iterations": 1, + "n_col": 5, # only for hotcold workload + "filename": "test.file", }, "f2fs_gc_after_workload": True, - ############## LBAGENERATOR ######### # if you choose LBAGENERATOR for workload_src, the following will # be used # "lba_workload_class" : "Sequential", # "lba_workload_class" : "HotCold", # "lba_workload_class" : "Random", - "lba_workload_class" : "Manual", - "lba_workload_configs" : {}, - + "lba_workload_class": "Manual", + "lba_workload_configs": {}, ############# PERF ##################### - "wrap_by_perf" : False, - "perf" : { - "perf_path" : "perf", - "flamegraph_dir" : None - }, - + "wrap_by_perf": False, + "perf": {"perf_path": "perf", "flamegraph_dir": None}, ############# OS ##################### "linux_version": utils.linux_kernel_version(), - "n_online_cpus": 'all' + "n_online_cpus": "all", } return confdic @property def n_pages_per_block(self): - return self['flash_npage_per_block'] + return self["flash_npage_per_block"] @property def page_size(self): - return self['flash_page_size'] + return self["flash_page_size"] @property def device_type(self): - if self['device_path'].startswith("/dev/loop"): - return 'loop' + if self["device_path"].startswith("/dev/loop"): + return "loop" else: - return 'real' + return "real" def get_device_name_no_num(self): - return re.search(r'/dev/([a-z]+)[0-9]*', - self['device_path']).group(1) + return re.search(r"/dev/([a-z]+)[0-9]*", self["device_path"]).group(1) def get_partition_name(self): - return re.search(r'/dev/([a-z]+[0-9]*)', - self['device_path']).group(1) + return re.search(r"/dev/([a-z]+[0-9]*)", self["device_path"]).group(1) class ConfigNewFlash(Config): @@ -381,11 +388,12 @@ class ConfigNewFlash(Config): This config class uses more complex flash configuration with channels, chips, packages... """ - def __init__(self, confdic = None): - super(ConfigNewFlash, self).__init__(confdic) + + def __init__(self, confdic=None): + super().__init__(confdic) flash_config = self.flash_default() - self['flash_config'] = flash_config + self["flash_config"] = flash_config # remove duplication del self["flash_page_size"] @@ -394,42 +402,43 @@ def __init__(self, confdic = None): @property def n_pages_per_plane(self): - return self['flash_config']['n_pages_per_block'] * \ - self['flash_config']['n_blocks_per_plane'] + return ( + self["flash_config"]["n_pages_per_block"] + * self["flash_config"]["n_blocks_per_plane"] + ) @property def n_pages_per_chip(self): - return self.n_pages_per_plane * self['flash_config']['n_planes_per_chip'] + return self.n_pages_per_plane * self["flash_config"]["n_planes_per_chip"] @property def n_pages_per_package(self): - return self.n_pages_per_chip * \ - self['flash_config']['n_chips_per_package'] + return self.n_pages_per_chip * self["flash_config"]["n_chips_per_package"] @property def n_blocks_per_channel(self): - return self['flash_config']['n_blocks_per_plane'] * \ - self['flash_config']['n_planes_per_chip'] * \ - self['flash_config']['n_chips_per_package'] * \ - self['flash_config']['n_packages_per_channel'] + return ( + self["flash_config"]["n_blocks_per_plane"] + * self["flash_config"]["n_planes_per_chip"] + * self["flash_config"]["n_chips_per_package"] + * self["flash_config"]["n_packages_per_channel"] + ) @property def n_pages_per_channel(self): - return self.n_blocks_per_channel * \ - self['flash_config']['n_pages_per_block'] + return self.n_blocks_per_channel * self["flash_config"]["n_pages_per_block"] @property def n_blocks_per_dev(self): - return self.n_blocks_per_channel * \ - self['flash_config']['n_channels_per_dev'] + return self.n_blocks_per_channel * self["flash_config"]["n_channels_per_dev"] @property def n_pages_per_block(self): - return self['flash_config']['n_pages_per_block'] + return self["flash_config"]["n_pages_per_block"] @property def page_size(self): - return self['flash_config']['page_size'] + return self["flash_config"]["page_size"] @property def block_bytes(self): @@ -437,113 +446,127 @@ def block_bytes(self): @property def n_secs_per_page(self): - return self['flash_config']['page_size'] / self['sector_size'] + return old_div(self["flash_config"]["page_size"], self["sector_size"]) @property def n_channels_per_dev(self): - return self['flash_config']['n_channels_per_dev'] + return self["flash_config"]["n_channels_per_dev"] @n_channels_per_dev.setter def n_channels_per_dev(self, value): - self['flash_config']['n_channels_per_dev'] = value + self["flash_config"]["n_channels_per_dev"] = value def flash_default(self): flash_config = { # layout info - "page_size" : 4*KB, - "n_pages_per_block" : 2, - "n_blocks_per_plane" : 2048, - "n_planes_per_chip" : 2, - "n_chips_per_package" : 2, - "n_packages_per_channel" : 1, - "n_channels_per_dev" : 1, - + "page_size": 4 * KB, + "n_pages_per_block": 2, + "n_blocks_per_plane": 2048, + "n_planes_per_chip": 2, + "n_chips_per_package": 2, + "n_packages_per_channel": 1, + "n_channels_per_dev": 1, # OOB info - "oob_size_per_page" : 128, - + "oob_size_per_page": 128, # time info # TODO: these are fixed numbers, but they are random in real world # TODO: Note that the SSD time is different than the flash package time # DES flash does not use the following three - "page_read_time" : 200*MICROSEC, # Max - "page_prog_time" : 1200*MICROSEC, # Typical - "block_erase_time" : 1.6*MILISEC, # Typical - + "page_read_time": 200 * MICROSEC, # Max + "page_prog_time": 1200 * MICROSEC, # Typical + "block_erase_time": 1.6 * MILISEC, # Typical # "t_WC" : 45*NANOSEC, # "t_RC" : 50*NANOSEC, - "t_WC" : 0*NANOSEC, - "t_RC" : 0*NANOSEC, - + "t_WC": 0 * NANOSEC, + "t_RC": 0 * NANOSEC, # "t_R" : 20*MICROSEC, # "t_PROG" : 200*MICROSEC, - "t_R" : 200*MICROSEC, - "t_PROG" : 1200*MICROSEC, - "t_BERS" : 1.5*MILISEC - } + "t_R": 200 * MICROSEC, + "t_PROG": 1200 * MICROSEC, + "t_BERS": 1.5 * MILISEC, + } return flash_config def set_flash_num_blocks_by_bytes(self, size_byte): """ This function will only change n_blocks_per_plane """ - pagesize = self['flash_config']['page_size'] - n_pages_per_block = self['flash_config']['n_pages_per_block'] + pagesize = self["flash_config"]["page_size"] + n_pages_per_block = self["flash_config"]["n_pages_per_block"] - nblocks = size_byte / (pagesize * n_pages_per_block) + nblocks = old_div(size_byte, (pagesize * n_pages_per_block)) rem = size_byte % (pagesize * n_pages_per_block) if rem != 0: - print 'WARNING: set_flash_num_blocks_by_bytes() cannot set to '\ - 'exact bytes. rem:', rem + print( + ( + "WARNING: set_flash_num_blocks_by_bytes() cannot set to " + "exact bytes. rem:", + rem, + ) + ) # change only n_blocks_per_plane - fconf = self['flash_config'] - n_blocks_per_plane = nblocks / (fconf['n_planes_per_chip'] * \ - fconf['n_chips_per_package'] * fconf['n_packages_per_channel'] * \ - fconf['n_packages_per_channel'] * fconf['n_channels_per_dev']) - assert n_blocks_per_plane > 0, 'n_blocks_per_plane must be larger' \ - 'than zero. Not it is {}'.format(n_blocks_per_plane) - fconf['n_blocks_per_plane'] = n_blocks_per_plane - - def byte_to_pagenum(self, offset, force_alignment = True): + fconf = self["flash_config"] + n_blocks_per_plane = old_div( + nblocks, + ( + fconf["n_planes_per_chip"] + * fconf["n_chips_per_package"] + * fconf["n_packages_per_channel"] + * fconf["n_packages_per_channel"] + * fconf["n_channels_per_dev"] + ), + ) + assert n_blocks_per_plane > 0, ( + "n_blocks_per_plane must be larger" + f"than zero. Not it is {n_blocks_per_plane}" + ) + fconf["n_blocks_per_plane"] = n_blocks_per_plane + + def byte_to_pagenum(self, offset, force_alignment=True): "offset to page number" - if force_alignment and offset % self['flash_config']['page_size'] != 0: - raise RuntimeError('offset: {off}, page_size: {ps}'.format( - off=offset, ps = self['flash_config']['page_size'])) - return offset / self['flash_config']['page_size'] + if force_alignment and offset % self["flash_config"]["page_size"] != 0: + raise RuntimeError( + "offset: {off}, page_size: {ps}".format( + off=offset, ps=self["flash_config"]["page_size"] + ) + ) + return old_div(offset, self["flash_config"]["page_size"]) def total_flash_bytes(self): return self.n_pages_per_block * self.n_blocks_per_dev * self.page_size - def off_size_to_page_list(self, off, size, force_alignment = True): + def off_size_to_page_list(self, off, size, force_alignment=True): if force_alignment: - assert size % self['flash_config']['page_size'] == 0, \ - 'size:{}, page_size:{}'.format(size, - self['flash_config']['page_size']) - npages = size / self['flash_config']['page_size'] + assert size % self["flash_config"]["page_size"] == 0, ( + "size:{}, page_size:{}".format(size, self["flash_config"]["page_size"]) + ) + npages = old_div(size, self["flash_config"]["page_size"]) start_page = self.byte_to_pagenum(off) - return range(start_page, start_page+npages) + return list(range(start_page, start_page + npages)) else: - start_page = self.byte_to_pagenum(off, force_alignment = False) - npages = int(math.ceil(float(size) / \ - self['flash_config']['page_size'])) + start_page = self.byte_to_pagenum(off, force_alignment=False) + npages = int(math.ceil(float(size) / self["flash_config"]["page_size"])) - return range(start_page, start_page+npages) + return list(range(start_page, start_page + npages)) - def off_size_to_page_range(self, off, size, force_alignment = True): + def off_size_to_page_range(self, off, size, force_alignment=True): "The input is in bytes" if force_alignment: - if size % self['flash_config']['page_size'] != 0: - raise RuntimeError('Not aligned size:{}, page_size:{}'.format( - size, self['flash_config']['page_size'])) - npages = size / self['flash_config']['page_size'] + if size % self["flash_config"]["page_size"] != 0: + raise RuntimeError( + "Not aligned size:{}, page_size:{}".format( + size, self["flash_config"]["page_size"] + ) + ) + npages = old_div(size, self["flash_config"]["page_size"]) start_page = self.byte_to_pagenum(off) return start_page, npages else: - start_page = self.byte_to_pagenum(off, force_alignment = False) - npages = int(math.ceil(float(size) / - self['flash_config']['page_size'])) + start_page = self.byte_to_pagenum(off, force_alignment=False) + npages = int(math.ceil(float(size) / self["flash_config"]["page_size"])) return start_page, npages @@ -552,14 +575,14 @@ def sec_ext_to_page_ext(self, sector, count): The sector extent has to be aligned with page return page_start, page_count """ - page = sector / self.n_secs_per_page - assert sector % self.n_secs_per_page == 0,\ - "sector {} is not multiple of n_secs_per_page {}"\ - .format(sector, self.n_secs_per_page) - page_count = count / self.n_secs_per_page - assert count % self.n_secs_per_page == 0, \ - "count {} is not multiple of n_secs_per_page {}"\ - .format(count, self.n_secs_per_page) + page = old_div(sector, self.n_secs_per_page) + assert sector % self.n_secs_per_page == 0, ( + f"sector {sector} is not multiple of n_secs_per_page {self.n_secs_per_page}" + ) + page_count = old_div(count, self.n_secs_per_page) + assert count % self.n_secs_per_page == 0, ( + f"count {count} is not multiple of n_secs_per_page {self.n_secs_per_page}" + ) return page, page_count def page_ext_to_sec_ext(self, page, count): @@ -569,31 +592,32 @@ def page_ext_to_sec_ext(self, page, count): return sec, sec_count def total_num_pages(self): - return self['flash_config']['n_pages_per_block'] *\ - self.n_blocks_per_dev + return self["flash_config"]["n_pages_per_block"] * self.n_blocks_per_dev def block_off_to_page(self, blocknum, pageoff): "convert block number and page offset to page number" - return blocknum * self['flash_config']['n_pages_per_block'] + pageoff + return blocknum * self["flash_config"]["n_pages_per_block"] + pageoff def page_to_block(self, pagenum): d = {} - d['blocknum'] = pagenum / self['flash_config']['n_pages_per_block'] - d['pageoffset'] = pagenum % self['flash_config']['n_pages_per_block'] + d["blocknum"] = old_div(pagenum, self["flash_config"]["n_pages_per_block"]) + d["pageoffset"] = pagenum % self["flash_config"]["n_pages_per_block"] return d def page_to_block_off(self, pagenum): "return block, page_offset" - return pagenum / self['flash_config']['n_pages_per_block'], \ - pagenum % self['flash_config']['n_pages_per_block'] + return old_div( + pagenum, self["flash_config"]["n_pages_per_block"] + ), pagenum % self["flash_config"]["n_pages_per_block"] def block_off_to_page(self, blocknum, pageoff): "convert block number and page offset to page number" - return blocknum * self['flash_config']['n_pages_per_block'] + pageoff + return blocknum * self["flash_config"]["n_pages_per_block"] + pageoff def block_to_page_range(self, blocknum): - return blocknum * self['flash_config']['n_pages_per_block'], \ - (blocknum + 1) * self['flash_config']['n_pages_per_block'] + return blocknum * self["flash_config"]["n_pages_per_block"], ( + blocknum + 1 + ) * self["flash_config"]["n_pages_per_block"] class ConfigNotForceAlign(ConfigNewFlash): @@ -602,8 +626,8 @@ def sec_ext_to_page_ext(self, sector, count): The sector extent has to be aligned with page return page_start, page_count """ - page = sector / self.n_secs_per_page - page_end = (sector + count) / self.n_secs_per_page + page = old_div(sector, self.n_secs_per_page) + page_end = old_div((sector + count), self.n_secs_per_page) page_count = page_end - page if (sector + count) % self.n_secs_per_page != 0: page_count += 1 @@ -611,25 +635,23 @@ def sec_ext_to_page_ext(self, sector, count): class ConfigNCQFTL(ConfigNewFlash): - def __init__(self, confdic = None): - super(ConfigNCQFTL, self).__init__(confdic) + def __init__(self, confdic=None): + super().__init__(confdic) - self['SSDFramework'] = {'ncq_depth': 32, - } - self['process_queue_depth'] = 32 - self['simulator_enable_interval'] = False + self["SSDFramework"] = { + "ncq_depth": 32, + } + self["process_queue_depth"] = 32 + self["simulator_enable_interval"] = False def ssd_ncq_depth(self): - return self['SSDFramework']['ncq_depth'] + return self["SSDFramework"]["ncq_depth"] def page_read_time(self): - return self['flash_config']['t_R'] + return self["flash_config"]["t_R"] def page_prog_time(self): - return self['flash_config']['t_PROG'] + return self["flash_config"]["t_PROG"] def block_erase_time(self): - return self['flash_config']['t_BERS'] - - - + return self["flash_config"]["t_BERS"] diff --git a/wiscsee/config_helper/experiment.py b/wiscsee/config_helper/experiment.py index f488573..4001f96 100644 --- a/wiscsee/config_helper/experiment.py +++ b/wiscsee/config_helper/experiment.py @@ -1,3 +1,4 @@ +from past.utils import old_div import os import csv import collections @@ -9,200 +10,209 @@ from config import MountOption as MOpt from config import LBAGENERATOR -class Experiment(object): + +class Experiment: def __init__(self, para): - if para.ftl == 'nkftl2': + if para.ftl == "nkftl2": self.conf = wiscsim.nkftl2.Config() - elif para.ftl == 'dftldes': + elif para.ftl == "dftldes": self.conf = wiscsim.dftldes.Config() - elif para.ftl == 'dftlext': + elif para.ftl == "dftlext": self.conf = wiscsim.dftlext.Config() - elif para.ftl == 'ftlcounter': + elif para.ftl == "ftlcounter": self.conf = wiscsim.ftlcounter.Config() else: - print para.ftl + print(para.ftl) raise NotImplementedError() self.para = para - self.conf['exp_parameters'] = self.para._asdict() + self.conf["exp_parameters"] = self.para._asdict() def setup_environment(self): - self.conf['device_path'] = self.para.device_path - self.conf['dev_size_mb'] = self.para.lbabytes / MB + self.conf["device_path"] = self.para.device_path + self.conf["dev_size_mb"] = old_div(self.para.lbabytes, MB) self.conf["n_online_cpus"] = self.para.n_online_cpus - self.conf['linux_ncq_depth'] = self.para.linux_ncq_depth + self.conf["linux_ncq_depth"] = self.para.linux_ncq_depth if not self.para.dirty_bytes is None: set_vm_default() set_vm("dirty_bytes", self.para.dirty_bytes) - self.conf['do_fstrim'] = False + self.conf["do_fstrim"] = False - self.conf['trace_issue_and_complete'] = self.para.trace_issue_and_complete + self.conf["trace_issue_and_complete"] = self.para.trace_issue_and_complete # filesystem - self.conf['filesystem'] = self.para.filesystem + self.conf["filesystem"] = self.para.filesystem - self.conf['dump_ext4_after_workload'] = self.para.dump_ext4_after_workload + self.conf["dump_ext4_after_workload"] = self.para.dump_ext4_after_workload - self.conf['stop_sim_on_bytes'] = self.para.stop_sim_on_bytes + self.conf["stop_sim_on_bytes"] = self.para.stop_sim_on_bytes - self.conf['sort_block_trace'] = self.para.sort_block_trace + self.conf["sort_block_trace"] = self.para.sort_block_trace - if self.para.filesystem == 'ext4-nj': - self.conf['filesystem'] = 'ext4' + if self.para.filesystem == "ext4-nj": + self.conf["filesystem"] = "ext4" def setup_workload(self): raise NotImplementedError() def setup_fs(self): updates = { - "f2fs": { - 'discard': MOpt(opt_name = 'discard', - value = 'discard', - include_name = False), - # 'background_gc': MOpt(opt_name = 'background_gc', - # value = 'off', - # include_name = True) - }, - "ext4": { - 'discard': MOpt(opt_name = "discard", - value = "discard", - include_name = False), - 'data': MOpt(opt_name = "data", - value = self.para.ext4datamode, - include_name = True) }, - "btrfs": { - "discard": MOpt(opt_name = "discard", - value = "discard", - include_name = False), - "ssd": MOpt(opt_name = 'ssd', - value = 'ssd', - include_name = False), - "autodefrag": MOpt(opt_name = 'autodefrag', - value = 'autodefrag', - include_name = False) }, - "xfs": { - 'discard': MOpt(opt_name = 'discard', - value = 'discard', - include_name = False) - }, - } + "f2fs": { + "discard": MOpt( + opt_name="discard", value="discard", include_name=False + ), + # 'background_gc': MOpt(opt_name = 'background_gc', + # value = 'off', + # include_name = True) + }, + "ext4": { + "discard": MOpt( + opt_name="discard", value="discard", include_name=False + ), + "data": MOpt( + opt_name="data", value=self.para.ext4datamode, include_name=True + ), + }, + "btrfs": { + "discard": MOpt( + opt_name="discard", value="discard", include_name=False + ), + "ssd": MOpt(opt_name="ssd", value="ssd", include_name=False), + "autodefrag": MOpt( + opt_name="autodefrag", value="autodefrag", include_name=False + ), + }, + "xfs": { + "discard": MOpt(opt_name="discard", value="discard", include_name=False) + }, + } if self.para.fs_discard is False: - for fs, dic in updates.items(): - del dic['discard'] - + for fs, dic in list(updates.items()): + del dic["discard"] - self.conf['mnt_opts'].update(updates) + self.conf["mnt_opts"].update(updates) if self.para.ext4hasjournal is True: enable_ext4_journal(self.conf) else: disable_ext4_journal(self.conf) - if self.para.filesystem == 'ext4-nj': + if self.para.filesystem == "ext4-nj": disable_ext4_journal(self.conf) - self.conf['f2fs_gc_after_workload'] = self.para.f2fs_gc_after_workload - self.conf['f2fs']['sysfs']['ipu_policy'] = self.para.f2fs_ipu_policy - self.conf['f2fs']['sysfs']['min_fsync_blocks'] = self.para.f2fs_min_fsync_blocks + self.conf["f2fs_gc_after_workload"] = self.para.f2fs_gc_after_workload + self.conf["f2fs"]["sysfs"]["ipu_policy"] = self.para.f2fs_ipu_policy + self.conf["f2fs"]["sysfs"]["min_fsync_blocks"] = self.para.f2fs_min_fsync_blocks def setup_flash(self): - self.conf['SSDFramework']['ncq_depth'] = self.para.ssd_ncq_depth + self.conf["SSDFramework"]["ncq_depth"] = self.para.ssd_ncq_depth - self.conf['flash_config']['page_size'] = 2048 - self.conf['flash_config']['n_pages_per_block'] = self.para.n_pages_per_block - self.conf['flash_config']['n_blocks_per_plane'] = 2 - self.conf['flash_config']['n_planes_per_chip'] = 1 - self.conf['flash_config']['n_chips_per_package'] = 1 - self.conf['flash_config']['n_packages_per_channel'] = 1 - self.conf['flash_config']['n_channels_per_dev'] = self.para.n_channels_per_dev + self.conf["flash_config"]["page_size"] = 2048 + self.conf["flash_config"]["n_pages_per_block"] = self.para.n_pages_per_block + self.conf["flash_config"]["n_blocks_per_plane"] = 2 + self.conf["flash_config"]["n_planes_per_chip"] = 1 + self.conf["flash_config"]["n_chips_per_package"] = 1 + self.conf["flash_config"]["n_packages_per_channel"] = 1 + self.conf["flash_config"]["n_channels_per_dev"] = self.para.n_channels_per_dev - self.conf['do_not_check_gc_setting'] = self.para.not_check_gc_setting + self.conf["do_not_check_gc_setting"] = self.para.not_check_gc_setting - if self.para.ftl not in ('dftlext', 'ftlcounter'): + if self.para.ftl not in ("dftlext", "ftlcounter"): self.conf.GC_high_threshold_ratio = self.para.gc_high_ratio self.conf.GC_low_threshold_ratio = self.para.gc_low_ratio def setup_ftl(self): - self.conf['enable_blktrace'] = self.para.enable_blktrace - self.conf['enable_simulation'] = self.para.enable_simulation - self.conf['stripe_size'] = self.para.stripe_size - self.conf['segment_bytes'] = self.para.segment_bytes - self.conf['snapshot_interval'] = self.para.snapshot_interval - self.conf['do_wear_leveling'] = self.para.do_wear_leveling - self.conf['wear_leveling_check_interval'] = \ - self.para.wear_leveling_check_interval - self.conf['wear_leveling_factor'] = self.para.wear_leveling_factor - self.conf['wear_leveling_diff'] = self.para.wear_leveling_diff - self.conf['only_get_traffic'] = self.para.only_get_traffic - - if self.para.ftl == 'dftldes': - self.conf['simulator_class'] = 'SimulatorDESNew' - self.conf['ftl_type'] = 'dftldes' - self.conf['snapshot_valid_ratios'] = self.para.snapshot_valid_ratios - self.conf['snapshot_erasure_count_dist'] = self.para.snapshot_erasure_count_dist - self.conf['do_gc_after_workload'] = self.para.do_gc_after_workload + self.conf["enable_blktrace"] = self.para.enable_blktrace + self.conf["enable_simulation"] = self.para.enable_simulation + self.conf["stripe_size"] = self.para.stripe_size + self.conf["segment_bytes"] = self.para.segment_bytes + self.conf["snapshot_interval"] = self.para.snapshot_interval + self.conf["do_wear_leveling"] = self.para.do_wear_leveling + self.conf["wear_leveling_check_interval"] = ( + self.para.wear_leveling_check_interval + ) + self.conf["wear_leveling_factor"] = self.para.wear_leveling_factor + self.conf["wear_leveling_diff"] = self.para.wear_leveling_diff + self.conf["only_get_traffic"] = self.para.only_get_traffic + + if self.para.ftl == "dftldes": + self.conf["simulator_class"] = "SimulatorDESNew" + self.conf["ftl_type"] = "dftldes" + self.conf["snapshot_valid_ratios"] = self.para.snapshot_valid_ratios + self.conf["snapshot_erasure_count_dist"] = ( + self.para.snapshot_erasure_count_dist + ) + self.conf["do_gc_after_workload"] = self.para.do_gc_after_workload self.conf.cache_mapped_data_bytes = int(self.para.cache_mapped_data_bytes) - self.conf['write_gc_log'] = self.para.write_gc_log - self.conf['n_gc_procs'] = self.para.n_gc_procs - - elif self.para.ftl == 'nkftl2': - self.conf['simulator_class'] = 'SimulatorDESNew' - self.conf['ftl_type'] = 'nkftl2' - self.conf['write_gc_log'] = self.para.write_gc_log - - self.conf['nkftl']['n_blocks_in_data_group'] = \ - self.para.segment_bytes / self.conf.block_bytes - self.conf['nkftl']['max_blocks_in_log_group'] = \ - self.conf['nkftl']['n_blocks_in_data_group'] * self.para.log_group_factor - print 'N:', self.conf['nkftl']['n_blocks_in_data_group'] - print 'K:', self.conf['nkftl']['max_blocks_in_log_group'] - self.conf['nkftl']['max_ratio_of_log_blocks'] = self.para.max_log_blocks_ratio - self.conf['snapshot_valid_ratios'] = False - self.conf['snapshot_erasure_count_dist'] = self.para.snapshot_erasure_count_dist - self.conf['do_gc_after_workload'] = self.para.do_gc_after_workload - self.conf['n_gc_procs'] = self.para.n_gc_procs - - elif self.para.ftl == 'dftlext': - self.conf['simulator_class'] = 'SimulatorNonDESSpeed' - self.conf['ftl_type'] = 'dftlext' + self.conf["write_gc_log"] = self.para.write_gc_log + self.conf["n_gc_procs"] = self.para.n_gc_procs + + elif self.para.ftl == "nkftl2": + self.conf["simulator_class"] = "SimulatorDESNew" + self.conf["ftl_type"] = "nkftl2" + self.conf["write_gc_log"] = self.para.write_gc_log + + self.conf["nkftl"]["n_blocks_in_data_group"] = old_div( + self.para.segment_bytes, self.conf.block_bytes + ) + self.conf["nkftl"]["max_blocks_in_log_group"] = ( + self.conf["nkftl"]["n_blocks_in_data_group"] + * self.para.log_group_factor + ) + print(("N:", self.conf["nkftl"]["n_blocks_in_data_group"])) + print(("K:", self.conf["nkftl"]["max_blocks_in_log_group"])) + self.conf["nkftl"]["max_ratio_of_log_blocks"] = ( + self.para.max_log_blocks_ratio + ) + self.conf["snapshot_valid_ratios"] = False + self.conf["snapshot_erasure_count_dist"] = ( + self.para.snapshot_erasure_count_dist + ) + self.conf["do_gc_after_workload"] = self.para.do_gc_after_workload + self.conf["n_gc_procs"] = self.para.n_gc_procs + + elif self.para.ftl == "dftlext": + self.conf["simulator_class"] = "SimulatorNonDESSpeed" + self.conf["ftl_type"] = "dftlext" self.conf.cache_mapped_data_bytes = self.para.cache_mapped_data_bytes # Force wear leveling off since it is not implemented in nkftl - self.conf['do_wear_leveling'] = False + self.conf["do_wear_leveling"] = False - elif self.para.ftl == 'ftlcounter': - self.conf['simulator_class'] = 'SimulatorNonDESSpeed' - self.conf['ftl_type'] = 'ftlcounter' + elif self.para.ftl == "ftlcounter": + self.conf["simulator_class"] = "SimulatorNonDESSpeed" + self.conf["ftl_type"] = "ftlcounter" self.conf.cache_mapped_data_bytes = self.para.cache_mapped_data_bytes - self.conf['gen_ncq_depth_table'] = self.para.gen_ncq_depth_table - self.conf['do_dump_lpn_sem'] = self.para.do_dump_lpn_sem + self.conf["gen_ncq_depth_table"] = self.para.gen_ncq_depth_table + self.conf["do_dump_lpn_sem"] = self.para.do_dump_lpn_sem # Force wear leveling off since it is not implemented in nkftl - self.conf['do_wear_leveling'] = False + self.conf["do_wear_leveling"] = False else: raise NotImplementedError() - logicsize_mb = self.conf['dev_size_mb'] + logicsize_mb = self.conf["dev_size_mb"] self.conf.set_flash_num_blocks_by_bytes( - int(logicsize_mb * 2**20 * self.para.over_provisioning)) + int(logicsize_mb * 2**20 * self.para.over_provisioning) + ) def check_config(self): - if self.conf['ftl_type'] == 'dftldes': + if self.conf["ftl_type"] == "dftldes": assert isinstance(self.conf, wiscsim.dftldes.Config) - assert self.conf['simulator_class'] == 'SimulatorDESNew' - elif self.conf['ftl_type'] == 'nkftl2': + assert self.conf["simulator_class"] == "SimulatorDESNew" + elif self.conf["ftl_type"] == "nkftl2": assert isinstance(self.conf, wiscsim.nkftl2.Config) - assert self.conf['simulator_class'] == 'SimulatorDESNew' - elif self.conf['ftl_type'] == 'dftlext': + assert self.conf["simulator_class"] == "SimulatorDESNew" + elif self.conf["ftl_type"] == "dftlext": assert isinstance(self.conf, wiscsim.dftlext.Config) - assert self.conf['simulator_class'] == 'SimulatorNonDESSpeed' - elif self.conf['ftl_type'] == 'ftlcounter': + assert self.conf["simulator_class"] == "SimulatorNonDESSpeed" + elif self.conf["ftl_type"] == "ftlcounter": assert isinstance(self.conf, wiscsim.ftlcounter.Config) - assert self.conf['simulator_class'] == 'SimulatorNonDESSpeed' + assert self.conf["simulator_class"] == "SimulatorNonDESSpeed" else: RuntimeError("ftl type may not be supported here") @@ -214,14 +224,17 @@ def after_running(self): def run(self): # dict_for_name = {k:v for k,v in self.para._asdict() if k in (' - set_exp_metadata(self.conf, save_data = True, - expname = self.para.expname, - subexpname = 'subexp-' + str(hash(chain_items_as_filename(self.para)))) + set_exp_metadata( + self.conf, + save_data=True, + expname=self.para.expname, + subexpname="subexp-" + str(hash(chain_items_as_filename(self.para))), + ) runtime_update(self.conf) self.check_config() - print self.conf + print(self.conf) run_workflow(self.conf) @@ -238,131 +251,131 @@ def main(self): def get_shared_para_dict(expname, lbabytes): para_dict = { - 'ftl' : ['dftldes'], - 'device_path' : ['/dev/loop0'], - # 'filesystem' : ['f2fs', 'ext4', 'ext4-nj', 'btrfs', 'xfs'], - 'filesystem' : ['ext4'], - 'ext4datamode' : ['ordered'], - 'ext4hasjournal' : [True], - 'expname' : [expname], - 'dirty_bytes' : [4*GB], - 'linux_ncq_depth': [31], - 'ssd_ncq_depth' : [1], - 'cache_mapped_data_bytes' :[lbabytes], - 'lbabytes' : [lbabytes], - 'n_pages_per_block': [64], - 'stripe_size' : [1], - 'enable_blktrace': [True], - 'enable_simulation': [True], - 'f2fs_gc_after_workload': [False], - 'segment_bytes' : [2*MB], - 'max_log_blocks_ratio': [100], - 'n_online_cpus' : ['all'], - 'over_provisioning': [32], # 1.28 is a good number - 'gc_high_ratio' : [0.9], - 'gc_low_ratio' : [0.0], - 'not_check_gc_setting': [True], - 'snapshot_interval': [0.1*SEC], - 'write_gc_log' : [True], - 'f2fs_ipu_policy' : [F2FS_IPU_FSYNC], - 'f2fs_min_fsync_blocks':[8], - 'dump_ext4_after_workload': [True], - 'wear_leveling_check_interval': [10*SEC], - 'do_wear_leveling' : [False], - 'wear_leveling_factor': [2], - 'wear_leveling_diff': [10], - 'snapshot_valid_ratios': [True], - 'snapshot_erasure_count_dist': [True], - 'n_channels_per_dev' : [16], - 'do_gc_after_workload': [False], - 'trace_issue_and_complete': [False], - 'fs_discard': [True], - 'only_get_traffic': [True], - 'age_workload_class': ['NoOp'], - 'aging_appconfs': [None], - 'testname': [None], - 'stop_sim_on_bytes': ['inf'], - 'log_group_factor': [1000], - 'gen_ncq_depth_table': [True], - 'do_dump_lpn_sem': [True], - 'rm_blkparse_events': [False], - 'sort_block_trace': [True], - 'n_gc_procs': [16], - } + "ftl": ["dftldes"], + "device_path": ["/dev/loop0"], + # 'filesystem' : ['f2fs', 'ext4', 'ext4-nj', 'btrfs', 'xfs'], + "filesystem": ["ext4"], + "ext4datamode": ["ordered"], + "ext4hasjournal": [True], + "expname": [expname], + "dirty_bytes": [4 * GB], + "linux_ncq_depth": [31], + "ssd_ncq_depth": [1], + "cache_mapped_data_bytes": [lbabytes], + "lbabytes": [lbabytes], + "n_pages_per_block": [64], + "stripe_size": [1], + "enable_blktrace": [True], + "enable_simulation": [True], + "f2fs_gc_after_workload": [False], + "segment_bytes": [2 * MB], + "max_log_blocks_ratio": [100], + "n_online_cpus": ["all"], + "over_provisioning": [32], # 1.28 is a good number + "gc_high_ratio": [0.9], + "gc_low_ratio": [0.0], + "not_check_gc_setting": [True], + "snapshot_interval": [0.1 * SEC], + "write_gc_log": [True], + "f2fs_ipu_policy": [F2FS_IPU_FSYNC], + "f2fs_min_fsync_blocks": [8], + "dump_ext4_after_workload": [True], + "wear_leveling_check_interval": [10 * SEC], + "do_wear_leveling": [False], + "wear_leveling_factor": [2], + "wear_leveling_diff": [10], + "snapshot_valid_ratios": [True], + "snapshot_erasure_count_dist": [True], + "n_channels_per_dev": [16], + "do_gc_after_workload": [False], + "trace_issue_and_complete": [False], + "fs_discard": [True], + "only_get_traffic": [True], + "age_workload_class": ["NoOp"], + "aging_appconfs": [None], + "testname": [None], + "stop_sim_on_bytes": ["inf"], + "log_group_factor": [1000], + "gen_ncq_depth_table": [True], + "do_dump_lpn_sem": [True], + "rm_blkparse_events": [False], + "sort_block_trace": [True], + "n_gc_procs": [16], + } return para_dict + def get_shared_nolist_para_dict(expname, lbabytes): d = get_shared_para_dict(expname, lbabytes) dd = {} - for k, vlist in d.items(): + for k, vlist in list(d.items()): assert len(vlist) == 1 dd[k] = vlist[0] return dd -class StatsMixin(object): +class StatsMixin: def write_stats(self): - stats_path = os.path.join(self.conf['result_dir'], 'stats.json') - disk_used_bytes = get_dir_size(self.conf['fs_mount_point']) + stats_path = os.path.join(self.conf["result_dir"], "stats.json") + disk_used_bytes = get_dir_size(self.conf["fs_mount_point"]) written_bytes = self.get_traffic_size() - d = {'disk_used_bytes': disk_used_bytes, - 'written_bytes': written_bytes - } + d = {"disk_used_bytes": disk_used_bytes, "written_bytes": written_bytes} dump_json(d, stats_path) - print 'disk_used_bytes', disk_used_bytes / GB - print 'written_bytes', written_bytes / GB + print(("disk_used_bytes", old_div(disk_used_bytes, GB))) + print(("written_bytes", old_div(written_bytes, GB))) def get_traffic_size(self): - filepath = os.path.join(self.conf['result_dir'], 'recorder.json') + filepath = os.path.join(self.conf["result_dir"], "recorder.json") if os.path.exists(filepath): dic = load_json(filepath) - traffic = dic['general_accumulator']['traffic_size'] + traffic = dic["general_accumulator"]["traffic_size"] - print 'write:', traffic['write'] / float(GB), 'GB' - print 'read:', traffic['read'] / float(GB), 'GB' - print 'discard:', traffic['discard'] / float(GB), 'GB' + print(("write:", traffic["write"] / float(GB), "GB")) + print(("read:", traffic["read"] / float(GB), "GB")) + print(("discard:", traffic["discard"] / float(GB), "GB")) - return traffic['write'] + return traffic["write"] else: - filepath = os.path.join(self.conf['result_dir'], - 'blkparse-events-for-ftlsim.txt') - with open(filepath, 'rb') as f: - reader = csv.reader(f, delimiter=' ') + filepath = os.path.join( + self.conf["result_dir"], "blkparse-events-for-ftlsim.txt" + ) + with open(filepath, "rb") as f: + reader = csv.reader(f, delimiter=" ") total = 0 for row in reader: op = row[1] size = int(row[3]) - if op == 'write': + if op == "write": total += size return total class RealDevExperiment(Experiment, StatsMixin): def setup_workload(self): - self.conf['workload_class'] = self.para.workload_class - self.conf['workload_config'] = { - 'appconfs': self.para.appconfs, - 'run_seconds': self.para.run_seconds, - } - self.conf['workload_conf_key'] = 'workload_config' - - self.conf['age_workload_class'] = self.para.age_workload_class - self.conf['aging_workload_config'] = { - 'appconfs': self.para.aging_appconfs, - 'run_seconds': None, - } - self.conf['aging_config_key'] = 'aging_workload_config' + self.conf["workload_class"] = self.para.workload_class + self.conf["workload_config"] = { + "appconfs": self.para.appconfs, + "run_seconds": self.para.run_seconds, + } + self.conf["workload_conf_key"] = "workload_config" + + self.conf["age_workload_class"] = self.para.age_workload_class + self.conf["aging_workload_config"] = { + "appconfs": self.para.aging_appconfs, + "run_seconds": None, + } + self.conf["aging_config_key"] = "aging_workload_config" def after_running(self): self.write_stats() if self.para.rm_blkparse_events is True: - with cd(self.conf['result_dir']): - shcmd('rm blkparse-events-for-ftlsim*') + with cd(self.conf["result_dir"]): + shcmd("rm blkparse-events-for-ftlsim*") class ExistingTraceExperiment(Experiment): @@ -371,15 +384,13 @@ def setup_workload(self): self.conf["lba_workload_class"] = "BlktraceEvents" - self.conf['lba_workload_configs']['mkfs_event_path'] = \ - self.para.mkfs_path - self.conf['lba_workload_configs']['ftlsim_event_path'] = \ - self.para.ftlsim_path + self.conf["lba_workload_configs"]["mkfs_event_path"] = self.para.mkfs_path + self.conf["lba_workload_configs"]["ftlsim_event_path"] = self.para.ftlsim_path def run_on_real_dev(para): - Parameters = collections.namedtuple("Parameters", ','.join(para.keys())) - obj = RealDevExperiment( Parameters(**para) ) + Parameters = collections.namedtuple("Parameters", ",".join(list(para.keys()))) + obj = RealDevExperiment(Parameters(**para)) obj.main() @@ -392,9 +403,6 @@ def execute_simulation(para): default_para = get_shared_nolist_para_dict(None, None) default_para.update(para) para = default_para - Parameters = collections.namedtuple("Parameters", ','.join(para.keys())) - obj = ExistingTraceExperiment( Parameters(**para) ) + Parameters = collections.namedtuple("Parameters", ",".join(list(para.keys()))) + obj = ExistingTraceExperiment(Parameters(**para)) obj.main() - - - diff --git a/wiscsee/config_helper/rule_parameter.py b/wiscsee/config_helper/rule_parameter.py index f998949..d466885 100644 --- a/wiscsee/config_helper/rule_parameter.py +++ b/wiscsee/config_helper/rule_parameter.py @@ -1,13 +1,14 @@ +from past.utils import old_div import os import copy import time -from experiment import * +from .experiment import * from utilities import utils from config import WLRUNNER, LBAGENERATOR, LBAMULTIPROC -class ParaDict(object): +class ParaDict: def __init__(self, expname, trace_expnames, rule): self.expname = expname self.trace_expnames = trace_expnames @@ -18,60 +19,60 @@ def __iter__(self): subexps = self.subexps(self.trace_expnames) - print 'Number of subexps to simulate', len(subexps) + print(("Number of subexps to simulate", len(subexps))) time.sleep(1) for event_set in subexps: - print event_set['mkfs_path'] - para_dict = event_set['original_config']['exp_parameters'] - para_dict.update({ - 'mkfs_path': event_set['mkfs_path'], - 'ftlsim_path': event_set['ftlsim_path'], - 'expname': expname, - - }) + print(event_set["mkfs_path"]) + para_dict = event_set["original_config"]["exp_parameters"] + para_dict.update( + { + "mkfs_path": event_set["mkfs_path"], + "ftlsim_path": event_set["ftlsim_path"], + "expname": expname, + } + ) para_iter = self.get_para_iter(para_dict) - for local_para_dict in para_iter: - yield local_para_dict + yield from para_iter def get_para_iter(self, para_dict): - if self.rule == 'locality': - para_iter = LocalityParaIter(para_dict, coverage='large') + if self.rule == "locality": + para_iter = LocalityParaIter(para_dict, coverage="large") - elif self.rule == 'localitysmall': - para_iter = ParaIter(para_dict, coverage='small') + elif self.rule == "localitysmall": + para_iter = ParaIter(para_dict, coverage="small") - elif self.rule == 'alignment': + elif self.rule == "alignment": para_iter = AlignmentParaIter(para_dict) - elif self.rule == 'grouping': + elif self.rule == "grouping": para_iter = GroupingParaIter(para_dict) - elif self.rule == 'integration': + elif self.rule == "integration": para_iter = IntegrationParaIter(para_dict) else: - raise NotImplementedError( - '{} not supported here.'.format(self.rule)) + raise NotImplementedError(f"{self.rule} not supported here.") return para_iter def subexps(self, expnames): subexp_sets = [] for expname in expnames: - path = os.path.join('/tmp/results', expname) + path = os.path.join("/tmp/results", expname) sets = EventFileSets(path) subexp_sets.extend(sets.get_sets()) return subexp_sets -class EventFileSets(object): +class EventFileSets: """ Given a dirpath, it returns a list. Each item in the list is a dictionary contain event files of mkfs, for-ftlsim, and config. """ + def __init__(self, dirpath): self.dirpath = dirpath @@ -82,160 +83,174 @@ def get_sets(self): pairs = [] for root, dirs, files in os.walk(self.dirpath, topdown=False): for name in files: - if name == 'blkparse-events-for-ftlsim-mkfs.txt': - mkfs_path = os.path.join(root, 'blkparse-events-for-ftlsim-mkfs.txt') - ftlsim_path = os.path.join(root, 'blkparse-events-for-ftlsim.txt') + if name == "blkparse-events-for-ftlsim-mkfs.txt": + mkfs_path = os.path.join( + root, "blkparse-events-for-ftlsim-mkfs.txt" + ) + ftlsim_path = os.path.join(root, "blkparse-events-for-ftlsim.txt") confjson = self._get_confjson(root) - d = {'mkfs_path': mkfs_path, - 'ftlsim_path': ftlsim_path, - 'original_config': confjson, - } + d = { + "mkfs_path": mkfs_path, + "ftlsim_path": ftlsim_path, + "original_config": confjson, + } pairs.append(d) return pairs def _get_confjson(self, subexp_path): - confpath = os.path.join(subexp_path, 'config.json') + confpath = os.path.join(subexp_path, "config.json") confjson = load_json(confpath) return confjson -class LocalityParaIter(object): + +class LocalityParaIter: """ Given a para_dict (exp_parameters), generate a series of para_dict with different cache sizes """ - def __init__(self, para_dict, coverage='large'): + + def __init__(self, para_dict, coverage="large"): self.para_dict = para_dict self.coverage = coverage def __iter__(self): - if self.coverage == 'large': + if self.coverage == "large": coverage_ratios = [0.1, 0.5, 1] - elif self.coverage == 'small': + elif self.coverage == "small": coverage_ratios = [0.05, 0.01] for coverage_ratio in coverage_ratios: local_dict = copy.deepcopy(self.para_dict) - lbabytes = local_dict['lbabytes'] - - local_dict.update({ - 'ftl': 'dftldes', - 'cache_mapped_data_bytes' :int(lbabytes * coverage_ratio), - 'ssd_ncq_depth' : 1, - 'n_pages_per_block' : 64, - 'stripe_size' : 1, - 'enable_blktrace' : False, - 'enable_simulation' : True, - 'segment_bytes' : 2*GB, - 'over_provisioning': 32, # 1.28 is a good number - 'gc_high_ratio' : 0.9, - 'gc_low_ratio' : 0.8, - 'snapshot_interval': 1*SEC, - 'write_gc_log' : False, - 'wear_leveling_check_interval': 1000*SEC, - 'do_wear_leveling' : False, - 'snapshot_valid_ratios': False, - 'snapshot_erasure_count_dist': False, - 'n_channels_per_dev' : 16, - 'do_gc_after_workload': False, - 'trace_issue_and_complete': False, - 'stop_sim_on_bytes': 1*GB, - }) + lbabytes = local_dict["lbabytes"] + + local_dict.update( + { + "ftl": "dftldes", + "cache_mapped_data_bytes": int(lbabytes * coverage_ratio), + "ssd_ncq_depth": 1, + "n_pages_per_block": 64, + "stripe_size": 1, + "enable_blktrace": False, + "enable_simulation": True, + "segment_bytes": 2 * GB, + "over_provisioning": 32, # 1.28 is a good number + "gc_high_ratio": 0.9, + "gc_low_ratio": 0.8, + "snapshot_interval": 1 * SEC, + "write_gc_log": False, + "wear_leveling_check_interval": 1000 * SEC, + "do_wear_leveling": False, + "snapshot_valid_ratios": False, + "snapshot_erasure_count_dist": False, + "n_channels_per_dev": 16, + "do_gc_after_workload": False, + "trace_issue_and_complete": False, + "stop_sim_on_bytes": 1 * GB, + } + ) yield local_dict -class AlignmentParaIter(object): + +class AlignmentParaIter: """ Given a para_dict (exp_parameters), generate a series of para_dict with different cache sizes """ + def __init__(self, para_dict): self.para_dict = para_dict def __iter__(self): local_dict = copy.deepcopy(self.para_dict) - lbabytes = local_dict['lbabytes'] - - for blocksize in [128*KB, 1*MB]: - local_dict.update({ - 'ftl': 'nkftl2', - 'ssd_ncq_depth' : 1, - 'cache_mapped_data_bytes' :lbabytes, - 'n_pages_per_block': blocksize / (2*KB), - 'stripe_size' : blocksize / (2*KB), - 'enable_blktrace': False, - 'enable_simulation': True, - 'segment_bytes' : blocksize, # isolate each block - 'max_log_blocks_ratio': 100, # never gc - 'over_provisioning': 8, # 1.28 is a good number - 'gc_high_ratio' : 10, # never trigger gc - 'gc_low_ratio' : 0.8, - 'not_check_gc_setting': True, - 'snapshot_interval': 10*SEC, - 'write_gc_log' : False, - 'wear_leveling_check_interval': 100*SEC, - 'do_wear_leveling' : False, - 'snapshot_valid_ratios': False, - 'snapshot_erasure_count_dist': False, - 'n_channels_per_dev' : 16, - 'do_gc_after_workload': True, - 'stop_sim_on_bytes': 100*GB, - 'log_group_factor': 100000, - 'trace_issue_and_complete': False, - }) + lbabytes = local_dict["lbabytes"] + + for blocksize in [128 * KB, 1 * MB]: + local_dict.update( + { + "ftl": "nkftl2", + "ssd_ncq_depth": 1, + "cache_mapped_data_bytes": lbabytes, + "n_pages_per_block": old_div(blocksize, (2 * KB)), + "stripe_size": old_div(blocksize, (2 * KB)), + "enable_blktrace": False, + "enable_simulation": True, + "segment_bytes": blocksize, # isolate each block + "max_log_blocks_ratio": 100, # never gc + "over_provisioning": 8, # 1.28 is a good number + "gc_high_ratio": 10, # never trigger gc + "gc_low_ratio": 0.8, + "not_check_gc_setting": True, + "snapshot_interval": 10 * SEC, + "write_gc_log": False, + "wear_leveling_check_interval": 100 * SEC, + "do_wear_leveling": False, + "snapshot_valid_ratios": False, + "snapshot_erasure_count_dist": False, + "n_channels_per_dev": 16, + "do_gc_after_workload": True, + "stop_sim_on_bytes": 100 * GB, + "log_group_factor": 100000, + "trace_issue_and_complete": False, + } + ) yield local_dict -class GroupingParaIter(object): +class GroupingParaIter: def __init__(self, para_dict): self.para_dict = para_dict def __iter__(self): local_dict = copy.deepcopy(self.para_dict) - lbabytes = local_dict['lbabytes'] - - for segment_bytes in [128*MB, 2*GB]: - local_dict.update({ - 'ftl': 'dftldes', - 'ssd_ncq_depth' : 1, - 'cache_mapped_data_bytes' :lbabytes, - 'n_pages_per_block': 64, - 'stripe_size' : 1, - 'enable_blktrace': False, - 'enable_simulation': True, - 'segment_bytes' : segment_bytes, - 'over_provisioning': 8, # 1.28 is a good number - 'gc_high_ratio' : 0.9, - 'gc_low_ratio' : 0.8, - 'not_check_gc_setting': True, - 'snapshot_interval': 0.1*SEC, - 'write_gc_log' : False, - 'wear_leveling_check_interval': 100*SEC, - 'do_wear_leveling' : False, - 'snapshot_valid_ratios': True, - 'snapshot_erasure_count_dist': False, - 'n_channels_per_dev' : 16, - 'do_gc_after_workload': False, - 'stop_sim_on_bytes': 100*GB, - 'trace_issue_and_complete': False, - }) + lbabytes = local_dict["lbabytes"] + + for segment_bytes in [128 * MB, 2 * GB]: + local_dict.update( + { + "ftl": "dftldes", + "ssd_ncq_depth": 1, + "cache_mapped_data_bytes": lbabytes, + "n_pages_per_block": 64, + "stripe_size": 1, + "enable_blktrace": False, + "enable_simulation": True, + "segment_bytes": segment_bytes, + "over_provisioning": 8, # 1.28 is a good number + "gc_high_ratio": 0.9, + "gc_low_ratio": 0.8, + "not_check_gc_setting": True, + "snapshot_interval": 0.1 * SEC, + "write_gc_log": False, + "wear_leveling_check_interval": 100 * SEC, + "do_wear_leveling": False, + "snapshot_valid_ratios": True, + "snapshot_erasure_count_dist": False, + "n_channels_per_dev": 16, + "do_gc_after_workload": False, + "stop_sim_on_bytes": 100 * GB, + "trace_issue_and_complete": False, + } + ) yield local_dict -class IntegrationParaIter(object): +class IntegrationParaIter: """ It simulates a block trace on realistic FTL. """ + def __init__(self, para_dict): self.para_dict = para_dict - self.lbabytes = para_dict['lbabytes'] + self.lbabytes = para_dict["lbabytes"] def __iter__(self): yield self.get_for_dftl() @@ -243,69 +258,71 @@ def __iter__(self): def get_for_dftl(self): local_dict = copy.deepcopy(self.para_dict) - local_dict.update({ - 'ftl': 'dftldes', - 'ssd_ncq_depth' : 1, #<--------------------------------has to set - 'dirty_bytes' : None, - 'cache_mapped_data_bytes' : 0.1 * self.lbabytes, - 'n_pages_per_block': 64, - 'stripe_size' : 1, - 'enable_blktrace': False, - 'enable_simulation': True, - 'segment_bytes' : 128*MB, - 'over_provisioning': 1.50, # 1.28 is a good number - 'gc_high_ratio' : 0.9, - 'gc_low_ratio' : 0.8, - 'not_check_gc_setting': True, - 'snapshot_interval': 0.1*SEC, - 'write_gc_log' : False, - 'wear_leveling_check_interval': 100*SEC, - 'do_wear_leveling' : False, - 'snapshot_valid_ratios': True, - 'snapshot_erasure_count_dist': False, - 'n_channels_per_dev' : 16, - 'do_gc_after_workload': False, - 'stop_sim_on_bytes': 'inf', - 'trace_issue_and_complete': False, - }) + local_dict.update( + { + "ftl": "dftldes", + "ssd_ncq_depth": 1, # <--------------------------------has to set + "dirty_bytes": None, + "cache_mapped_data_bytes": 0.1 * self.lbabytes, + "n_pages_per_block": 64, + "stripe_size": 1, + "enable_blktrace": False, + "enable_simulation": True, + "segment_bytes": 128 * MB, + "over_provisioning": 1.50, # 1.28 is a good number + "gc_high_ratio": 0.9, + "gc_low_ratio": 0.8, + "not_check_gc_setting": True, + "snapshot_interval": 0.1 * SEC, + "write_gc_log": False, + "wear_leveling_check_interval": 100 * SEC, + "do_wear_leveling": False, + "snapshot_valid_ratios": True, + "snapshot_erasure_count_dist": False, + "n_channels_per_dev": 16, + "do_gc_after_workload": False, + "stop_sim_on_bytes": "inf", + "trace_issue_and_complete": False, + } + ) return local_dict def get_for_nkftl(self): local_dict = copy.deepcopy(self.para_dict) - local_dict.update({ - 'ftl' : 'nkftl2', - 'dirty_bytes' : None, - 'ssd_ncq_depth' : 1, #<--------------------------------has to set - 'n_pages_per_block': 64, # 128KB block - 'stripe_size' : 64, # stripe size is the block size - 'enable_blktrace': False, - 'enable_simulation': True, - 'f2fs_gc_after_workload': False, - 'segment_bytes' : 1*MB, # thus N=8 - 'max_log_blocks_ratio': 0.07, - 'n_online_cpus' : 'all', - 'over_provisioning': 1.5, # 1.28 is a good number - 'gc_high_ratio' : 0.9, - 'gc_low_ratio' : 0.8, - 'not_check_gc_setting': True, - 'snapshot_interval': 0.1*SEC, - 'write_gc_log' : False, - 'wear_leveling_check_interval': 100*SEC, - 'do_wear_leveling' : False, - 'wear_leveling_factor': 2, - 'wear_leveling_diff': 10, - 'snapshot_valid_ratios': True, - 'snapshot_erasure_count_dist': False, - 'n_channels_per_dev' : 16, - 'do_gc_after_workload': False, - 'trace_issue_and_complete': False, - 'age_workload_class': 'NoOp', - 'aging_appconfs': None, - 'stop_sim_on_bytes': 'inf', - 'log_group_factor': 10, # So N=8, K=80. max log blocks =800 - }) + local_dict.update( + { + "ftl": "nkftl2", + "dirty_bytes": None, + "ssd_ncq_depth": 1, # <--------------------------------has to set + "n_pages_per_block": 64, # 128KB block + "stripe_size": 64, # stripe size is the block size + "enable_blktrace": False, + "enable_simulation": True, + "f2fs_gc_after_workload": False, + "segment_bytes": 1 * MB, # thus N=8 + "max_log_blocks_ratio": 0.07, + "n_online_cpus": "all", + "over_provisioning": 1.5, # 1.28 is a good number + "gc_high_ratio": 0.9, + "gc_low_ratio": 0.8, + "not_check_gc_setting": True, + "snapshot_interval": 0.1 * SEC, + "write_gc_log": False, + "wear_leveling_check_interval": 100 * SEC, + "do_wear_leveling": False, + "wear_leveling_factor": 2, + "wear_leveling_diff": 10, + "snapshot_valid_ratios": True, + "snapshot_erasure_count_dist": False, + "n_channels_per_dev": 16, + "do_gc_after_workload": False, + "trace_issue_and_complete": False, + "age_workload_class": "NoOp", + "aging_appconfs": None, + "stop_sim_on_bytes": "inf", + "log_group_factor": 10, # So N=8, K=80. max log blocks =800 + } + ) return local_dict - - diff --git a/wiscsee/leaftl_scripts/run_ftl b/wiscsee/leaftl_scripts/run_ftl index 01b2aa7..6be49ef 100755 --- a/wiscsee/leaftl_scripts/run_ftl +++ b/wiscsee/leaftl_scripts/run_ftl @@ -1,4 +1,4 @@ -#!/usr/bin/python2 +#!/usr/bin/env python3 import unittest import collections import shutil @@ -6,7 +6,7 @@ import os #import matplotlib import argparse -from scipy import percentile +# from scipy import percentile #matplotlib.use('Agg') #import matplotlib.pyplot as plt #import statsmodels.distributions diff --git a/wiscsee/pyreuse/apputils/fio.py b/wiscsee/pyreuse/apputils/fio.py index 73d8ff8..0262f97 100644 --- a/wiscsee/pyreuse/apputils/fio.py +++ b/wiscsee/pyreuse/apputils/fio.py @@ -1,7 +1,8 @@ import collections from pyreuse.helpers import * -NOVALUE, HIDE_ATTR = 'NOVALUE', 'HIDE_ATTR' +NOVALUE, HIDE_ATTR = "NOVALUE", "HIDE_ATTR" + class JobConfig(collections.OrderedDict): """ @@ -18,6 +19,7 @@ class JobConfig(collections.OrderedDict): } } """ + def append_section(self, section_name, section_dict): self[section_name] = section_dict @@ -35,44 +37,45 @@ def as_ordered_dict(self): def __str__(self): lines = [] - for section_name, section_dict in self.items(): - lines.append("[{}]".format(section_name)) + for section_name, section_dict in list(self.items()): + lines.append(f"[{section_name}]") - for attr_name, attr_value in section_dict.items(): + for attr_name, attr_value in list(section_dict.items()): if attr_value == NOVALUE: lines.append(attr_name) elif attr_value == HIDE_ATTR: continue else: - lines.append("{}={}".format(attr_name, attr_value)) + lines.append(f"{attr_name}={attr_value}") - return '\n'.join(lines) + return "\n".join(lines) def save(self, filepath): prepare_dir_for_path(filepath) - with open(filepath, 'w') as f: + with open(filepath, "w") as f: f.write(str(self)) -class Fio(object): - def __init__(self, conf_path, result_dir, to_json = True): +class Fio: + def __init__(self, conf_path, result_dir, to_json=True): self.conf_path = conf_path self.result_dir = result_dir - self.result_path = os.path.join(result_dir, 'fio.result.json') + self.result_path = os.path.join(result_dir, "fio.result.json") self.to_json = to_json def parse_results(self): d = load_json(self.result_path) table = parse_json_results(d) - table_to_file(table, self.result_path + '.parsed') + table_to_file(table, self.result_path + ".parsed") def run(self): if self.to_json == True: prepare_dir_for_path(self.result_path) - fio_cmd = "fio {} --output-format=json --output {}".format( - self.conf_path, self.result_path) + fio_cmd = ( + f"fio {self.conf_path} --output-format=json --output {self.result_path}" + ) else: - fio_cmd = "fio {}".format(self.conf_path) + fio_cmd = f"fio {self.conf_path}" with cd(self.result_dir): shcmd(fio_cmd) @@ -87,24 +90,18 @@ def parse_json_results(d): All job perfs will be put into table and returned. """ table = [] - for job in d['jobs']: + for job in d["jobs"]: my_dict = { - 'jobname': job['jobname'], - 'read_bw': job['read']['bw'], - 'read_iops': job['read']['iops'], - 'read_iobytes': job['read']['io_bytes'], - 'read_runtime': job['read']['runtime'], - - 'write_bw': job['write']['bw'], - 'write_iops': job['write']['iops'], - 'write_iobytes': job['write']['io_bytes'], - 'write_runtime': job['write']['runtime'], - } + "jobname": job["jobname"], + "read_bw": job["read"]["bw"], + "read_iops": job["read"]["iops"], + "read_iobytes": job["read"]["io_bytes"], + "read_runtime": job["read"]["runtime"], + "write_bw": job["write"]["bw"], + "write_iops": job["write"]["iops"], + "write_iobytes": job["write"]["io_bytes"], + "write_runtime": job["write"]["runtime"], + } table.append(my_dict) return table - - - - - diff --git a/wiscsee/pyreuse/apputils/parseleveldboutput.py b/wiscsee/pyreuse/apputils/parseleveldboutput.py index 002afa8..824aee5 100644 --- a/wiscsee/pyreuse/apputils/parseleveldboutput.py +++ b/wiscsee/pyreuse/apputils/parseleveldboutput.py @@ -9,40 +9,41 @@ benchname op_duration bw keysize valuesize entries rawsize filesize """ + def parse_metadata(lines): meta = {} for line in lines: - if line.startswith('CPUCache'): - meta['CPUCache'] = line.split()[1] - elif line.startswith('Keys'): - meta['Keys'] = line.split()[1] - elif line.startswith('Values'): - meta['Values'] = line.split()[1] - meta['ValuesCompressed'] = line.split()[4].strip('(') - elif line.startswith('Entries'): - meta['Entries'] = line.split()[1] - elif line.startswith('RawSize'): - meta['RawSize'] = line.split()[1] - elif line.startswith('FileSize'): - meta['FileSize'] = line.split()[1] + if line.startswith("CPUCache"): + meta["CPUCache"] = line.split()[1] + elif line.startswith("Keys"): + meta["Keys"] = line.split()[1] + elif line.startswith("Values"): + meta["Values"] = line.split()[1] + meta["ValuesCompressed"] = line.split()[4].strip("(") + elif line.startswith("Entries"): + meta["Entries"] = line.split()[1] + elif line.startswith("RawSize"): + meta["RawSize"] = line.split()[1] + elif line.startswith("FileSize"): + meta["FileSize"] = line.split()[1] return meta def parse_benchresult_line(line): - if not 'micros/op' in line: + if not "micros/op" in line: return None d = {} - if 'MB/s' in line: - mo = re.search(r'(\w+)\s*:\s*(\S+) micros/op;\s*(\S+) MB/s', line) - d['bw'] = mo.group(3) + if "MB/s" in line: + mo = re.search(r"(\w+)\s*:\s*(\S+) micros/op;\s*(\S+) MB/s", line) + d["bw"] = mo.group(3) else: - mo = re.search(r'(\w+)\s*:\s*(\S+) micros/op;', line) - d['bw'] = 'NA' + mo = re.search(r"(\w+)\s*:\s*(\S+) micros/op;", line) + d["bw"] = "NA" - d['benchname'] = mo.group(1) - d['op_duration'] = mo.group(2) + d["benchname"] = mo.group(1) + d["op_duration"] = mo.group(2) return d @@ -57,16 +58,15 @@ def parse_benchresults(lines): def parse_file_text(text): - parts = text.split('------------------------------------------------') - meta = parse_metadata(parts[0].split('\n')) - table = parse_benchresults(parts[1].split('\n')) + parts = text.split("------------------------------------------------") + meta = parse_metadata(parts[0].split("\n")) + table = parse_benchresults(parts[1].split("\n")) tablestr = table_to_str(table, adddic=meta, width=12) return tablestr + def parse_file(filepath): - with open(filepath, 'r') as f: + with open(filepath) as f: text = f.read() tablestr = parse_file_text(text) return tablestr - - diff --git a/wiscsee/pyreuse/fsutils/__init__.py b/wiscsee/pyreuse/fsutils/__init__.py index d945e02..4049966 100644 --- a/wiscsee/pyreuse/fsutils/__init__.py +++ b/wiscsee/pyreuse/fsutils/__init__.py @@ -1 +1 @@ -from ext4dumpextents import dump_extents_of_a_file +from .ext4dumpextents import dump_extents_of_a_file diff --git a/wiscsee/pyreuse/fsutils/ext4dumpextents.py b/wiscsee/pyreuse/fsutils/ext4dumpextents.py index 4a27378..40a989a 100644 --- a/wiscsee/pyreuse/fsutils/ext4dumpextents.py +++ b/wiscsee/pyreuse/fsutils/ext4dumpextents.py @@ -8,40 +8,48 @@ dump_extents_of_a_file('/dev/loop0', 'datafile') """ + def dump_extents_of_a_file(devname, filepath): """ this is only for ext4 """ - cmd = ['debugfs', devname, '-R', 'dump_extents "' + filepath + '"'] - proc = subprocess.Popen(cmd, stdout = subprocess.PIPE) + cmd = ["debugfs", devname, "-R", 'dump_extents "' + filepath + '"'] + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE) proc.wait() lines = proc.stdout.readlines() - return ''.join(lines) + return "".join(lines) def parse_dump_extents_output(output_text): ret_list = [] - header = ["Level_index", "Max_level", - "Entry_index", "N_Entry", - "Logical_start", "Logical_end", - "Physical_start", "Physical_end", - "Length", "Flags"] - - lines = output_text.split('\n') + header = [ + "Level_index", + "Max_level", + "Entry_index", + "N_Entry", + "Logical_start", + "Logical_end", + "Physical_start", + "Physical_end", + "Length", + "Flags", + ] + + lines = output_text.split("\n") for line in lines: if "Level" in line or "debugfs" in line or len(line.strip()) == 0: continue - line = re.sub(r'[/\-]', " ", line) + line = re.sub(r"[/\-]", " ", line) tokens = line.split() if len(tokens) == 8: # there is no physical end - tokens.insert(7, tokens[6]) #TODO: this is dangerous + tokens.insert(7, tokens[6]) # TODO: this is dangerous d = {} for i in range(9): - d[ header[i] ] = int(tokens[i]) + d[header[i]] = int(tokens[i]) if len(tokens) == 10: d["Flags"] = tokens[9] @@ -55,7 +63,7 @@ def parse_dump_extents_output(output_text): def _add_file_path(extents, file_path): for extent in extents: - extent['file_path'] = file_path + extent["file_path"] = file_path return extents @@ -77,6 +85,3 @@ def get_extents_of_dir(dirpath, dev_path): all_extents.extend(file_extents) return all_extents - - - diff --git a/wiscsee/pyreuse/fsutils/formatfs.py b/wiscsee/pyreuse/fsutils/formatfs.py index 3efd8f9..4592dfc 100644 --- a/wiscsee/pyreuse/fsutils/formatfs.py +++ b/wiscsee/pyreuse/fsutils/formatfs.py @@ -2,25 +2,26 @@ from pyreuse.helpers import shcmd, prepare_dir + def create_fs(dev, mntpoint, fstype): - shcmd("sudo chmod 777 -R {}".format(mntpoint)) + shcmd(f"sudo chmod 777 -R {mntpoint}") - if fstype == 'ext4': - shcmd("sudo mkfs.ext4 {}".format(dev)) - elif fstype == 'ext3': - shcmd("sudo mkfs.ext3 {}".format(dev)) + if fstype == "ext4": + shcmd(f"sudo mkfs.ext4 {dev}") + elif fstype == "ext3": + shcmd(f"sudo mkfs.ext3 {dev}") else: - raise NotImplementedError('{} not supported yet'.format(fstype)) + raise NotImplementedError(f"{fstype} not supported yet") - shcmd("sudo mount {dev} {mnt}".format(dev = dev, mnt = mntpoint)) + shcmd(f"sudo mount {dev} {mntpoint}") -def register_fstab(dev, mntpoint, fstype): - line = "{dev} {mntpoint} {fstype} defaults 0 0"\ - .format(dev = dev, mntpoint = mntpoint, fstype = fstype) - with open('/etc/fstab', 'a') as f: +def register_fstab(dev, mntpoint, fstype): + line = f"{dev} {mntpoint} {fstype} defaults 0 0" + with open("/etc/fstab", "a") as f: f.write(line) + def format_fs(dev, mntpoint, fstype): """ It will format, mount the file system. Then it will register the file @@ -29,5 +30,3 @@ def format_fs(dev, mntpoint, fstype): prepare_dir(mntpoint) create_fs(dev, mntpoint, fstype) register_fstab(dev, mntpoint, fstype) - - diff --git a/wiscsee/pyreuse/general/maketree.py b/wiscsee/pyreuse/general/maketree.py index cc5de92..c1990dc 100644 --- a/wiscsee/pyreuse/general/maketree.py +++ b/wiscsee/pyreuse/general/maketree.py @@ -10,16 +10,17 @@ def mklevel(cur_level, max_level, dir_width, prefix): # print prefix for i in range(dir_width): - mklevel(cur_level = cur_level + 1, - max_level = max_level, - dir_width = dir_width, - prefix = os.path.join(prefix, str(i))) + mklevel( + cur_level=cur_level + 1, + max_level=max_level, + dir_width=dir_width, + prefix=os.path.join(prefix, str(i)), + ) def main(): - mklevel(cur_level=0, max_level=3, dir_width=3, prefix='./new4') - -if __name__ == '__main__': - main() + mklevel(cur_level=0, max_level=3, dir_width=3, prefix="./new4") +if __name__ == "__main__": + main() diff --git a/wiscsee/pyreuse/general/zipf.py b/wiscsee/pyreuse/general/zipf.py index 48c3dd7..7d68dac 100644 --- a/wiscsee/pyreuse/general/zipf.py +++ b/wiscsee/pyreuse/general/zipf.py @@ -1,6 +1,8 @@ +from past.utils import old_div import random import bisect import math +from functools import reduce class ZipfGenerator: @@ -8,22 +10,22 @@ class ZipfGenerator: from http://stackoverflow.com/questions/1366984/generate-random-numbers-distributed-by-zipf """ + def __init__(self, n, alpha): """ Generate numbers up to n alpha can be 0.x, or larger. Smaller -. more uniform """ # Calculate Zeta values from 1 to n: - tmp = [1. / (math.pow(float(i), alpha)) for i in range(1, n+1)] + tmp = [1.0 / (math.pow(float(i), alpha)) for i in range(1, n + 1)] zeta = reduce(lambda sums, x: sums + [sums[-1] + x], tmp, [0]) # Store the translation map: - self.distMap = [x / zeta[-1] for x in zeta] + self.distMap = [old_div(x, zeta[-1]) for x in zeta] - def next(self): + def __next__(self): # Take a uniform 0-1 pseudo-random value: u = random.random() # Translate the Zipf variable: return bisect.bisect(self.distMap, u) - 1 - diff --git a/wiscsee/pyreuse/helpers.py b/wiscsee/pyreuse/helpers.py index f786a3a..4c440b5 100644 --- a/wiscsee/pyreuse/helpers.py +++ b/wiscsee/pyreuse/helpers.py @@ -4,10 +4,11 @@ import shlex import json + def shcmd(cmd, ignore_error=False): - print 'Doing:', cmd + print("Doing:", cmd) ret = subprocess.call(cmd, shell=True) - print 'Returned', ret, cmd + print("Returned", ret, cmd) if ignore_error == False and ret != 0: exit(ret) return ret @@ -15,6 +16,7 @@ def shcmd(cmd, ignore_error=False): class cd: """Context manager for changing the current working directory""" + def __init__(self, newPath): self.newPath = newPath @@ -41,10 +43,10 @@ def parameter_combinations(parameter_dict): ] """ d = parameter_dict - return [dict(zip(d, v)) for v in itertools.product(*d.values())] + return [dict(list(zip(d, v))) for v in itertools.product(*list(d.values()))] -def run_and_get_output(cmd, shell = False): +def run_and_get_output(cmd, shell=False): output = [] cmd = shlex.split(cmd) p = subprocess.Popen(cmd, shell=shell, stdout=subprocess.PIPE) @@ -54,7 +56,7 @@ def run_and_get_output(cmd, shell = False): def load_json(fpath): - decoded = json.load(open(fpath, 'r')) + decoded = json.load(open(fpath)) return decoded @@ -77,15 +79,15 @@ def prepare_dir(dirpath): def linux_kernel_version(): - kernel_ver = run_and_get_output('uname -r')[0].strip() + kernel_ver = run_and_get_output("uname -r")[0].strip() return kernel_ver -def adjust_width(s, width = 32): +def adjust_width(s, width=32): return s.rjust(width) -def table_to_str(table, adddic=None, sep=';', width=32): +def table_to_str(table, adddic=None, sep=";", width=32): """ table is of format: [ {'col1':data, 'col2':data, ..}, @@ -100,27 +102,27 @@ def table_to_str(table, adddic=None, sep=';', width=32): if len(table) == 0: return "" - tablestr = '' - colnames = table[0].keys() + tablestr = "" + colnames = list(table[0].keys()) if adddic != None: - colnames += adddic.keys() - colnamestr = sep.join([adjust_width(s, width=width) for s in colnames]) + '\n' + colnames += list(adddic.keys()) + colnamestr = sep.join([adjust_width(s, width=width) for s in colnames]) + "\n" tablestr += colnamestr for row in table: if adddic != None: - rowcopy = dict(row.items() + adddic.items()) + rowcopy = dict(list(row.items()) + list(adddic.items())) else: rowcopy = row rowstr = [rowcopy[k] for k in colnames] rowstr = [adjust_width(str(x), width=width) for x in rowstr] - rowstr = sep.join(rowstr) + '\n' + rowstr = sep.join(rowstr) + "\n" tablestr += rowstr return tablestr def _tarfilename(tarname): - return '{}.tar.xz'.format(tarname) + return f"{tarname}.tar.xz" def download_kernel(dirpath, tarname): @@ -129,13 +131,12 @@ def download_kernel(dirpath, tarname): """ with cd(dirpath): tarfile = _tarfilename(tarname) - shcmd("wget https://www.kernel.org/pub/linux/kernel/v4.x/{}"\ - .format(tarfile)) - shcmd("tar xf {}".format(tarfile)) + shcmd(f"wget https://www.kernel.org/pub/linux/kernel/v4.x/{tarfile}") + shcmd(f"tar xf {tarfile}") def read_byte_range(filepath, start, size): - f = open(filepath, 'rb') + f = open(filepath, "rb") f.seek(start) data = [] @@ -148,9 +149,10 @@ def read_byte_range(filepath, start, size): return data + def display_binary(data): for v in data: - print '{v}({h})'.format(v=v, h=hex(v)), + print(f"{v}({hex(v)})", end=" ") def run_cmd_on_nodes(cmd, nodes, sync, id_map, do_not_run=False): @@ -163,30 +165,30 @@ def run_cmd_on_nodes(cmd, nodes, sync, id_map, do_not_run=False): """ procs = {} for node_id in nodes: - print '----------', node_id, '----------' + print("----------", node_id, "----------") p = run_cmd_on_node(cmd, node_id, sync, id_map, do_not_run) if not p is None: procs[node_id] = p # wait - for node_id, p in procs.items(): + for node_id, p in list(procs.items()): ret = p.wait() - print 'Node', node_id, 'returned', ret + print("Node", node_id, "returned", ret) def run_cmd_on_node(cmd, node_id, sync, id_map, do_not_run): - cmd = "ssh {host} '{cmd}'".format(host=id_map[node_id], cmd=cmd) + cmd = f"ssh {id_map[node_id]} '{cmd}'" if do_not_run is True: - print cmd + print(cmd) return None if sync is True: - print 'sync', cmd + print("sync", cmd) shcmd(cmd) return None else: - print 'async', cmd + print("async", cmd) p = subprocess.Popen(cmd, shell=True) return p @@ -198,19 +200,13 @@ def run_cmd_on_node(cmd, node_id, sync, id_map, do_not_run): # ... # ] def table_to_file(table, filepath, adddic=None, width=32): - 'save table to a file with additional columns' - with open(filepath, 'w') as f: + "save table to a file with additional columns" + with open(filepath, "w") as f: if len(table) == 0: return - f.write( table_to_str(table, adddic=adddic, width=width) ) + f.write(table_to_str(table, adddic=adddic, width=width)) + def drop_caches(): cmd = "echo 3 > /proc/sys/vm/drop_caches" subprocess.call(cmd, shell=True) - - - - - - - diff --git a/wiscsee/pyreuse/macros.py b/wiscsee/pyreuse/macros.py index d084e5b..74db275 100644 --- a/wiscsee/pyreuse/macros.py +++ b/wiscsee/pyreuse/macros.py @@ -1,4 +1,4 @@ -BYTE, KB, MB, GB, TB = [2**(10*i) for i in range(5)] +BYTE, KB, MB, GB, TB = [2 ** (10 * i) for i in range(5)] # unit is based on nanoseconds -SEC, MILISEC, MICROSEC, NANOSEC = [ 1000**3, 1000**2, 1000, 1 ] +SEC, MILISEC, MICROSEC, NANOSEC = [1000**3, 1000**2, 1000, 1] diff --git a/wiscsee/pyreuse/sysutils/blockclassifiers.py b/wiscsee/pyreuse/sysutils/blockclassifiers.py index 23c2329..18b5c47 100644 --- a/wiscsee/pyreuse/sysutils/blockclassifiers.py +++ b/wiscsee/pyreuse/sysutils/blockclassifiers.py @@ -1,10 +1,14 @@ -class BlockClassifierBase(object): +from past.utils import old_div + + +class BlockClassifierBase: def classify(self, offset): """ Given an offset set, tell the semantics of data stored in it. """ raise NotImplementedError() + class Ext4BlockClassifier(BlockClassifierBase): def __init__(self, range_table, blocksize=4096): """ @@ -19,14 +23,14 @@ def __init__(self, range_table, blocksize=4096): self._blocksize = blocksize def classify(self, offset): - blocknum = offset / self._blocksize + blocknum = old_div(offset, self._blocksize) for row in self._range_table: - for category, (start, end) in row.items(): + for category, (start, end) in list(row.items()): if blocknum >= start and blocknum <= end: return category - return 'UNKNOWN' + return "UNKNOWN" class OffsetClassifier(BlockClassifierBase): @@ -43,11 +47,11 @@ def __init__(self, range_table): def classify(self, offset): for row in self._range_table: - for category, (start, end) in row.items(): + for category, (start, end) in list(row.items()): if offset >= start and offset < end: return category - return 'UNKNOWN' + return "UNKNOWN" class Ext4FileClassifier(BlockClassifierBase): @@ -63,10 +67,14 @@ def __init__(self, extents, blocksize=4096): def _add_offsets(self, extents): blocksize = self._blocksize for extent in extents: - extent['physical_range'] = (extent['Physical_start'] * blocksize, - (extent['Physical_end'] + 1) * blocksize) - extent['logical_range'] = (extent['Logical_start'] * blocksize, - (extent['Logical_end'] + 1) * blocksize) + extent["physical_range"] = ( + extent["Physical_start"] * blocksize, + (extent["Physical_end"] + 1) * blocksize, + ) + extent["logical_range"] = ( + extent["Logical_start"] * blocksize, + (extent["Logical_end"] + 1) * blocksize, + ) return extents @@ -76,13 +84,12 @@ def classify(self, offset): def _find_file_of_offset(self, offset, extents): for extent in extents: if self._is_physical_in_extent(offset, extent) is True: - return extent['file_path'] + return extent["file_path"] return None def _is_physical_in_extent(self, offset, extent): - return offset >= extent['physical_range'][0] and \ - offset < extent['physical_range'][1] - - - + return ( + offset >= extent["physical_range"][0] + and offset < extent["physical_range"][1] + ) diff --git a/wiscsee/pyreuse/sysutils/blocktrace.py b/wiscsee/pyreuse/sysutils/blocktrace.py index 5751aac..5cb8209 100644 --- a/wiscsee/pyreuse/sysutils/blocktrace.py +++ b/wiscsee/pyreuse/sysutils/blocktrace.py @@ -1,3 +1,4 @@ +from past.utils import old_div import os import re import subprocess @@ -6,13 +7,21 @@ from pyreuse.helpers import * from pyreuse.macros import * -class BlktraceResult(object): + +class BlktraceResult: """ Parse blkparse output """ - def __init__(self, sector_size, event_file_column_names, - raw_blkparse_file_path, parsed_output_path, - padding_bytes=0, do_sort=True): + + def __init__( + self, + sector_size, + event_file_column_names, + raw_blkparse_file_path, + parsed_output_path, + padding_bytes=0, + do_sort=True, + ): self.raw_blkparse_file_path = raw_blkparse_file_path self.parsed_output_path = parsed_output_path self.sector_size = sector_size @@ -31,8 +40,8 @@ def __init__(self, sector_size, event_file_column_names, def create_event_file(self): prepare_dir_for_path(self.parsed_output_path) - out_file = open(self.parsed_output_path, 'w') - in_file = open(self.raw_blkparse_file_path, 'r') + out_file = open(self.parsed_output_path, "w") + in_file = open(self.raw_blkparse_file_path) for line in in_file: line = line.strip() @@ -41,26 +50,36 @@ def create_event_file(self): # get row dict row_dict = self.__line_to_dic(line) - row_dict['type'] = 'blkparse' + row_dict["type"] = "blkparse" line = self.__create_event_line(row_dict) - out_file.write( line + '\n' ) + out_file.write(line + "\n") out_file.flush() os.fsync(out_file) out_file.close() - def __line_to_dic(self, line): """ is_data_line() must be true for this line"\ ['8,0', '0', '1', '0.000000000', '440', 'A', 'W', '12912077', '+', '8', '<-', '(8,2)', '606224']" """ - names = ['devid', 'cpuid', 'seqid', 'timestamp', 'pid', 'action', 'RWBS', 'sector_start', 'ignore1', 'sector_count'] + names = [ + "devid", + "cpuid", + "seqid", + "timestamp", + "pid", + "action", + "RWBS", + "sector_start", + "ignore1", + "sector_count", + ] # 0 1 2 3 4 5 6 7 8 9 items = line.split() - dic = dict(zip(names, items)) + dic = dict(list(zip(names, items))) assert len(items) >= len(names) self.__parse_and_add_operation(dic) @@ -69,45 +88,54 @@ def __line_to_dic(self, line): return dic def __parse_and_add_operation(self, row): - if 'D' in row['RWBS']: - operation = 'discard' - elif 'W' in row['RWBS']: - operation = 'write' - elif 'R' in row['RWBS']: - operation = 'read' + if "D" in row["RWBS"]: + operation = "discard" + elif "W" in row["RWBS"]: + operation = "write" + elif "R" in row["RWBS"]: + operation = "read" else: - raise RuntimeError('unknow operation ' + row['RWBS']) + raise RuntimeError("unknow operation " + row["RWBS"]) - row['operation'] = operation + row["operation"] = operation - if 'S' in row['RWBS']: - row['sync'] = 'True' + if "S" in row["RWBS"]: + row["sync"] = "True" else: - row['sync'] = 'False' + row["sync"] = "False" def __parse_and_add_offset_size(self, row): - sec_start = int(row['sector_start']) - sec_count = int(row['sector_count']) + sec_start = int(row["sector_start"]) + sec_count = int(row["sector_count"]) byte_offset = sec_start * self.sector_size - self.padding_bytes byte_size = sec_count * self.sector_size - row['offset'] = byte_offset - row['size'] = byte_size + row["offset"] = byte_offset + row["size"] = byte_size def __create_event_line(self, line_dict): - columns = [str(line_dict.get(colname, 'NA')) - for colname in self.event_file_column_names] - line = ' '.join(columns) + columns = [ + str(line_dict.get(colname, "NA")) + for colname in self.event_file_column_names + ] + line = " ".join(columns) return line -class BlktraceResultInMem(object): +class BlktraceResultInMem: """ Parse blkparse output """ - def __init__(self, sector_size, event_file_column_names, - raw_blkparse_file_path, parsed_output_path, - padding_bytes=0, do_sort=True): + + def __init__( + self, + sector_size, + event_file_column_names, + raw_blkparse_file_path, + parsed_output_path, + padding_bytes=0, + do_sort=True, + ): self.raw_blkparse_file_path = raw_blkparse_file_path self.parsed_output_path = parsed_output_path self.sector_size = sector_size @@ -130,11 +158,22 @@ def __line_to_dic(self, line): is_data_line() must be true for this line"\ ['8,0', '0', '1', '0.000000000', '440', 'A', 'W', '12912077', '+', '8', '<-', '(8,2)', '606224']" """ - names = ['devid', 'cpuid', 'seqid', 'timestamp', 'pid', 'action', 'RWBS', 'sector_start', 'ignore1', 'sector_count'] + names = [ + "devid", + "cpuid", + "seqid", + "timestamp", + "pid", + "action", + "RWBS", + "sector_start", + "ignore1", + "sector_count", + ] # 0 1 2 3 4 5 6 7 8 9 items = line.split() - dic = dict(zip(names, items)) + dic = dict(list(zip(names, items))) assert len(items) >= len(names) self.__parse_and_add_operation(dic) @@ -143,55 +182,58 @@ def __line_to_dic(self, line): return dic def __parse_and_add_operation(self, row): - if 'D' in row['RWBS']: - operation = 'discard' - elif 'W' in row['RWBS']: - operation = 'write' - elif 'R' in row['RWBS']: - operation = 'read' + if "D" in row["RWBS"]: + operation = "discard" + elif "W" in row["RWBS"]: + operation = "write" + elif "R" in row["RWBS"]: + operation = "read" else: - raise RuntimeError('unknow operation ' + row['RWBS']) + raise RuntimeError("unknow operation " + row["RWBS"]) - row['operation'] = operation + row["operation"] = operation - if 'S' in row['RWBS']: - row['sync'] = 'True' + if "S" in row["RWBS"]: + row["sync"] = "True" else: - row['sync'] = 'False' + row["sync"] = "False" def __parse_and_add_offset_size(self, row): - sec_start = int(row['sector_start']) - sec_count = int(row['sector_count']) + sec_start = int(row["sector_start"]) + sec_count = int(row["sector_count"]) byte_offset = sec_start * self.sector_size - self.padding_bytes byte_size = sec_count * self.sector_size - row['offset'] = byte_offset - row['size'] = byte_size + row["offset"] = byte_offset + row["size"] = byte_size def __calculate_pre_wait_time(self, event_table): if self.do_sort is True: - event_table.sort(key = lambda k: float(k['timestamp'])) + event_table.sort(key=lambda k: float(k["timestamp"])) for i, row in enumerate(event_table): if i == 0: - row['pre_wait_time'] = 0 + row["pre_wait_time"] = 0 continue - row['pre_wait_time'] = float(event_table[i]['timestamp']) - \ - float(event_table[i-1]['timestamp']) + row["pre_wait_time"] = float(event_table[i]["timestamp"]) - float( + event_table[i - 1]["timestamp"] + ) if self.do_sort is True: - assert row['pre_wait_time'] >= 0, "data is {}".format(row['pre_wait_time']) + assert row["pre_wait_time"] >= 0, "data is {}".format( + row["pre_wait_time"] + ) return event_table def __parse_rawfile(self): - with open(self.raw_blkparse_file_path, 'r') as line_iter: + with open(self.raw_blkparse_file_path) as line_iter: table = [] for line in line_iter: line = line.strip() # print is_data_line(line), line if is_data_line(line): ret = self.__line_to_dic(line) - ret['type'] = 'blkparse' + ret["type"] = "blkparse" else: ret = None @@ -203,35 +245,35 @@ def __parse_rawfile(self): self.__parsed_table = table def __create_event_line(self, line_dict): - columns = [str(line_dict[colname]) - for colname in self.event_file_column_names] - line = ' '.join(columns) + columns = [str(line_dict[colname]) for colname in self.event_file_column_names] + line = " ".join(columns) return line def create_event_file(self): prepare_dir_for_path(self.parsed_output_path) - out = open(self.parsed_output_path, 'w') + out = open(self.parsed_output_path, "w") for row_dict in self.__parsed_table: - if row_dict['type'] == 'blkparse': + if row_dict["type"] == "blkparse": line = self.__create_event_line(row_dict) else: raise NotImplementedError() - out.write( line + '\n' ) + out.write(line + "\n") out.flush() os.fsync(out) out.close() def get_duration(self): - return float(self.__parsed_table[-1]['timestamp']) - \ - float(self.__parsed_table[0]['timestamp']) + return float(self.__parsed_table[-1]["timestamp"]) - float( + self.__parsed_table[0]["timestamp"] + ) def count_sectors(self, operation): sectors_cnt = 0 for row in self.__parsed_table: - if row['operation'] == operation: - sectors_cnt += int(row['sector_count']) + if row["operation"] == operation: + sectors_cnt += int(row["sector_count"]) return sectors_cnt @@ -240,14 +282,22 @@ def get_bandwidth_mb(self, operation): size_mb = sec_cnt * self.sector_size / float(MB) duration = self.get_duration() - return size_mb / duration + return old_div(size_mb, duration) -class BlockTraceManager(object): +class BlockTraceManager: "This class provides interfaces to interact with blktrace" - def __init__(self, dev, event_file_column_names, - resultpath, to_ftlsim_path, sector_size, padding_bytes=0, - do_sort=True): + + def __init__( + self, + dev, + event_file_column_names, + resultpath, + to_ftlsim_path, + sector_size, + padding_bytes=0, + do_sort=True, + ): self.dev = dev self.sector_size = sector_size self.event_file_column_names = event_file_column_names @@ -265,21 +315,25 @@ def stop_tracing_and_collecting(self): def create_event_file_from_blkparse(self): if self.do_sort is True: - rawparser = BlktraceResultInMem(self.sector_size, - self.event_file_column_names, - self.resultpath, self.to_ftlsim_path, - padding_bytes=self.padding_bytes, - do_sort=self.do_sort - ) + rawparser = BlktraceResultInMem( + self.sector_size, + self.event_file_column_names, + self.resultpath, + self.to_ftlsim_path, + padding_bytes=self.padding_bytes, + do_sort=self.do_sort, + ) rawparser.create_event_file() else: - rawparser = BlktraceResult(self.sector_size, - self.event_file_column_names, - self.resultpath, self.to_ftlsim_path, - padding_bytes=self.padding_bytes, - do_sort=self.do_sort - ) + rawparser = BlktraceResult( + self.sector_size, + self.event_file_column_names, + self.resultpath, + self.to_ftlsim_path, + padding_bytes=self.padding_bytes, + do_sort=self.do_sort, + ) rawparser.create_event_file() @@ -290,34 +344,35 @@ def start_blktrace_on_bg(dev, resultpath, trace_filter=None): if trace_filter is None: # trace_filter = '-a issue' - trace_filter = '' + trace_filter = "" else: - trace_filter = ' '.join(['-a ' + mask for mask in trace_filter]) - - cmd = "sudo blktrace {filtermask} -d {dev} -o - | "\ - "blkparse {filtermask} -i - >> "\ - "{resultpath}".format(dev = dev, resultpath = resultpath, - filtermask = trace_filter) - print cmd + trace_filter = " ".join(["-a " + mask for mask in trace_filter]) + + cmd = ( + f"sudo blktrace {trace_filter} -d {dev} -o - | " + f"blkparse {trace_filter} -i - >> " + f"{resultpath}" + ) + print(cmd) p = subprocess.Popen(cmd, shell=True) - time.sleep(0.3) # wait to see if there's any immediate error. + time.sleep(0.3) # wait to see if there's any immediate error. if p.poll() != None: raise RuntimeError("tracing failed to start") return p + def stop_blktrace_on_bg(): - shcmd('pkill blkparse', ignore_error=True) - shcmd('pkill blktrace', ignore_error=True) - shcmd('sync') + shcmd("pkill blkparse", ignore_error=True) + shcmd("pkill blktrace", ignore_error=True) + shcmd("sync") + def is_data_line(line): # devid sector_start + nblocks - match_obj = re.match( r'\d+,\d+.*\d+\s+\+\s+\d+', line) + match_obj = re.match(r"\d+,\d+.*\d+\s+\+\s+\d+", line) if match_obj == None: return False else: return True - - diff --git a/wiscsee/pyreuse/sysutils/cgroup.py b/wiscsee/pyreuse/sysutils/cgroup.py index 5c325ff..b9cb7ca 100644 --- a/wiscsee/pyreuse/sysutils/cgroup.py +++ b/wiscsee/pyreuse/sysutils/cgroup.py @@ -3,12 +3,13 @@ from pyreuse.helpers import * -class Cgroup(object): + +class Cgroup: def __init__(self, name, subs): self.name = name self.subs = subs - shcmd('cgcreate -g {subs}:{name}'.format(subs=subs, name=name)) + shcmd(f"cgcreate -g {subs}:{name}") def set_item(self, sub, item, value): """ @@ -20,35 +21,31 @@ def set_item(self, sub, item, value): ret_value = self._read(sub, item) if ret_value != str(value): - print 'Warning:', ret_value, '!=', value + print(("Warning:", ret_value, "!=", value)) def get_item(self, sub, item): return self._read(sub, item) def execute(self, cmd): - cg_cmd = ['cgexec', - '-g', '{subs}:{name}'.format(subs=self.subs, name=self.name), - '--sticky'] + cg_cmd = ["cgexec", "-g", f"{self.subs}:{self.name}", "--sticky"] cg_cmd += cmd - print cg_cmd + print(cg_cmd) p = subprocess.Popen(cg_cmd) return p def _write(self, sub, item, value): path = self._path(sub, item) - with open(path, 'w') as f: + with open(path, "w") as f: f.write(str(value)) def _read(self, sub, item): path = self._path(sub, item) - with open(path, 'r') as f: + with open(path) as f: value = f.read() return value.strip() def _path(self, sub, item): - path = os.path.join('/sys/fs/cgroup', sub, self.name, item) + path = os.path.join("/sys/fs/cgroup", sub, self.name, item) return path - - diff --git a/wiscsee/pyreuse/sysutils/dumpe2fsparser.py b/wiscsee/pyreuse/sysutils/dumpe2fsparser.py index f4b03a7..b52200e 100644 --- a/wiscsee/pyreuse/sysutils/dumpe2fsparser.py +++ b/wiscsee/pyreuse/sysutils/dumpe2fsparser.py @@ -12,12 +12,14 @@ superblock: start, end """ + def is_bg_start_line(line): return line.startswith("Group") + def convert_to_range(s): - if '-' in s: - tup = s.split('-') + if "-" in s: + tup = s.split("-") tup = [int(x) for x in tup] start, end = tup else: @@ -25,47 +27,54 @@ def convert_to_range(s): end = start return start, end + def parse_superblock(line): - mo = re.search(r'superblock at (\S+), Group descriptors at (\S+)', line) + mo = re.search(r"superblock at (\S+), Group descriptors at (\S+)", line) superblock_addr = convert_to_range(mo.group(1)) groupdesc = convert_to_range(mo.group(2)) - return {'superblock': superblock_addr, - 'groupdesc': groupdesc} + return {"superblock": superblock_addr, "groupdesc": groupdesc} + def parse_gdt(line): - mo = re.search(r'Reserved GDT blocks at (\S+)', line) - return {'reserved-gdt': convert_to_range(mo.group(1))} + mo = re.search(r"Reserved GDT blocks at (\S+)", line) + return {"reserved-gdt": convert_to_range(mo.group(1))} + def parse_bitmaps(line): - mo = re.search(r'Block bitmap at (\S+) .*, Inode bitmap at (\S+) .*', line) - return {'block-bitmap': convert_to_range(mo.group(1)), - 'inode-bitmap': convert_to_range(mo.group(2))} + mo = re.search(r"Block bitmap at (\S+) .*, Inode bitmap at (\S+) .*", line) + return { + "block-bitmap": convert_to_range(mo.group(1)), + "inode-bitmap": convert_to_range(mo.group(2)), + } + def parse_inodetable(line): - mo = re.search(r'Inode table at (\S+) .*', line) - return {'inode-table': convert_to_range(mo.group(1))} + mo = re.search(r"Inode table at (\S+) .*", line) + return {"inode-table": convert_to_range(mo.group(1))} + def parse_bg_lines(bg_lines): results = [] for line in bg_lines: line = line.strip() - if 'superblock' in line: + if "superblock" in line: d = parse_superblock(line) results.append(d) - elif 'Reserved GDT' in line: + elif "Reserved GDT" in line: d = parse_gdt(line) results.append(d) - elif 'Block bitmap' in line: + elif "Block bitmap" in line: d = parse_bitmaps(line) results.append(d) - elif 'Inode table' in line: + elif "Inode table" in line: d = parse_inodetable(line) results.append(d) return results + def parse_bg_text(text): bgs = [] - for line in text.split('\n'): + for line in text.split("\n"): if is_bg_start_line(line): cur_bg_lines = [] bgs.append(cur_bg_lines) @@ -76,32 +85,35 @@ def parse_bg_text(text): parsed_dicts.extend(parse_bg_lines(bg)) return parsed_dicts + def as_table(parsed_dicts): rows = ["type start end"] for dic in parsed_dicts: - for k, v in dic.items(): - vstr = ' '.join([str(x) for x in v]) - line = ' '.join([k, vstr]) + for k, v in list(dic.items()): + vstr = " ".join([str(x) for x in v]) + line = " ".join([k, vstr]) rows.append(line) return rows + def parse_header_text(text): """ header is the first part, before \n\n\n """ - lines = text.split('\n') + lines = text.split("\n") d = {} for line in lines: - items = line.split(':') + items = line.split(":") if line.startswith("Journal inode:"): - d['journal-inode'] = int(items[1]) + d["journal-inode"] = int(items[1]) elif line.startswith("Journal length:"): - d['journal-length'] = int(items[1]) + d["journal-length"] = int(items[1]) return d + def parse_file_text(text): # get the second part of dumpe2fs output text = text.split("\n\n\n")[1] @@ -109,11 +121,11 @@ def parse_file_text(text): range_table = parse_bg_text(text) return range_table + def parse_file(fpath): - with open(fpath, 'r') as f: + with open(fpath) as f: text = f.read() range_table = parse_file_text(text) rows = as_table(range_table) - return '\n'.join(rows) - + return "\n".join(rows) diff --git a/wiscsee/pyreuse/sysutils/filefragparser.py b/wiscsee/pyreuse/sysutils/filefragparser.py index ce1a807..1ce45f8 100644 --- a/wiscsee/pyreuse/sysutils/filefragparser.py +++ b/wiscsee/pyreuse/sysutils/filefragparser.py @@ -3,9 +3,10 @@ from pyreuse.helpers import run_and_get_output + def parse_file_frag_text(text): "text generated by filefrag -v" - lines = text.split('\n') + lines = text.split("\n") table = [] for line in lines[3:-2]: @@ -14,26 +15,35 @@ def parse_file_frag_text(text): return table + def _parse_line(line): items = line.split() items = [_clean_item(item) for item in items] - keys = ['ext', 'logical_start', 'logical_end', - 'physical_start', 'physical_end', 'length', - 'expected', 'flags'] + keys = [ + "ext", + "logical_start", + "logical_end", + "physical_start", + "physical_end", + "length", + "expected", + "flags", + ] if len(items) == 6: - items.extend(['NA', 'NA']) + items.extend(["NA", "NA"]) elif len(items) == 7: - items.extend(['NA']) + items.extend(["NA"]) - d = dict(zip(keys, items)) + d = dict(list(zip(keys, items))) return d + def _clean_item(item): - item = item.replace('.', '') - item = item.replace(':', '') + item = item.replace(".", "") + item = item.replace(":", "") item = item.strip() # convert if we can @@ -44,18 +54,21 @@ def _clean_item(item): return item + def parse_file_frag_file(path): - with open(path, 'r') as f: + with open(path) as f: text = f.read() return parse_file_frag_text(text) + def filefrag(filepath): - lines = run_and_get_output('filefrag -v {}'.format(filepath)) - text = ''.join(lines) + lines = run_and_get_output(f"filefrag -v {filepath}") + text = "".join(lines) return parse_file_frag_text(text) + # TODO: this function should be moved somewhere else def get_file_range_table(dirpath, BLOCKSIZE=4096): """ @@ -80,6 +93,7 @@ def get_file_range_table(dirpath, BLOCKSIZE=4096): return ret_table + def file_range(table, path, BLOCKSIZE=4096): """ table is the thing you get from filefrag() @@ -87,16 +101,9 @@ def file_range(table, path, BLOCKSIZE=4096): range_table = [] for row in table: - start_byte = row['physical_start'] * BLOCKSIZE - size = row['length'] * BLOCKSIZE - range_row = {'start_byte': start_byte, - 'size': size, - 'path': path} + start_byte = row["physical_start"] * BLOCKSIZE + size = row["length"] * BLOCKSIZE + range_row = {"start_byte": start_byte, "size": size, "path": path} range_table.append(range_row) return range_table - - - - - diff --git a/wiscsee/pyreuse/sysutils/ftrace.py b/wiscsee/pyreuse/sysutils/ftrace.py index e593a1a..4930663 100644 --- a/wiscsee/pyreuse/sysutils/ftrace.py +++ b/wiscsee/pyreuse/sysutils/ftrace.py @@ -5,58 +5,59 @@ from pyreuse.helpers import shcmd, cd -class Ftrace(object): + +class Ftrace: def __init__(self): self.rootdir = "/sys/kernel/debug/tracing" def write_file(self, filename, msg): with cd(self.rootdir): - with open(filename, 'w') as f: - print 'writing "{}" to {}'.format(msg, filename) + with open(filename, "w") as f: + print(f'writing "{msg}" to {filename}') f.write(msg) f.flush() def append_file(self, filename, msg): with cd(self.rootdir): - with open(filename, 'a') as f: - print 'appending "{}" to {}'.format(msg, filename) + with open(filename, "a") as f: + print(f'appending "{msg}" to {filename}') f.write(msg) f.flush() def read_file(self, filename): with cd(self.rootdir): - with open(filename, 'r') as f: + with open(filename) as f: text = f.read() return text def get_trace(self): - text = self.read_file('trace') + text = self.read_file("trace") return text def set_tracer(self, tracer): - self.write_file('current_tracer', tracer) + self.write_file("current_tracer", tracer) def start_tracing(self): - self.write_file('tracing_on', '1') + self.write_file("tracing_on", "1") def stop_tracing(self): - self.write_file('tracing_on', '0') + self.write_file("tracing_on", "0") def clean_trace(self): - self.write_file('trace', '') + self.write_file("trace", "") def write_marker(self, msg): - self.write_file('trace_marker', msg) + self.write_file("trace_marker", msg) def set_filter(self, filter_str): - self.write_file('set_ftrace_filter', filter_str) + self.write_file("set_ftrace_filter", filter_str) def add_filter(self, filter_str): - self.append_file('set_ftrace_filter', filter_str) + self.append_file("set_ftrace_filter", filter_str) def copy_trace(self, target_path): with cd(self.rootdir): - shcmd("cp trace {}".format(target_path)) + shcmd(f"cp trace {target_path}") def trace_cmd(cmd, tracer, ffilter): @@ -78,8 +79,7 @@ def trace_cmd(cmd, tracer, ffilter): return text - -if __name__ == '__main__': +if __name__ == "__main__": # An example ftr = Ftrace() ftr.clean_trace() diff --git a/wiscsee/pyreuse/sysutils/straceParser.py b/wiscsee/pyreuse/sysutils/straceParser.py index 4d1932a..8584e5b 100644 --- a/wiscsee/pyreuse/sysutils/straceParser.py +++ b/wiscsee/pyreuse/sysutils/straceParser.py @@ -1,4 +1,4 @@ -import sys,os +import sys, os import pprint import glob import re @@ -19,47 +19,49 @@ """ +UNFINISHED_MARK = "" -UNFINISHED_MARK = '' def match_line(line): - mo = re.match(r'(\S+)\s+(\S+)\s+(\w+)\((.+)\)\s+=\s+(\S+)', line) # with pid + mo = re.match(r"(\S+)\s+(\S+)\s+(\w+)\((.+)\)\s+=\s+(\S+)", line) # with pid return mo + def match_line_no_pid(line): - mo = re.match(r'(\S+)\s+(\w+)\((.+)\)\s+=\s+(\S+)', line) # without pid + mo = re.match(r"(\S+)\s+(\w+)\((.+)\)\s+=\s+(\S+)", line) # without pid return mo + def line_to_dic_no_pid(line, pid=None): """ This function only handles normal line. It does not handle interrupted lines. """ dic = { - 'pid' :pid, - 'time' :None, - 'callname': None, - 'args':None, - 'ret' :None, - } - line = line.replace('"', '') + "pid": pid, + "time": None, + "callname": None, + "args": None, + "ret": None, + } + line = line.replace('"', "") mo = match_line_no_pid(line) if mo: i = 1 - dic['time'] = mo.group(i) + dic["time"] = mo.group(i) i += 1 - dic['callname'] = mo.group(i) + dic["callname"] = mo.group(i) i += 1 - args = mo.group(i).split(',') - dic['args'] = [ x.strip() for x in args ] + args = mo.group(i).split(",") + dic["args"] = [x.strip() for x in args] i += 1 - dic['ret'] = mo.group(i) + dic["ret"] = mo.group(i) # dic['original_line'] = line else: - print 'cannot parse:', line - raise RuntimeError('cannot parse') + print(("cannot parse:", line)) + raise RuntimeError("cannot parse") return dic @@ -70,81 +72,82 @@ def line_to_dic(line): It does not handle interrupted lines. """ dic = { - 'pid' :None, - 'time' :None, - 'callname': None, - 'args':None, - 'ret' :None, - } - line = line.replace('"', '') + "pid": None, + "time": None, + "callname": None, + "args": None, + "ret": None, + } + line = line.replace('"', "") mo = match_line(line) - #fdmap = {} + # fdmap = {} if mo: - #print mo.groups() + # print mo.groups() i = 1 - dic['pid'] = mo.group(i) + dic["pid"] = mo.group(i) i += 1 - dic['time'] = mo.group(i) + dic["time"] = mo.group(i) i += 1 - dic['callname'] = mo.group(i) + dic["callname"] = mo.group(i) i += 1 - args = mo.group(i).split(',') - dic['args'] = [ x.strip() for x in args ] + args = mo.group(i).split(",") + dic["args"] = [x.strip() for x in args] i += 1 - dic['ret'] = mo.group(i) + dic["ret"] = mo.group(i) # dic['original_line'] = line else: - print 'cannot parse:', line - raise RuntimeError('cannot parse') + print(("cannot parse:", line)) + raise RuntimeError("cannot parse") return dic + def get_dic_from_unfinished(line): """ line has to be .... """ - mo = re.match(r'(\S+)\s+(\S+)\s+(\w+)\(', - line) + mo = re.match(r"(\S+)\s+(\S+)\s+(\w+)\(", line) dic = {} if mo: - #print mo.groups() + # print mo.groups() i = 1 - dic['pid'] = mo.group(i) + dic["pid"] = mo.group(i) i += 1 - dic['time'] = mo.group(i) + dic["time"] = mo.group(i) i += 1 - dic['callname'] = mo.group(i) + dic["callname"] = mo.group(i) n = len(line) m = len(UNFINISHED_MARK) - dic['trimedline'] = line[:(n-m)] + dic["trimedline"] = line[: (n - m)] return dic + def get_dic_from_resumed(line): """ line has to be .... <... xxxx resumed> """ - mo = re.match(r'(\S+)\s+(\S+)\s+\<\.\.\. (\S+) resumed\>', - line) + mo = re.match(r"(\S+)\s+(\S+)\s+\<\.\.\. (\S+) resumed\>", line) dic = {} if mo: - #print mo.groups() + # print mo.groups() i = 1 - dic['pid'] = mo.group(i) + dic["pid"] = mo.group(i) i += 1 - dic['time'] = mo.group(i) + dic["time"] = mo.group(i) i += 1 - dic['callname'] = mo.group(i) + dic["callname"] = mo.group(i) # remove all chars before resumed> - trimedline = re.sub(r'.*\<\.\.\. \S+ resumed\>', "", line) - dic['trimedline'] = trimedline + trimedline = re.sub(r".*\<\.\.\. \S+ resumed\>", "", line) + dic["trimedline"] = trimedline return dic + def maintain_filep(filep, entrydict): """ structure of filep = @@ -166,81 +169,88 @@ def maintain_filep(filep, entrydict): get filepath from fdmap by the fd add 'filepath' to entrydict """ - callname = entrydict['callname'] + callname = entrydict["callname"] # pid = entrydict['pid'] - pid = 0 # they ususally share the same file desciptors. - filepath = 'NA' - offset = 'NA' - length = 'NA' + pid = 0 # they ususally share the same file desciptors. + filepath = "NA" + offset = "NA" + length = "NA" - if entrydict['ret'] == '-1': + if entrydict["ret"] == "-1": # failed.. - entrydict['filepath'] = filepath - entrydict['offset'] = offset - entrydict['length'] = length + entrydict["filepath"] = filepath + entrydict["offset"] = offset + entrydict["length"] = length return - if callname == 'open': - filepath = entrydict['args'][0] - fd = entrydict['ret'] - if not filep.has_key(pid): + if callname == "open": + filepath = entrydict["args"][0] + fd = entrydict["ret"] + if pid not in filep: filep[pid] = {} - if not filep[pid].has_key(fd): + if fd not in filep[pid]: filep[pid][fd] = {} - filep[pid][fd]['filepath'] = filepath - filep[pid][fd]['pos'] = 0 - elif callname == 'openat': - filepath = entrydict['args'][1] - fd = entrydict['ret'] - if not filep.has_key(pid): + filep[pid][fd]["filepath"] = filepath + filep[pid][fd]["pos"] = 0 + elif callname == "openat": + filepath = entrydict["args"][1] + fd = entrydict["ret"] + if pid not in filep: filep[pid] = {} - if not filep[pid].has_key(fd): + if fd not in filep[pid]: filep[pid][fd] = {} - filep[pid][fd]['filepath'] = filepath - filep[pid][fd]['pos'] = 0 - elif callname == 'accept': - filepath = 'NETWORK' - fd = entrydict['ret'] - if not filep.has_key(pid): + filep[pid][fd]["filepath"] = filepath + filep[pid][fd]["pos"] = 0 + elif callname == "accept": + filepath = "NETWORK" + fd = entrydict["ret"] + if pid not in filep: filep[pid] = {} - if not filep[pid].has_key(fd): + if fd not in filep[pid]: filep[pid][fd] = {} - filep[pid][fd]['filepath'] = filepath - filep[pid][fd]['pos'] = 0 - elif callname == 'clone': - newpid = entrydict['ret'] - if 'CLONE_FILES' in entrydict['args'][1]: + filep[pid][fd]["filepath"] = filepath + filep[pid][fd]["pos"] = 0 + elif callname == "clone": + newpid = entrydict["ret"] + if "CLONE_FILES" in entrydict["args"][1]: filep[newpid] = filep[pid] - elif callname in ['dup', 'dup2', 'dup3']: - newfd = entrydict['ret'] - assert newfd != '-1' - oldfd = entrydict['args'][0] - if filep[pid].has_key(oldfd): + elif callname in ["dup", "dup2", "dup3"]: + newfd = entrydict["ret"] + assert newfd != "-1" + oldfd = entrydict["args"][0] + if oldfd in filep[pid]: filep[pid][newfd] = filep[pid][oldfd] else: - print 'dup() an non-existing oldfd' + print("dup() an non-existing oldfd") exit(1) fd = oldfd try: - filepath = filep[pid][fd]['filepath'] + filepath = filep[pid][fd]["filepath"] except: filepath = fd - elif callname == 'unlink': - filepath = entrydict['args'][0] - elif callname == 'close': - fd = entrydict['args'][0] + elif callname == "unlink": + filepath = entrydict["args"][0] + elif callname == "close": + fd = entrydict["args"][0] try: - filepath = filep[pid][fd]['filepath'] + filepath = filep[pid][fd]["filepath"] del filep[pid][fd] except: filepath = fd - elif callname in \ - ['write', 'read', 'pwrite', 'pread', 'fsync', 'fdatasync', 'lseek']: - fd = entrydict['args'][0] + elif callname in [ + "write", + "read", + "pwrite", + "pread", + "fsync", + "fdatasync", + "lseek", + ]: + fd = entrydict["args"][0] try: - filepath = filep[pid][fd]['filepath'] + filepath = filep[pid][fd]["filepath"] except KeyError: # pprint.pprint( entrydict ) # pprint.pprint( filep ) @@ -249,73 +259,73 @@ def maintain_filep(filep, entrydict): # raise try: - if callname in ['write', 'read']: - offset = filep[pid][fd]['pos'] - length = int(entrydict['ret']) - filep[pid][fd]['pos'] = offset + length - elif callname in ['pread', 'pwrite']: + if callname in ["write", "read"]: + offset = filep[pid][fd]["pos"] + length = int(entrydict["ret"]) + filep[pid][fd]["pos"] = offset + length + elif callname in ["pread", "pwrite"]: # they don't affect filep offset - offset = int(entrydict['args'][3]) - length = int(entrydict['ret']) - elif callname in ['lseek']: - #whence = entrydict['args'][2] - #offset = int(entrydict['args'][1]) - offset = int(entrydict['ret']) - filep[pid][fd]['pos'] = int(entrydict['ret']) + offset = int(entrydict["args"][3]) + length = int(entrydict["ret"]) + elif callname in ["lseek"]: + # whence = entrydict['args'][2] + # offset = int(entrydict['args'][1]) + offset = int(entrydict["ret"]) + filep[pid][fd]["pos"] = int(entrydict["ret"]) except KeyError: # print 'pid', pid, 'fd', fd pass - entrydict['filepath'] = filepath - entrydict['offset'] = offset - entrydict['length'] = length + entrydict["filepath"] = filepath + entrydict["offset"] = offset + entrydict["length"] = length def parse_lines(line_iter, pid=None): - unfinished_dic = {} #indexed by call name + unfinished_dic = {} # indexed by call name filep = {} - header=['pid', 'time', 'callname', - 'offset', 'length', 'filepath', 'trace_name'] + header = ["pid", "time", "callname", "offset", "length", "filepath", "trace_name"] - trace_name = 'tr-name' + trace_name = "tr-name" ret_table = [] for line in line_iter: line = line.strip() if match_line_no_pid(line): entrydict = line_to_dic_no_pid(line) - entrydict['pid'] = pid + entrydict["pid"] = pid elif match_line(line): entrydict = line_to_dic(line) elif line.endswith(UNFINISHED_MARK): udic = get_dic_from_unfinished(line) - unfinished_dic[(udic['pid'], udic['callname'])] = udic + unfinished_dic[(udic["pid"], udic["callname"])] = udic continue - elif 'resumed' in line: + elif "resumed" in line: udic = get_dic_from_resumed(line) - name = udic['callname'] - pid = udic['pid'] + name = udic["callname"] + pid = udic["pid"] try: - completeline = unfinished_dic[(pid, name)]['trimedline'] +\ - udic['trimedline'] + completeline = ( + unfinished_dic[(pid, name)]["trimedline"] + udic["trimedline"] + ) entrydict = line_to_dic(completeline) del unfinished_dic[(pid, name)] except Exception as ex: - print ex - print unfinished_dic + print(ex) + print(unfinished_dic) continue - #raise + # raise - maintain_filep( filep, entrydict ) - entrydict['trace_name'] = trace_name + maintain_filep(filep, entrydict) + entrydict["trace_name"] = trace_name ret_table.append(entrydict) return ret_table -class StraceParser(object): +class StraceParser: def __init__(self, lines): self.lines = lines @@ -324,14 +334,14 @@ def parse(self): def scan_trace(tracepath): - pid = tracepath.split('.')[-1] + pid = tracepath.split(".")[-1] try: pid = int(pid) except: pid = None - with open(tracepath, 'r') as f: + with open(tracepath) as f: table = parse_lines(f, pid=pid) return table @@ -340,43 +350,48 @@ def scan_trace(tracepath): def parse_file(filepath): return scan_trace(filepath) + def parse_to_simple_table(filepath): table = scan_trace(filepath) for row in table: try: - del row['args'] + del row["args"] except KeyError: pass return table + def convert_to_dirty_data_table(table): dirty_size_dict = Counter() dirty_table = [] for row in table: - if row['callname'] in ['write', 'pwrite']: - add_dirty_size(dirty_size_dict, row['filepath'], int(row['ret'])) - elif row['callname'] in ['fdatasync', 'fsync']: + if row["callname"] in ["write", "pwrite"]: + add_dirty_size(dirty_size_dict, row["filepath"], int(row["ret"])) + elif row["callname"] in ["fdatasync", "fsync"]: # new row, and ret - filepath = row['filepath'] + filepath = row["filepath"] total = dirty_size_dict[filepath] dirty_size_dict[filepath] = 0 - new_row = {'callname': row['callname'], - 'dirty_size': total, - 'pid': row['pid'], - 'filepath': filepath, - 'time': row['time']} + new_row = { + "callname": row["callname"], + "dirty_size": total, + "pid": row["pid"], + "filepath": filepath, + "time": row["time"], + } dirty_table.append(new_row) return dirty_table + def add_dirty_size(dirty_size_dict, filepath, size): """ dirty_size_dict = {filepath: bytes} """ - if not dirty_size_dict.has_key(filepath): + if filepath not in dirty_size_dict: dirty_size_dict[filepath] = 0 dirty_size_dict[filepath] += size @@ -387,21 +402,20 @@ def parse_and_write_dirty_table(filepath, output_path=None): dirty_tab = convert_to_dirty_data_table(tab) if output_path is None: - output_path = filepath + '.dirty_table' + output_path = filepath + ".dirty_table" - print 'write dirty table to', output_path - with open(output_path, 'w') as f: + print(("write dirty table to", output_path)) + with open(output_path, "w") as f: f.write(table_to_str(dirty_tab, width=0)) def main(): if len(sys.argv) != 2: - print 'usage: python', sys.argv[0], 'tracepath' + print(("usage: python", sys.argv[0], "tracepath")) filepath = sys.argv[1] - print 'Doing', filepath, '...........' + print(("Doing", filepath, "...........")) df = scan_trace(filepath) -if __name__ == '__main__': - main() - +if __name__ == "__main__": + main() diff --git a/wiscsee/tests/__init__.py b/wiscsee/tests/__init__.py index cdf75f2..2f7d08f 100644 --- a/wiscsee/tests/__init__.py +++ b/wiscsee/tests/__init__.py @@ -1,8 +1,8 @@ -import config_test -import flashcontroller_test -import lrulist_test -import misc_test -import recorder_test -import simulator_test -import test_dftldes -import test_bitmap +from . import config_test +from . import flashcontroller_test +from . import lrulist_test +from . import misc_test +from . import recorder_test +from . import simulator_test +from . import test_dftldes +from . import test_bitmap diff --git a/wiscsee/tests/config_test.py b/wiscsee/tests/config_test.py index abb8067..510a622 100644 --- a/wiscsee/tests/config_test.py +++ b/wiscsee/tests/config_test.py @@ -1,7 +1,9 @@ +from past.utils import old_div import unittest import config + class TestConfig(unittest.TestCase): def test_basic(self): conf = config.Config({"para1": "value1"}) @@ -11,16 +13,17 @@ def test_basic(self): def test_default(self): conf = config.Config() - self.assertIn('workload_class', conf) - self.assertIn('expname', conf) + self.assertIn("workload_class", conf) + self.assertIn("expname", conf) def test_sec_translation(self): conf = config.Config() - pagesize = conf['flash_page_size'] - secsize = conf['sector_size'] - page, cnt = conf.sec_ext_to_page_ext(pagesize*3/secsize, - pagesize*2/secsize) + pagesize = conf["flash_page_size"] + secsize = conf["sector_size"] + page, cnt = conf.sec_ext_to_page_ext( + old_div(pagesize * 3, secsize), old_div(pagesize * 2, secsize) + ) self.assertEqual(page, 3) self.assertEqual(cnt, 2) @@ -28,26 +31,23 @@ def test_sec_translation(self): def test_offset_size_translation(self): conf = config.Config() - secsize = conf['sector_size'] - sec, count = conf.off_size_to_sec_count( - offset = secsize * 10, - size = secsize * 31) + secsize = conf["sector_size"] + sec, count = conf.off_size_to_sec_count(offset=secsize * 10, size=secsize * 31) self.assertEqual(sec, 10) self.assertEqual(count, 31) + class TestConfigNewFlash(unittest.TestCase): def test_npages(self): conf = config.ConfigNewFlash() npages1 = conf.total_num_pages() - npages2 = conf['flash_config']['n_pages_per_block'] \ - * conf.n_blocks_per_dev - print 'npages1', npages1 - print 'npages2', npages2 + npages2 = conf["flash_config"]["n_pages_per_block"] * conf.n_blocks_per_dev + print(("npages1", npages1)) + print(("npages2", npages2)) self.assertEqual(npages1, npages2) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() - diff --git a/wiscsee/tests/flashcontroller_test.py b/wiscsee/tests/flashcontroller_test.py index a23446a..7900d72 100644 --- a/wiscsee/tests/flashcontroller_test.py +++ b/wiscsee/tests/flashcontroller_test.py @@ -45,14 +45,15 @@ def setup_workload(self): def setup_ftl(self): pass - def access(self, env, channel): - yield env.process( channel.write_page(None) ) + def access(self, env, channel): + yield env.process(channel.write_page(None)) self.assertEqual(env.now, channel.program_time) - yield env.process( channel.read_page(None) ) + yield env.process(channel.read_page(None)) self.assertEqual(env.now, channel.program_time + channel.read_time) - yield env.process( channel.erase_block(None) ) - self.assertEqual(env.now, channel.program_time + channel.read_time + - channel.erase_time) + yield env.process(channel.erase_block(None)) + self.assertEqual( + env.now, channel.program_time + channel.read_time + channel.erase_time + ) def my_run(self): env = simpy.Environment() @@ -82,24 +83,30 @@ def setup_workload(self): def setup_ftl(self): pass - def access(self, env, controller): + def access(self, env, controller): addr = wiscsim.controller.FlashAddress() addr.channel = 1 channel = controller.channels[addr.channel] - yield env.process( controller.write_page(addr) ) + yield env.process(controller.write_page(addr)) self.assertEqual(env.now, channel.program_time) - yield env.process( controller.read_page(addr) ) + yield env.process(controller.read_page(addr)) self.assertEqual(env.now, channel.program_time + channel.read_time) - yield env.process( controller.erase_block(addr) ) - self.assertEqual(env.now, channel.program_time + channel.read_time + - channel.erase_time) + yield env.process(controller.erase_block(addr)) + self.assertEqual( + env.now, channel.program_time + channel.read_time + channel.erase_time + ) - e1 = env.process( controller.read_page(addr) ) - e2 = env.process( controller.read_page(addr) ) + e1 = env.process(controller.read_page(addr)) + e2 = env.process(controller.read_page(addr)) yield e1 & e2 - self.assertEqual(env.now, channel.program_time + channel.read_time + - channel.erase_time + 2 * channel.read_time) + self.assertEqual( + env.now, + channel.program_time + + channel.read_time + + channel.erase_time + + 2 * channel.read_time, + ) def my_run(self): env = simpy.Environment() @@ -137,7 +144,7 @@ def my_run(self): self.assertEqual(n, 0) addr = controller.physical_to_machine_page(1) - if self.conf['flash_config']['n_pages_per_block'] > 1: + if self.conf["flash_config"]["n_pages_per_block"] > 1: self.assertEqual(addr.page, 1) def test_main(self): @@ -166,36 +173,32 @@ def create_request(self, channel, op): req = wiscsim.controller.FlashRequest() req.addr = wiscsim.controller.FlashAddress() req.addr.channel = channel - if op == 'read': + if op == "read": req.operation = OP_READ - elif op == 'write': + elif op == "write": req.operation = OP_WRITE - elif op == 'erase': + elif op == "erase": req.operation = OP_ERASE else: raise RuntimeError() return req - def access(self, env, controller): - + def access(self, env, controller): channel = controller.channels[1] - yield env.process( controller.execute_request( - self.create_request(1, 'read') ) ) - self.assertEqual( env.now, channel.read_time ) + yield env.process(controller.execute_request(self.create_request(1, "read"))) + self.assertEqual(env.now, channel.read_time) - yield env.process( controller.execute_request( - self.create_request(1, 'read') ) ) - self.assertEqual( env.now, channel.read_time * 2 ) + yield env.process(controller.execute_request(self.create_request(1, "read"))) + self.assertEqual(env.now, channel.read_time * 2) - yield env.process( controller.execute_request( - self.create_request(1, 'write') ) ) - self.assertEqual( env.now, channel.read_time * 2 + channel.program_time) + yield env.process(controller.execute_request(self.create_request(1, "write"))) + self.assertEqual(env.now, channel.read_time * 2 + channel.program_time) - yield env.process( controller.execute_request( - self.create_request(1, 'erase') ) ) - self.assertEqual( env.now, channel.read_time * 2 - + channel.program_time + channel.erase_time) + yield env.process(controller.execute_request(self.create_request(1, "erase"))) + self.assertEqual( + env.now, channel.read_time * 2 + channel.program_time + channel.erase_time + ) def my_run(self): env = simpy.Environment() @@ -245,12 +248,12 @@ def setup_config(self): self.conf = config.ConfigNewFlash() # 2 pages per block, 2 blocks per channel, 2 channels in total - self.conf['flash_config']['n_pages_per_block'] = 2 - self.conf['flash_config']['n_blocks_per_plane'] = 2 - self.conf['flash_config']['n_planes_per_chip'] = 1 - self.conf['flash_config']['n_chips_per_package'] = 1 - self.conf['flash_config']['n_packages_per_channel'] = 1 - self.conf['flash_config']['n_channels_per_dev'] = 2 + self.conf["flash_config"]["n_pages_per_block"] = 2 + self.conf["flash_config"]["n_blocks_per_plane"] = 2 + self.conf["flash_config"]["n_planes_per_chip"] = 1 + self.conf["flash_config"]["n_chips_per_package"] = 1 + self.conf["flash_config"]["n_packages_per_channel"] = 1 + self.conf["flash_config"]["n_channels_per_dev"] = 2 def setup_environment(self): pass @@ -265,36 +268,36 @@ def create_request(self, channel, op): req = wiscsim.controller.FlashRequest() req.addr = wiscsim.controller.FlashAddress() req.addr.channel = channel - if op == 'read': + if op == "read": req.operation = OP_READ - elif op == 'write': + elif op == "write": req.operation = OP_WRITE - elif op == 'erase': + elif op == "erase": req.operation = OP_ERASE else: raise RuntimeError() return req - def access(self, env, controller): + def access(self, env, controller): channel = controller.channels[1] rt = channel.read_time wt = channel.program_time et = channel.erase_time - yield env.process( controller.rw_ppn_extent(0, 2, 'read') ) - self.assertEqual( env.now, rt *2 ) # two pages go to the same channel + yield env.process(controller.rw_ppn_extent(0, 2, "read")) + self.assertEqual(env.now, rt * 2) # two pages go to the same channel - yield env.process( controller.rw_ppn_extent(4, 2, 'write') ) - self.assertEqual( env.now, rt *2 + wt*2 ) # two pages go to the same channel + yield env.process(controller.rw_ppn_extent(4, 2, "write")) + self.assertEqual(env.now, rt * 2 + wt * 2) # two pages go to the same channel - yield env.process( controller.rw_ppn_extent(0, 8, 'write') ) + yield env.process(controller.rw_ppn_extent(0, 8, "write")) # 4 pages go to one channel - self.assertEqual( env.now, rt *2 + wt*2 + wt*4 ) + self.assertEqual(env.now, rt * 2 + wt * 2 + wt * 4) # one req goes to channel 1, another one goes to channel 2 - yield env.process( controller.erase_pbn_extent(1, 2) ) - self.assertEqual( env.now, rt *2 + wt*2 + wt*4 + et ) + yield env.process(controller.erase_pbn_extent(1, 2)) + self.assertEqual(env.now, rt * 2 + wt * 2 + wt * 4 + et) def my_run(self): env = simpy.Environment() @@ -309,25 +312,25 @@ def test_main(self): self.setup_ftl() self.my_run() + class TestControllerTime(unittest.TestCase): def setup_config(self): self.conf = config.ConfigNewFlash() # 2 pages per block, 2 blocks per channel, 2 channels in total - self.conf['flash_config']['n_pages_per_block'] = 2 - self.conf['flash_config']['n_blocks_per_plane'] = 2 - self.conf['flash_config']['n_planes_per_chip'] = 1 - self.conf['flash_config']['n_chips_per_package'] = 1 - self.conf['flash_config']['n_packages_per_channel'] = 1 - self.conf['flash_config']['n_channels_per_dev'] = 2 - - self.conf['flash_config']['t_WC'] = 1 - self.conf['flash_config']['t_R'] = 1 - self.conf['flash_config']['t_RC'] = 1 - self.conf['flash_config']['t_PROG'] = 1 - self.conf['flash_config']['t_BERS'] = 1 - self.conf['flash_config']['page_size'] = 1 - + self.conf["flash_config"]["n_pages_per_block"] = 2 + self.conf["flash_config"]["n_blocks_per_plane"] = 2 + self.conf["flash_config"]["n_planes_per_chip"] = 1 + self.conf["flash_config"]["n_chips_per_package"] = 1 + self.conf["flash_config"]["n_packages_per_channel"] = 1 + self.conf["flash_config"]["n_channels_per_dev"] = 2 + + self.conf["flash_config"]["t_WC"] = 1 + self.conf["flash_config"]["t_R"] = 1 + self.conf["flash_config"]["t_RC"] = 1 + self.conf["flash_config"]["t_PROG"] = 1 + self.conf["flash_config"]["t_BERS"] = 1 + self.conf["flash_config"]["page_size"] = 1 def setup_environment(self): pass @@ -353,17 +356,18 @@ def test_main(self): self.setup_ftl() self.my_run() + class TestControllerTag(unittest.TestCase): def setup_config(self): self.conf = config.ConfigNewFlash() # 2 pages per block, 2 blocks per channel, 2 channels in total - self.conf['flash_config']['n_pages_per_block'] = 2 - self.conf['flash_config']['n_blocks_per_plane'] = 2 - self.conf['flash_config']['n_planes_per_chip'] = 1 - self.conf['flash_config']['n_chips_per_package'] = 1 - self.conf['flash_config']['n_packages_per_channel'] = 1 - self.conf['flash_config']['n_channels_per_dev'] = 2 + self.conf["flash_config"]["n_pages_per_block"] = 2 + self.conf["flash_config"]["n_blocks_per_plane"] = 2 + self.conf["flash_config"]["n_planes_per_chip"] = 1 + self.conf["flash_config"]["n_chips_per_package"] = 1 + self.conf["flash_config"]["n_packages_per_channel"] = 1 + self.conf["flash_config"]["n_channels_per_dev"] = 2 def setup_environment(self): pass @@ -378,54 +382,55 @@ def create_request(self, channel, op): req = wiscsim.controller.FlashRequest() req.addr = wiscsim.controller.FlashAddress() req.addr.channel = channel - if op == 'read': + if op == "read": req.operation = OP_READ - elif op == 'write': + elif op == "write": req.operation = OP_WRITE - elif op == 'erase': + elif op == "erase": req.operation = OP_ERASE else: raise RuntimeError() return req - def access(self, env, controller): + def access(self, env, controller): channel = controller.channels[0] rt = channel.read_time wt = channel.program_time et = channel.erase_time - yield env.process( controller.rw_ppn_extent(0, 2, 'read', - tag = 'mytag1') ) - self.assertEqual( env.now, rt *2 ) # two pages go to the same channel + yield env.process(controller.rw_ppn_extent(0, 2, "read", tag="mytag1")) + self.assertEqual(env.now, rt * 2) # two pages go to the same channel self.assertEqual( - controller.recorder.\ - general_accumulator['channel_busy_time']['channel_0-read-mytag1'], rt * 2) + controller.recorder.general_accumulator["channel_busy_time"][ + "channel_0-read-mytag1" + ], + rt * 2, + ) - yield env.process( controller.rw_ppn_extent(4, 2, 'write', - tag = 'mytag2') ) - self.assertEqual( env.now, rt *2 + wt*2 ) # two pages go to the same channel + yield env.process(controller.rw_ppn_extent(4, 2, "write", tag="mytag2")) + self.assertEqual(env.now, rt * 2 + wt * 2) # two pages go to the same channel - yield env.process( controller.rw_ppn_extent(0, 8, 'write', - tag = 'mytag2') ) + yield env.process(controller.rw_ppn_extent(0, 8, "write", tag="mytag2")) # 4 pages go to one channel - self.assertEqual( env.now, rt *2 + wt*2 + wt*4 ) + self.assertEqual(env.now, rt * 2 + wt * 2 + wt * 4) # one req goes to channel 1, another one goes to channel 2 - yield env.process( controller.erase_pbn_extent(1, 2, tag = 'mytag3') ) - self.assertEqual( env.now, rt *2 + wt*2 + wt*4 + et ) + yield env.process(controller.erase_pbn_extent(1, 2, tag="mytag3")) + self.assertEqual(env.now, rt * 2 + wt * 2 + wt * 4 + et) def my_run(self): env = simpy.Environment() - set_exp_metadata(self.conf, save_data = False, - expname = 'default', - subexpname = 'default-sub') + set_exp_metadata( + self.conf, save_data=False, expname="default", subexpname="default-sub" + ) runtime_update(self.conf) - rec = wiscsim.recorder.Recorder(output_target = self.conf['output_target'], - output_directory = self.conf['result_dir'], - verbose_level = self.conf['verbose_level'], - print_when_finished = False - ) + rec = wiscsim.recorder.Recorder( + output_target=self.conf["output_target"], + output_directory=self.conf["result_dir"], + verbose_level=self.conf["verbose_level"], + print_when_finished=False, + ) rec.enable() controller = wiscsim.controller.Controller3(env, self.conf, rec) @@ -440,12 +445,9 @@ def test_main(self): self.my_run() - def main(): unittest.main() -if __name__ == '__main__': - main() - - +if __name__ == "__main__": + main() diff --git a/wiscsee/tests/lrulist_test.py b/wiscsee/tests/lrulist_test.py index 3f9b8f7..0e622bc 100644 --- a/wiscsee/tests/lrulist_test.py +++ b/wiscsee/tests/lrulist_test.py @@ -25,20 +25,21 @@ def test1(self): self.assertEqual(lrucache.peek(1), 111) del lrucache[1] - self.assertEqual(lrucache.has_key(1), False) + self.assertEqual(1 in lrucache, False) + class Test_LruCache(unittest.TestCase): def get_lrucache(self): d = LruCache() for i in range(10): - d[i] = i*10 + d[i] = i * 10 return d def test_init(self): d = LruCache() - d = LruCache({1:2}) + d = LruCache({1: 2}) d = LruCache(((1, 2), (2, 3))) - d = LruCache(a = 1, b = 2) + d = LruCache(a=1, b=2) def test1(self): d = self.get_lrucache() @@ -52,11 +53,11 @@ def test_del(self): del d[2] self.assertEqual(len(d), 9) - self.assertEqual(d.has_key(2), False) + self.assertEqual(2 in d, False) def test_iter(self): d = self.get_lrucache() - self.assertListEqual(list(d), list(reversed(range(10)))) + self.assertListEqual(list(d), list(reversed(list(range(10))))) def test_reversed(self): d = self.get_lrucache() @@ -66,7 +67,7 @@ def test_items(self): d = self.get_lrucache() lk = [] lv = [] - for k, v in d.items(): + for k, v in list(d.items()): # suppose to go from least to most recently lk.append(k) lv.append(v) @@ -121,28 +122,25 @@ def test_add_to_least_used(self): def _test_performance(self): d = LruDict() for i in range(2048): - d[i] = i+1 - + d[i] = i + 1 def go_through(self, d): for k, v in d.least_to_most_items(): v = 1 - - class Test_LruDict(unittest.TestCase): def get_lrudict(self): d = LruDict() for i in range(10): - d[i] = i*10 + d[i] = i * 10 return d def test_init(self): d = LruDict() - d = LruDict({1:2}) + d = LruDict({1: 2}) d = LruDict(((1, 2), (2, 3))) - d = LruDict(a = 1, b = 2) + d = LruDict(a=1, b=2) def test1(self): d = self.get_lrudict() @@ -156,7 +154,7 @@ def test_del(self): del d[2] self.assertEqual(len(d), 9) - self.assertEqual(d.has_key(2), False) + self.assertEqual(2 in d, False) def test_iter(self): d = self.get_lrudict() @@ -164,13 +162,13 @@ def test_iter(self): def test_revsersed(self): d = self.get_lrudict() - self.assertListEqual(list(reversed(d)), list(reversed(range(10)))) + self.assertListEqual(list(reversed(d)), list(reversed(list(range(10))))) def test_items(self): d = self.get_lrudict() lk = [] lv = [] - for k, v in d.items(): + for k, v in list(d.items()): lk.append(k) lv.append(v) self.assertListEqual(lk, list(range(10))) @@ -217,7 +215,7 @@ def test_peek(self): def _test_performance(self): d = LruDict() for i in range(2048): - d[i] = i+1 + d[i] = i + 1 self.go_through(d) @@ -227,7 +225,8 @@ def go_through(self, d): def has_key(d, key): - return d.has_key(key) + return key in d + def compare_dict_performance(): # d = LruCache() @@ -239,12 +238,10 @@ def compare_dict_performance(): # d.has_key(i) has_key(d, i) + def main(): unittest.main() - -if __name__ == '__main__': +if __name__ == "__main__": main() - - diff --git a/wiscsee/tests/misc_test.py b/wiscsee/tests/misc_test.py index 7eb9bc1..f7da196 100644 --- a/wiscsee/tests/misc_test.py +++ b/wiscsee/tests/misc_test.py @@ -1,3 +1,4 @@ +from past.utils import old_div import socket import unittest import time @@ -17,6 +18,7 @@ from config_helper.rule_parameter import EventFileSets from commons import * + class TestCpuhandler(unittest.TestCase): def test_cpu(self): possible_cpus = workrunner.cpuhandler.get_possible_cpus() @@ -25,6 +27,7 @@ def test_cpu(self): online_cpus = workrunner.cpuhandler.get_online_cpuids() self.assertListEqual(possible_cpus, online_cpus) + class TestRandomChannelID(unittest.TestCase): def test(self): n = 16 @@ -36,12 +39,13 @@ def test(self): self.assertTrue(channel_id >= 0) self.assertTrue(channel_id < n) - self.assertTrue(len(channels) > n/3) + self.assertTrue(len(channels) > old_div(n, 3)) + @unittest.skip("Need real device that supports NCQ") class TestLinuxNCQDepth(unittest.TestCase): def test_ncq_depth_setting(self): - if not 'wisc.cloudlab.us' in socket.gethostname(): + if not "wisc.cloudlab.us" in socket.gethostname(): return depth = 2 @@ -49,66 +53,70 @@ def test_ncq_depth_setting(self): read_depth = utils.get_linux_ncq_depth("sdc") self.assertEqual(depth, read_depth) + @unittest.skip("Need real device that supports setting scheduler") class TestSettingScheduler(unittest.TestCase): def test_setting(self): - scheduler = 'noop' + scheduler = "noop" utils.set_linux_io_scheduler("sdc", scheduler) read_scheduler = utils.get_linux_io_scheduler("sdc") self.assertEqual(scheduler, read_scheduler) -class Experiment(object): +class Experiment: def __init__(self): self.conf = wiscsim.dftldes.Config() def setup_environment(self): - self.conf['device_path'] = '/dev/loop0' - self.conf['dev_size_mb'] = 256 - self.conf['filesystem'] = 'f2fs' - self.conf["n_online_cpus"] = 'all' + self.conf["device_path"] = "/dev/loop0" + self.conf["dev_size_mb"] = 256 + self.conf["filesystem"] = "f2fs" + self.conf["n_online_cpus"] = "all" - self.conf['linux_ncq_depth'] = 31 - self.conf['sort_block_trace'] = True + self.conf["linux_ncq_depth"] = 31 + self.conf["sort_block_trace"] = True def setup_workload(self): - self.conf['workload_class'] = 'NoOp' - self.conf['NoOp'] = {} - self.conf['workload_conf_key'] = 'NoOp' + self.conf["workload_class"] = "NoOp" + self.conf["NoOp"] = {} + self.conf["workload_conf_key"] = "NoOp" def setup_fs(self): pass # self.conf['mnt_opts'].update({ - # "f2fs": { - # 'discard': MOpt(opt_name = 'discard', - # value = 'discard', - # include_name = False), - # 'background_gc': MOpt(opt_name = 'background_gc', - # value = 'off', - # include_name = True) - # } - # } - # ) + # "f2fs": { + # 'discard': MOpt(opt_name = 'discard', + # value = 'discard', + # include_name = False), + # 'background_gc': MOpt(opt_name = 'background_gc', + # value = 'off', + # include_name = True) + # } + # } + # ) def setup_flash(self): pass def setup_ftl(self): - self.conf['enable_blktrace'] = False - self.conf['enable_simulation'] = False + self.conf["enable_blktrace"] = False + self.conf["enable_simulation"] = False def run(self): - utils.set_exp_metadata(self.conf, save_data = False, - expname = 'tmp', - subexpname = 'subtmp') + utils.set_exp_metadata( + self.conf, save_data=False, expname="tmp", subexpname="subtmp" + ) utils.runtime_update(self.conf) run_workflow(self.conf) - utils.shcmd("fio -name hello -rw=randwrite -size=16mb -fsync=1 -filename {}/data2"\ - .format(self.conf['fs_mount_point'])) + utils.shcmd( + "fio -name hello -rw=randwrite -size=16mb -fsync=1 -filename {}/data2".format( + self.conf["fs_mount_point"] + ) + ) time.sleep(1) - ret = utils.invoke_f2fs_gc(self.conf['fs_mount_point'], 1) + ret = utils.invoke_f2fs_gc(self.conf["fs_mount_point"], 1) assert ret == 0 def main(self): @@ -119,28 +127,33 @@ def main(self): self.setup_ftl() self.run() + @unittest.skip("Need FIO to create some random workload to create F2FS garbage") class TestF2FSGCCall(unittest.TestCase): def test(self): obj = Experiment() obj.main() + class TestImportPyreuse(unittest.TestCase): def test(self): import pyreuse + pyreuse.helpers.shcmd("echo 33333") + class TestClassifyGcLOG(unittest.TestCase): @unittest.skip("Need real device mounted") def test(self): gclog = GcLog( - device_path='/dev/sdc1', - result_dir='/tmp/results/1gbnojournalok/Leveldb.ext4.1gbnojournalok.4294967296.devsdc1.128.31.1073741824.1073741824.True.ordered.False.overwrite.1000000.1.True.64.4.4.2000000.dftldes.1-ext4-06-29-08-06-52--7574556694461561217', - flash_page_size=2048 - ) - print gclog._get_range_table() + device_path="/dev/sdc1", + result_dir="/tmp/results/1gbnojournalok/Leveldb.ext4.1gbnojournalok.4294967296.devsdc1.128.31.1073741824.1073741824.True.ordered.False.overwrite.1000000.1.True.64.4.4.2000000.dftldes.1-ext4-06-29-08-06-52--7574556694461561217", + flash_page_size=2048, + ) + print(gclog._get_range_table()) gclog.classify_lpn_in_gclog() + class TestExtent(unittest.TestCase): def test_copy(self): ext1 = Extent(lpn_start=3, lpn_count=8) @@ -161,20 +174,20 @@ def test(self): self.assertListEqual(batches, [[0], [1], [2]]) def test_larger(self): - a = range(7) + a = list(range(7)) batches = utils.group_to_batches(a, 3) - self.assertListEqual(batches, [[0, 1, 2], - [3, 4, 5], - [6]]) + self.assertListEqual(batches, [[0, 1, 2], [3, 4, 5], [6]]) -@unittest.skip('need real device') + +@unittest.skip("need real device") class TestLpnClassification(unittest.TestCase): def test(self): classifier = LpnClassification( - lpns = [1, 8], - device_path = '/dev/sdc1', - result_dir = '/tmp/results/test002/subexp--6155052293192590053-ext4-09-16-09-32-22-1439596482389025085', - flash_page_size = 2048) + lpns=[1, 8], + device_path="/dev/sdc1", + result_dir="/tmp/results/test002/subexp--6155052293192590053-ext4-09-16-09-32-22-1439596482389025085", + flash_page_size=2048, + ) classifier.classify() @@ -184,36 +197,37 @@ def test(self): conf = ConfigNCQFTL() workload_line_iter = hostevent.FileLineIterator( - "tests/testdata/blkparse-events-for-ftlsim.txt") + "tests/testdata/blkparse-events-for-ftlsim.txt" + ) event_workload_iter = hostevent.EventIterator(conf, workload_line_iter) # for event in event_workload_iter: - # print str(event) + # print str(event) parser = EventNCQParser(event_workload_iter) table = parser.parse() - self.assertEqual(table[0]['pre_depth'], 0) - self.assertEqual(table[0]['post_depth'], 1) + self.assertEqual(table[0]["pre_depth"], 0) + self.assertEqual(table[0]["post_depth"], 1) - self.assertEqual(table[1]['pre_depth'], 1) - self.assertEqual(table[1]['post_depth'], 0) + self.assertEqual(table[1]["pre_depth"], 1) + self.assertEqual(table[1]["post_depth"], 0) class TestEventFileSets(unittest.TestCase): def test(self): - filesets = EventFileSets('tests/testdata/64mbfile') + filesets = EventFileSets("tests/testdata/64mbfile") sets = filesets.get_sets() self.assertEqual(len(sets), 1) - self.assertEqual(sets[0]['mkfs_path'], - 'tests/testdata/64mbfile/subexp-3563455040949707047-ext4-10-05-16-29-19-3141981191822244772/blkparse-events-for-ftlsim-mkfs.txt') - + self.assertEqual( + sets[0]["mkfs_path"], + "tests/testdata/64mbfile/subexp-3563455040949707047-ext4-10-05-16-29-19-3141981191822244772/blkparse-events-for-ftlsim-mkfs.txt", + ) def main(): unittest.main() -if __name__ == '__main__': - main() - +if __name__ == "__main__": + main() diff --git a/wiscsee/tests/recorder_test.py b/wiscsee/tests/recorder_test.py index 365a013..9518f54 100644 --- a/wiscsee/tests/recorder_test.py +++ b/wiscsee/tests/recorder_test.py @@ -4,16 +4,17 @@ import wiscsim from utilities.utils import * + class TestFTLwithDFTL(unittest.TestCase): def setup_config(self): self.conf = config.ConfigNCQFTL() def setup_environment(self): - metadata_dic = choose_exp_metadata(self.conf, interactive = False) + metadata_dic = choose_exp_metadata(self.conf, interactive=False) self.conf.update(metadata_dic) - self.conf['enable_blktrace'] = True - self.conf['enable_simulation'] = True + self.conf["enable_blktrace"] = True + self.conf["enable_simulation"] = True def setup_workload(self): pass @@ -25,15 +26,13 @@ def my_run(self): runtime_update(self.conf) recorder = wiscsim.recorder.Recorder( - output_target = wiscsim.recorder.FILE_TARGET, - output_directory = "/tmp" - ) + output_target=wiscsim.recorder.FILE_TARGET, output_directory="/tmp" + ) recorder.enable() recorder.add_to_general_accumulater("counter_set_1", "counter1", 3) recorder.add_to_general_accumulater("counter_set_1", "counter1", 4) - self.assertEqual( - recorder.general_accumulator["counter_set_1"]["counter1"], 7) + self.assertEqual(recorder.general_accumulator["counter_set_1"]["counter1"], 7) def test_main(self): self.setup_config() @@ -42,16 +41,17 @@ def test_main(self): self.setup_ftl() self.my_run() + class TestCountMe(unittest.TestCase): def setup_config(self): self.conf = config.ConfigNCQFTL() def setup_environment(self): - metadata_dic = choose_exp_metadata(self.conf, interactive = False) + metadata_dic = choose_exp_metadata(self.conf, interactive=False) self.conf.update(metadata_dic) - self.conf['enable_blktrace'] = True - self.conf['enable_simulation'] = True + self.conf["enable_blktrace"] = True + self.conf["enable_simulation"] = True def setup_workload(self): pass @@ -63,9 +63,8 @@ def my_run(self): runtime_update(self.conf) recorder = wiscsim.recorder.Recorder( - output_target = wiscsim.recorder.FILE_TARGET, - output_directory = '/tmp' - ) + output_target=wiscsim.recorder.FILE_TARGET, output_directory="/tmp" + ) recorder.enable() recorder.count_me("counter_name_1", "item1") @@ -80,14 +79,9 @@ def test_main(self): self.my_run() - def main(): unittest.main() -if __name__ == '__main__': - main() - - - - +if __name__ == "__main__": + main() diff --git a/wiscsee/tests/simulator_test.py b/wiscsee/tests/simulator_test.py index 246ba85..080bae2 100644 --- a/wiscsee/tests/simulator_test.py +++ b/wiscsee/tests/simulator_test.py @@ -47,11 +47,6 @@ def use_one_slot(self, env, ncq): def main(): unittest.main() -if __name__ == '__main__': - main() - - - - - +if __name__ == "__main__": + main() diff --git a/wiscsee/tests/test_bitmap.py b/wiscsee/tests/test_bitmap.py index 5917210..23a7b40 100644 --- a/wiscsee/tests/test_bitmap.py +++ b/wiscsee/tests/test_bitmap.py @@ -4,20 +4,21 @@ from utilities import utils from wiscsim.bitmap import FlashBitmap2 + def create_config(): conf = wiscsim.dftldes.Config() - conf['SSDFramework']['ncq_depth'] = 1 + conf["SSDFramework"]["ncq_depth"] = 1 - conf['flash_config']['n_pages_per_block'] = 64 - conf['flash_config']['n_blocks_per_plane'] = 2 - conf['flash_config']['n_planes_per_chip'] = 1 - conf['flash_config']['n_chips_per_package'] = 1 - conf['flash_config']['n_packages_per_channel'] = 1 - conf['flash_config']['n_channels_per_dev'] = 4 + conf["flash_config"]["n_pages_per_block"] = 64 + conf["flash_config"]["n_blocks_per_plane"] = 2 + conf["flash_config"]["n_planes_per_chip"] = 1 + conf["flash_config"]["n_chips_per_package"] = 1 + conf["flash_config"]["n_packages_per_channel"] = 1 + conf["flash_config"]["n_channels_per_dev"] = 4 - utils.set_exp_metadata(conf, save_data = False, - expname = 'test_expname', - subexpname = 'test_subexpname') + utils.set_exp_metadata( + conf, save_data=False, expname="test_expname", subexpname="test_subexpname" + ) logicsize_mb = 64 conf.n_cache_entries = conf.n_mapping_entries_per_page @@ -27,6 +28,7 @@ def create_config(): return conf + def create_bitmap(conf): bitmap = FlashBitmap2(conf) return bitmap @@ -64,16 +66,12 @@ def test_invalidating(self): bitmap.invalidate_page(0) self.assertEqual(bitmap.is_page_invalid(0), True) self.assertEqual(bitmap.page_state(0), bitmap.INVALID) - self.assertEqual(bitmap.block_valid_ratio(0), - 1 - 1.0/conf.n_pages_per_block) + self.assertEqual(bitmap.block_valid_ratio(0), 1 - 1.0 / conf.n_pages_per_block) def main(): unittest.main() -if __name__ == '__main__': - main() - - - +if __name__ == "__main__": + main() diff --git a/wiscsee/tests/test_blockpool.py b/wiscsee/tests/test_blockpool.py index ce9fb12..60078ee 100644 --- a/wiscsee/tests/test_blockpool.py +++ b/wiscsee/tests/test_blockpool.py @@ -1,9 +1,11 @@ +from past.utils import old_div import unittest from utilities.utils import * from wiscsim.blkpool import * import wiscsim + def create_blockpool(conf): return wiscsim.dftldes.BlockPool(conf) @@ -11,31 +13,31 @@ def create_blockpool(conf): def create_config(): conf = wiscsim.dftldes.Config() - conf['flash_config']['n_pages_per_block'] = 64 - conf['flash_config']['n_blocks_per_plane'] = 2 - conf['flash_config']['n_planes_per_chip'] = 1 - conf['flash_config']['n_chips_per_package'] = 1 - conf['flash_config']['n_packages_per_channel'] = 1 - conf['flash_config']['n_channels_per_dev'] = 4 + conf["flash_config"]["n_pages_per_block"] = 64 + conf["flash_config"]["n_blocks_per_plane"] = 2 + conf["flash_config"]["n_planes_per_chip"] = 1 + conf["flash_config"]["n_chips_per_package"] = 1 + conf["flash_config"]["n_packages_per_channel"] = 1 + conf["flash_config"]["n_channels_per_dev"] = 4 - set_exp_metadata(conf, save_data = False, - expname = 'test_expname', - subexpname = 'test_subexpname') + set_exp_metadata( + conf, save_data=False, expname="test_expname", subexpname="test_subexpname" + ) runtime_update(conf) return conf - class TestBlockPool_data(unittest.TestCase): """ Test pop_a_free_block_data """ + def setup_config(self): self.conf = wiscsim.dftlext.Config() def setup_environment(self): - metadata_dic = choose_exp_metadata(self.conf, interactive = False) + metadata_dic = choose_exp_metadata(self.conf, interactive=False) self.conf.update(metadata_dic) def setup_workload(self): @@ -62,10 +64,9 @@ def my_run(self): for i in range(n_channels): self.assertEqual( - block_pool.count_blocks(tag=TFREE, channels=[i]), - n_blocks_per_channel) - self.assertEqual( - block_pool.count_blocks(tag=TTRANS, channels=[i]), 0) + block_pool.count_blocks(tag=TFREE, channels=[i]), n_blocks_per_channel + ) + self.assertEqual(block_pool.count_blocks(tag=TTRANS, channels=[i]), 0) def test_main(self): self.setup_config() @@ -79,11 +80,12 @@ class TestBlockPool_next_data(unittest.TestCase): """ Test pop_a_free_block_data """ + def setup_config(self): self.conf = wiscsim.dftlext.Config() def setup_environment(self): - metadata_dic = choose_exp_metadata(self.conf, interactive = False) + metadata_dic = choose_exp_metadata(self.conf, interactive=False) self.conf.update(metadata_dic) def setup_workload(self): @@ -103,13 +105,18 @@ def my_run(self): block_pool.next_data_page_to_program() # each channel now has 2 less blocks - nblocks_used = (k + self.conf['flash_config']['n_pages_per_block'] - 1) / \ - self.conf['flash_config']['n_pages_per_block'] + nblocks_used = old_div( + (k + self.conf["flash_config"]["n_pages_per_block"] - 1), + self.conf["flash_config"]["n_pages_per_block"], + ) for i in range(n_channels): - self.assertEqual(block_pool.count_blocks(tag=TFREE, channels=[i]), - n_blocks_per_channel - nblocks_used) self.assertEqual( - block_pool.count_blocks(tag=TDATA, channels=[i]), nblocks_used) + block_pool.count_blocks(tag=TFREE, channels=[i]), + n_blocks_per_channel - nblocks_used, + ) + self.assertEqual( + block_pool.count_blocks(tag=TDATA, channels=[i]), nblocks_used + ) def test_main(self): self.setup_config() @@ -123,11 +130,12 @@ class TestBlockPool_next_gc_data(unittest.TestCase): """ Test pop_a_free_block_data """ + def setup_config(self): self.conf = wiscsim.dftlext.Config() def setup_environment(self): - metadata_dic = choose_exp_metadata(self.conf, interactive = False) + metadata_dic = choose_exp_metadata(self.conf, interactive=False) self.conf.update(metadata_dic) def setup_workload(self): @@ -147,15 +155,18 @@ def my_run(self): block_pool.next_gc_data_page_to_program() # each channel now has 2 less blocks - nblocks_used = (k + self.conf['flash_config']['n_pages_per_block'] - 1) / \ - self.conf['flash_config']['n_pages_per_block'] + nblocks_used = old_div( + (k + self.conf["flash_config"]["n_pages_per_block"] - 1), + self.conf["flash_config"]["n_pages_per_block"], + ) for i in range(n_channels): self.assertEqual( block_pool.count_blocks(tag=TFREE, channels=[i]), - n_blocks_per_channel - nblocks_used) + n_blocks_per_channel - nblocks_used, + ) self.assertEqual( - block_pool.count_blocks(tag=TDATA, channels=[i]), - nblocks_used) + block_pool.count_blocks(tag=TDATA, channels=[i]), nblocks_used + ) def test_main(self): self.setup_config() @@ -165,16 +176,16 @@ def test_main(self): self.my_run() - class TestBlockPool_stripping(unittest.TestCase): """ Test pop_a_free_block_data """ + def setup_config(self): self.conf = create_config() def setup_environment(self): - metadata_dic = choose_exp_metadata(self.conf, interactive = False) + metadata_dic = choose_exp_metadata(self.conf, interactive=False) self.conf.update(metadata_dic) def setup_workload(self): @@ -184,12 +195,12 @@ def setup_ftl(self): pass def ppn_to_channel(self, ppn): - return ppn / self.conf.n_pages_per_channel + return old_div(ppn, self.conf.n_pages_per_channel) def my_run1(self): runtime_update(self.conf) - self.conf['stripe_size'] = 2 + self.conf["stripe_size"] = 2 block_pool = wiscsim.blkpool.BlockPool(self.conf) block_pool.pool._next_channel = 0 n_channels = block_pool.n_channels @@ -197,15 +208,13 @@ def my_run1(self): block_pool.pool._next_channel = 0 n = 3 - ppns_to_write = block_pool.next_n_data_pages_to_program_striped( - n = n) + ppns_to_write = block_pool.next_n_data_pages_to_program_striped(n=n) self.assertEqual(len(ppns_to_write), n) self.assertEqual(self.ppn_to_channel(ppns_to_write[0]), 0) self.assertEqual(self.ppn_to_channel(ppns_to_write[1]), 0) self.assertEqual(self.ppn_to_channel(ppns_to_write[2]), 1) - ppns_to_write = block_pool.next_n_data_pages_to_program_striped( - n = n) + ppns_to_write = block_pool.next_n_data_pages_to_program_striped(n=n) self.assertEqual(len(ppns_to_write), n) self.assertEqual(self.ppn_to_channel(ppns_to_write[0]), 2) self.assertEqual(self.ppn_to_channel(ppns_to_write[1]), 2) @@ -214,15 +223,14 @@ def my_run1(self): def my_run2(self): runtime_update(self.conf) - self.conf['stripe_size'] = 1 + self.conf["stripe_size"] = 1 block_pool = wiscsim.blkpool.BlockPool(self.conf) block_pool.pool._next_channel = 0 n_channels = block_pool.n_channels n_blocks_per_channel = self.conf.n_blocks_per_channel n = 5 - ppns_to_write = block_pool.next_n_data_pages_to_program_striped( - n = n) + ppns_to_write = block_pool.next_n_data_pages_to_program_striped(n=n) self.assertEqual(len(ppns_to_write), n) self.assertEqual(self.ppn_to_channel(ppns_to_write[0]), 0) self.assertEqual(self.ppn_to_channel(ppns_to_write[1]), 1) @@ -233,15 +241,14 @@ def my_run2(self): def my_run3_inf_stripesize(self): runtime_update(self.conf) - self.conf['stripe_size'] = float('inf') + self.conf["stripe_size"] = float("inf") block_pool = wiscsim.blkpool.BlockPool(self.conf) block_pool.pool._next_channel = 0 n_channels = block_pool.n_channels n_blocks_per_channel = self.conf.n_blocks_per_channel n = 5 - ppns_to_write = block_pool.next_n_data_pages_to_program_striped( - n = n) + ppns_to_write = block_pool.next_n_data_pages_to_program_striped(n=n) self.assertEqual(len(ppns_to_write), n) for i in range(n): self.assertEqual(self.ppn_to_channel(ppns_to_write[i]), 0) @@ -260,11 +267,12 @@ class TestBlockPool_outofspace(unittest.TestCase): """ Test pop_a_free_block_data """ + def setup_config(self): self.conf = create_config() def setup_environment(self): - metadata_dic = choose_exp_metadata(self.conf, interactive = False) + metadata_dic = choose_exp_metadata(self.conf, interactive=False) self.conf.update(metadata_dic) def setup_workload(self): @@ -274,19 +282,18 @@ def setup_ftl(self): pass def ppn_to_channel(self, ppn): - return ppn / self.conf.n_pages_per_channel + return old_div(ppn, self.conf.n_pages_per_channel) def my_run_1(self): runtime_update(self.conf) - self.conf['stripe_size'] = 2 + self.conf["stripe_size"] = 2 block_pool = wiscsim.blkpool.BlockPool(self.conf) n_channels = block_pool.n_channels n_blocks_per_channel = self.conf.n_blocks_per_channel n = self.conf.total_num_pages() - ppns_to_write = block_pool.next_n_data_pages_to_program_striped( - n = n) + ppns_to_write = block_pool.next_n_data_pages_to_program_striped(n=n) # should not have exception @@ -299,12 +306,11 @@ def my_run_2(self): n = self.conf.total_num_pages() + 1 with self.assertRaises(wiscsim.blkpool.OutOfSpaceError): - ppns_to_write = block_pool.next_n_data_pages_to_program_striped( - n = n) + ppns_to_write = block_pool.next_n_data_pages_to_program_striped(n=n) def my_run_3(self): runtime_update(self.conf) - self.conf['stripe_size'] = float('inf') + self.conf["stripe_size"] = float("inf") block_pool = wiscsim.blkpool.BlockPool(self.conf) n_channels = block_pool.n_channels n_blocks_per_channel = self.conf.n_blocks_per_channel @@ -312,8 +318,7 @@ def my_run_3(self): n = self.conf.total_num_pages() + 1 with self.assertRaises(wiscsim.blkpool.OutOfSpaceError): - ppns_to_write = block_pool.next_n_data_pages_to_program_striped( - n = n) + ppns_to_write = block_pool.next_n_data_pages_to_program_striped(n=n) def test_main(self): self.setup_config() @@ -347,7 +352,7 @@ def test_use_some(self): pool.move_used_trans_block_to_free(block) dist = pool.get_erasure_count_dist() - self.assertEqual(dist[0], nblocks-1) + self.assertEqual(dist[0], nblocks - 1) self.assertEqual(dist[1], 1) self.assertEqual(sum(dist.values()), nblocks) @@ -355,6 +360,6 @@ def test_use_some(self): def main(): unittest.main() -if __name__ == '__main__': - main() +if __name__ == "__main__": + main() diff --git a/wiscsee/tests/test_demo.py b/wiscsee/tests/test_demo.py index 0b773e0..62d4d00 100644 --- a/wiscsee/tests/test_demo.py +++ b/wiscsee/tests/test_demo.py @@ -17,15 +17,15 @@ class Test_TraceOnly(unittest.TestCase): def test_run(self): class LocalExperiment(experiment.Experiment): def setup_workload(self): - self.conf['workload_class'] = "SimpleRandReadWrite" + self.conf["workload_class"] = "SimpleRandReadWrite" - para = experiment.get_shared_nolist_para_dict("test_exp_TraceOnly2", 16*MB) - para['device_path'] = "/dev/loop0" - para['enable_simulation'] = False - para['enable_blktrace'] = True + para = experiment.get_shared_nolist_para_dict("test_exp_TraceOnly2", 16 * MB) + para["device_path"] = "/dev/loop0" + para["enable_simulation"] = False + para["enable_blktrace"] = True - Parameters = collections.namedtuple("Parameters", ','.join(para.keys())) - obj = LocalExperiment( Parameters(**para) ) + Parameters = collections.namedtuple("Parameters", ",".join(list(para.keys()))) + obj = LocalExperiment(Parameters(**para)) obj.main() @@ -33,13 +33,15 @@ class Test_TraceAndSimulateDFTLDES(unittest.TestCase): def test_run(self): class LocalExperiment(experiment.Experiment): def setup_workload(self): - self.conf['workload_class'] = "SimpleRandReadWrite" - - para = experiment.get_shared_nolist_para_dict("test_exp_TraceAndSimulateDFTLDES_xjjj", 16*MB) - para['device_path'] = "/dev/loop0" - para['ftl'] = "dftldes" - Parameters = collections.namedtuple("Parameters", ','.join(para.keys())) - obj = LocalExperiment( Parameters(**para) ) + self.conf["workload_class"] = "SimpleRandReadWrite" + + para = experiment.get_shared_nolist_para_dict( + "test_exp_TraceAndSimulateDFTLDES_xjjj", 16 * MB + ) + para["device_path"] = "/dev/loop0" + para["ftl"] = "dftldes" + Parameters = collections.namedtuple("Parameters", ",".join(list(para.keys()))) + obj = LocalExperiment(Parameters(**para)) obj.main() @@ -47,35 +49,39 @@ class Test_TraceAndSimulateNKFTL(unittest.TestCase): def test_run(self): class LocalExperiment(experiment.Experiment): def setup_workload(self): - self.conf['workload_class'] = "SimpleRandReadWrite" - - para = experiment.get_shared_nolist_para_dict("test_exp_TraceAndSimulateNKFTL_xjjj", 16*MB) - para['device_path'] = "/dev/loop0" - para['ftl'] = "nkftl2" - Parameters = collections.namedtuple("Parameters", ','.join(para.keys())) - obj = LocalExperiment( Parameters(**para) ) + self.conf["workload_class"] = "SimpleRandReadWrite" + + para = experiment.get_shared_nolist_para_dict( + "test_exp_TraceAndSimulateNKFTL_xjjj", 16 * MB + ) + para["device_path"] = "/dev/loop0" + para["ftl"] = "nkftl2" + Parameters = collections.namedtuple("Parameters", ",".join(list(para.keys()))) + obj = LocalExperiment(Parameters(**para)) obj.main() class Test_SimulateForSyntheticWorkload(unittest.TestCase): - class LocalExperiment(experiment.Experiment): - def setup_workload(self): - self.conf['workload_src'] = config.LBAGENERATOR - self.conf['lba_workload_class'] = "AccessesWithDist" - self.conf['AccessesWithDist'] = { - 'lba_access_dist': 'uniform', - 'traffic_size': 8*MB, - 'chunk_size': 64*KB, - 'space_size': 8*MB, - 'skew_factor': None, - 'zipf_alpha': None, - } - - para = experiment.get_shared_nolist_para_dict("test_exp_SimulateForSyntheticWorkload", 16*MB) - para['ftl'] = "nkftl2" - Parameters = collections.namedtuple("Parameters", ','.join(para.keys())) - obj = LocalExperiment( Parameters(**para) ) - obj.main() + class LocalExperiment(experiment.Experiment): + def setup_workload(self): + self.conf["workload_src"] = config.LBAGENERATOR + self.conf["lba_workload_class"] = "AccessesWithDist" + self.conf["AccessesWithDist"] = { + "lba_access_dist": "uniform", + "traffic_size": 8 * MB, + "chunk_size": 64 * KB, + "space_size": 8 * MB, + "skew_factor": None, + "zipf_alpha": None, + } + + para = experiment.get_shared_nolist_para_dict( + "test_exp_SimulateForSyntheticWorkload", 16 * MB + ) + para["ftl"] = "nkftl2" + Parameters = collections.namedtuple("Parameters", ",".join(list(para.keys()))) + obj = LocalExperiment(Parameters(**para)) + obj.main() class TestUsingExistingTraceToSimulate(unittest.TestCase): @@ -84,51 +90,61 @@ class LocalExperiment(experiment.Experiment): def setup_workload(self): self.conf["workload_src"] = config.LBAGENERATOR self.conf["lba_workload_class"] = "BlktraceEvents" - self.conf['lba_workload_configs']['mkfs_event_path'] = \ - self.para.mkfs_path - self.conf['lba_workload_configs']['ftlsim_event_path'] = \ - self.para.ftlsim_path - - para = experiment.get_shared_nolist_para_dict("test_exp_TestUsingExistingTraceToSimulate_jj23hx", 1*GB) - para.update({ - 'ftl': "dftldes", - "mkfs_path": "./tests/testdata/sqlitewal-update/subexp-7928737328932659543-ext4-10-07-23-50-10--2726320246496492803/blkparse-events-for-ftlsim-mkfs.txt", - "ftlsim_path": "./tests/testdata/sqlitewal-update/subexp-7928737328932659543-ext4-10-07-23-50-10--2726320246496492803/blkparse-events-for-ftlsim.txt", - }) - - Parameters = collections.namedtuple("Parameters", ','.join(para.keys())) - obj = LocalExperiment( Parameters(**para) ) + self.conf["lba_workload_configs"]["mkfs_event_path"] = ( + self.para.mkfs_path + ) + self.conf["lba_workload_configs"]["ftlsim_event_path"] = ( + self.para.ftlsim_path + ) + + para = experiment.get_shared_nolist_para_dict( + "test_exp_TestUsingExistingTraceToSimulate_jj23hx", 1 * GB + ) + para.update( + { + "ftl": "dftldes", + "mkfs_path": "./tests/testdata/sqlitewal-update/subexp-7928737328932659543-ext4-10-07-23-50-10--2726320246496492803/blkparse-events-for-ftlsim-mkfs.txt", + "ftlsim_path": "./tests/testdata/sqlitewal-update/subexp-7928737328932659543-ext4-10-07-23-50-10--2726320246496492803/blkparse-events-for-ftlsim.txt", + } + ) + + Parameters = collections.namedtuple("Parameters", ",".join(list(para.keys()))) + obj = LocalExperiment(Parameters(**para)) obj.main() - - class TestUsingExistingTraceToStudyRequestScale(unittest.TestCase): def test_run(self): class LocalExperiment(experiment.Experiment): def setup_workload(self): self.conf["workload_src"] = config.LBAGENERATOR self.conf["lba_workload_class"] = "BlktraceEvents" - self.conf['lba_workload_configs']['mkfs_event_path'] = \ - self.para.mkfs_path - self.conf['lba_workload_configs']['ftlsim_event_path'] = \ - self.para.ftlsim_path - - para = experiment.get_shared_nolist_para_dict("test_exp_TestUsingExistingTraceToStudyRequestScale_jj23hx", 1*GB) - para.update({ - 'ftl': "ftlcounter", - "mkfs_path": "./tests/testdata/sqlitewal-update/subexp-7928737328932659543-ext4-10-07-23-50-10--2726320246496492803/blkparse-events-for-ftlsim-mkfs.txt", - "ftlsim_path": "./tests/testdata/sqlitewal-update/subexp-7928737328932659543-ext4-10-07-23-50-10--2726320246496492803/blkparse-events-for-ftlsim.txt", - 'ftl' : 'ftlcounter', - 'enable_simulation': True, - 'dump_ext4_after_workload': True, - 'only_get_traffic': False, - 'trace_issue_and_complete': True, - 'do_dump_lpn_sem': False, - }) - - Parameters = collections.namedtuple("Parameters", ','.join(para.keys())) - obj = LocalExperiment( Parameters(**para) ) + self.conf["lba_workload_configs"]["mkfs_event_path"] = ( + self.para.mkfs_path + ) + self.conf["lba_workload_configs"]["ftlsim_event_path"] = ( + self.para.ftlsim_path + ) + + para = experiment.get_shared_nolist_para_dict( + "test_exp_TestUsingExistingTraceToStudyRequestScale_jj23hx", 1 * GB + ) + para.update( + { + "ftl": "ftlcounter", + "mkfs_path": "./tests/testdata/sqlitewal-update/subexp-7928737328932659543-ext4-10-07-23-50-10--2726320246496492803/blkparse-events-for-ftlsim-mkfs.txt", + "ftlsim_path": "./tests/testdata/sqlitewal-update/subexp-7928737328932659543-ext4-10-07-23-50-10--2726320246496492803/blkparse-events-for-ftlsim.txt", + "ftl": "ftlcounter", + "enable_simulation": True, + "dump_ext4_after_workload": True, + "only_get_traffic": False, + "trace_issue_and_complete": True, + "do_dump_lpn_sem": False, + } + ) + + Parameters = collections.namedtuple("Parameters", ",".join(list(para.keys()))) + obj = LocalExperiment(Parameters(**para)) obj.main() @@ -136,26 +152,30 @@ def setup_workload(self): # Experiments setting similar to SSD Contract paper ################################################################### + class TestRunningWorkloadAndOutputRequestScale(unittest.TestCase): def test_run(self): class LocalExperiment(experiment.Experiment): def setup_workload(self): - self.conf['workload_class'] = "SimpleRandReadWrite" + self.conf["workload_class"] = "SimpleRandReadWrite" - para = experiment.get_shared_nolist_para_dict("test_exp_TestRequestScale_jjj3nx", 16*MB) - para['device_path'] = "/dev/loop0" + para = experiment.get_shared_nolist_para_dict( + "test_exp_TestRequestScale_jjj3nx", 16 * MB + ) + para["device_path"] = "/dev/loop0" para.update( { - 'device_path': "/dev/loop0", - 'ftl' : 'ftlcounter', - 'enable_simulation': True, - 'dump_ext4_after_workload': True, - 'only_get_traffic': False, - 'trace_issue_and_complete': True, - }) - - Parameters = collections.namedtuple("Parameters", ','.join(para.keys())) - obj = LocalExperiment( Parameters(**para) ) + "device_path": "/dev/loop0", + "ftl": "ftlcounter", + "enable_simulation": True, + "dump_ext4_after_workload": True, + "only_get_traffic": False, + "trace_issue_and_complete": True, + } + ) + + Parameters = collections.namedtuple("Parameters", ",".join(list(para.keys()))) + obj = LocalExperiment(Parameters(**para)) obj.main() @@ -168,9 +188,12 @@ def test(self): # copy the data to shcmd("cp -r ./tests/testdata/sqlitewal-update /tmp/results/") - for para in rule_parameter.ParaDict("testexpname", ['sqlitewal-update'], "locality"): + for para in rule_parameter.ParaDict( + "testexpname", ["sqlitewal-update"], "locality" + ): experiment.execute_simulation(para) + class TestAlignment(unittest.TestCase): def test(self): old_dir = "/tmp/results/sqlitewal-update" @@ -180,7 +203,9 @@ def test(self): # copy the data to shcmd("cp -r ./tests/testdata/sqlitewal-update /tmp/results/") - for para in rule_parameter.ParaDict("testexpname", ['sqlitewal-update'], "alignment"): + for para in rule_parameter.ParaDict( + "testexpname", ["sqlitewal-update"], "alignment" + ): experiment.execute_simulation(para) @@ -193,7 +218,9 @@ def test(self): # copy the data to shcmd("cp -r ./tests/testdata/sqlitewal-update /tmp/results/") - for para in rule_parameter.ParaDict("testexpname", ['sqlitewal-update'], "grouping"): + for para in rule_parameter.ParaDict( + "testexpname", ["sqlitewal-update"], "grouping" + ): experiment.execute_simulation(para) @@ -201,45 +228,48 @@ class TestUniformDataLifetime(unittest.TestCase): def test_run(self): class LocalExperiment(experiment.Experiment): def setup_workload(self): - self.conf['workload_class'] = "SimpleRandReadWrite" + self.conf["workload_class"] = "SimpleRandReadWrite" - para = experiment.get_shared_nolist_para_dict("test_exp_TestUniformDataLifetime", 16*MB) + para = experiment.get_shared_nolist_para_dict( + "test_exp_TestUniformDataLifetime", 16 * MB + ) para.update( { - 'ftl' : 'ftlcounter', - 'device_path' : '/dev/loop0', - 'enable_simulation': True, - 'dump_ext4_after_workload': True, - 'only_get_traffic': False, - 'trace_issue_and_complete': False, - 'gen_ncq_depth_table': False, - 'do_dump_lpn_sem': False, - 'rm_blkparse_events': True, - 'sort_block_trace': False, - }) - - Parameters = collections.namedtuple("Parameters", ','.join(para.keys())) - obj = LocalExperiment( Parameters(**para) ) + "ftl": "ftlcounter", + "device_path": "/dev/loop0", + "enable_simulation": True, + "dump_ext4_after_workload": True, + "only_get_traffic": False, + "trace_issue_and_complete": False, + "gen_ncq_depth_table": False, + "do_dump_lpn_sem": False, + "rm_blkparse_events": True, + "sort_block_trace": False, + } + ) + + Parameters = collections.namedtuple("Parameters", ",".join(list(para.keys()))) + obj = LocalExperiment(Parameters(**para)) obj.main() # class Test_TraceAndSimulateLinuxDD(unittest.TestCase): - # def test_run(self): - # class LocalExperiment(experiment.Experiment): - # def setup_workload(self): - # self.conf['workload_class'] = "LinuxDD" - - # para = experiment.get_shared_nolist_para_dict("test_exp_LinuxDD", 16*MB) - # para['device_path'] = "/dev/loop0" - # para['filesystem'] = "ext4" - # para['ftl'] = "dftldes" - # Parameters = collections.namedtuple("Parameters", ','.join(para.keys())) - # obj = LocalExperiment( Parameters(**para) ) - # obj.main() - - -if __name__ == '__main__': - #unittest.main() +# def test_run(self): +# class LocalExperiment(experiment.Experiment): +# def setup_workload(self): +# self.conf['workload_class'] = "LinuxDD" + +# para = experiment.get_shared_nolist_para_dict("test_exp_LinuxDD", 16*MB) +# para['device_path'] = "/dev/loop0" +# para['filesystem'] = "ext4" +# para['ftl'] = "dftldes" +# Parameters = collections.namedtuple("Parameters", ','.join(para.keys())) +# obj = LocalExperiment( Parameters(**para) ) +# obj.main() + + +if __name__ == "__main__": + # unittest.main() old_dir = "/tmp/results/sqlitewal-update" if os.path.exists(old_dir): shutil.rmtree(old_dir) @@ -247,6 +277,7 @@ def setup_workload(self): # copy the data to shcmd("cp -r ./tests/testdata/sqlitewal-update /tmp/results/") - for para in rule_parameter.ParaDict("testexpname", ['sqlitewal-update'], "alignment"): + for para in rule_parameter.ParaDict( + "testexpname", ["sqlitewal-update"], "alignment" + ): experiment.execute_simulation(para) - diff --git a/wiscsee/tests/test_devblockpool.py b/wiscsee/tests/test_devblockpool.py index 7416f5d..48f0bcb 100644 --- a/wiscsee/tests/test_devblockpool.py +++ b/wiscsee/tests/test_devblockpool.py @@ -1,3 +1,4 @@ +from past.utils import old_div import unittest from wiscsim.devblockpool import * from collections import Counter @@ -5,42 +6,46 @@ from utilities import utils -TDATA = 'TDATA' -TTRANS = 'TTRANS' +TDATA = "TDATA" +TTRANS = "TTRANS" class TestMultiChannelBlockPool(unittest.TestCase): def test_init(self): pool = MultiChannelBlockPool( - n_channels=8, - n_blocks_per_channel=64, - n_pages_per_block=32, - tags=[TDATA, TTRANS]) + n_channels=8, + n_blocks_per_channel=64, + n_pages_per_block=32, + tags=[TDATA, TTRANS], + ) def test_count_blocks(self): pool = MultiChannelBlockPool( - n_channels=8, - n_blocks_per_channel=64, - n_pages_per_block=32, - tags=[TDATA, TTRANS]) + n_channels=8, + n_blocks_per_channel=64, + n_pages_per_block=32, + tags=[TDATA, TTRANS], + ) - self.assertEqual(pool.count_blocks(tag=TFREE), 8*64) + self.assertEqual(pool.count_blocks(tag=TFREE), 8 * 64) def test_count_channel_blocks(self): pool = MultiChannelBlockPool( - n_channels=8, - n_blocks_per_channel=64, - n_pages_per_block=32, - tags=[TDATA, TTRANS]) + n_channels=8, + n_blocks_per_channel=64, + n_pages_per_block=32, + tags=[TDATA, TTRANS], + ) self.assertEqual(pool.count_blocks(tag=TFREE, channels=[0]), 64) def test_change_tag(self): pool = MultiChannelBlockPool( - n_channels=8, - n_blocks_per_channel=64, - n_pages_per_block=32, - tags=[TDATA, TTRANS]) + n_channels=8, + n_blocks_per_channel=64, + n_pages_per_block=32, + tags=[TDATA, TTRANS], + ) pool.change_tag(0, src=TFREE, dst=TDATA) blocks = pool.get_blocks_of_tag(tag=TDATA) @@ -48,10 +53,11 @@ def test_change_tag(self): def test_change_tag_2(self): pool = MultiChannelBlockPool( - n_channels=8, - n_blocks_per_channel=64, - n_pages_per_block=32, - tags=[TDATA, TTRANS]) + n_channels=8, + n_blocks_per_channel=64, + n_pages_per_block=32, + tags=[TDATA, TTRANS], + ) blocks = [] for i in range(15): @@ -65,13 +71,14 @@ def test_change_tag_2(self): def test_pick_and_move(self): pool = MultiChannelBlockPool( - n_channels=8, - n_blocks_per_channel=64, - n_pages_per_block=32, - tags=[TDATA, TTRANS]) + n_channels=8, + n_blocks_per_channel=64, + n_pages_per_block=32, + tags=[TDATA, TTRANS], + ) blocknum = pool.pick_and_move(src=TFREE, dst=TDATA) - self.assertEqual(pool.count_blocks(tag=TFREE), 8*64-1) + self.assertEqual(pool.count_blocks(tag=TFREE), 8 * 64 - 1) self.assertEqual(pool.count_blocks(tag=TDATA), 1) datablocks = pool.get_blocks_of_tag(TDATA) @@ -79,95 +86,103 @@ def test_pick_and_move(self): def test_pick(self): pool = MultiChannelBlockPool( - n_channels=8, - n_blocks_per_channel=64, - n_pages_per_block=32, - tags=[TDATA, TTRANS]) + n_channels=8, + n_blocks_per_channel=64, + n_pages_per_block=32, + tags=[TDATA, TTRANS], + ) blocknum = pool.pick(tag=TFREE) self.assertIn(blocknum, pool.get_blocks_of_tag(tag=TFREE)) def test_pick_failure(self): pool = MultiChannelBlockPool( - n_channels=8, - n_blocks_per_channel=64, - n_pages_per_block=32, - tags=[TDATA, TTRANS]) + n_channels=8, + n_blocks_per_channel=64, + n_pages_per_block=32, + tags=[TDATA, TTRANS], + ) blocknum = pool.pick(tag=TDATA) self.assertEqual(blocknum, None) def test_pick_from_channel(self): pool = MultiChannelBlockPool( - n_channels=8, - n_blocks_per_channel=64, - n_pages_per_block=32, - tags=[TDATA, TTRANS]) + n_channels=8, + n_blocks_per_channel=64, + n_pages_per_block=32, + tags=[TDATA, TTRANS], + ) blocknum = pool.pick(tag=TFREE, channel_id=2) - self.assertIn(blocknum, pool.get_blocks_of_tag(tag=TFREE, - channel_id=2)) + self.assertIn(blocknum, pool.get_blocks_of_tag(tag=TFREE, channel_id=2)) def test_pick_from_channel_failure(self): pool = MultiChannelBlockPool( - n_channels=8, - n_blocks_per_channel=64, - n_pages_per_block=32, - tags=[TDATA, TTRANS]) + n_channels=8, + n_blocks_per_channel=64, + n_pages_per_block=32, + tags=[TDATA, TTRANS], + ) blocknum = pool.pick(tag=TDATA, channel_id=2) self.assertEqual(blocknum, None) def test_cur_blocks(self): pool = MultiChannelBlockPool( - n_channels=8, - n_blocks_per_channel=64, - n_pages_per_block=32, - tags=[TDATA, TTRANS]) + n_channels=8, + n_blocks_per_channel=64, + n_pages_per_block=32, + tags=[TDATA, TTRANS], + ) def test_next_ppns_in_channel(self): pool = MultiChannelBlockPool( - n_channels=8, - n_blocks_per_channel=64, - n_pages_per_block=32, - tags=[TDATA, TTRANS]) + n_channels=8, + n_blocks_per_channel=64, + n_pages_per_block=32, + tags=[TDATA, TTRANS], + ) - ppns = pool._next_ppns_in_channel( - channel_id=1, n=1, tag=TDATA, block_index=0) + ppns = pool._next_ppns_in_channel(channel_id=1, n=1, tag=TDATA, block_index=0) self.assertEqual(len(ppns), 1) other_ppns = pool._next_ppns_in_channel( - channel_id=1, n=100000, tag=TDATA, block_index=0) - self.assertEqual(len(other_ppns), 64*32-1) + channel_id=1, n=100000, tag=TDATA, block_index=0 + ) + self.assertEqual(len(other_ppns), 64 * 32 - 1) more_ppns = pool._next_ppns_in_channel( - channel_id=1, n=1, tag=TDATA, block_index=0) + channel_id=1, n=1, tag=TDATA, block_index=0 + ) self.assertEqual(len(more_ppns), 0) def test_next_ppns_in_channel_2_tags(self): pool = MultiChannelBlockPool( - n_channels=8, - n_blocks_per_channel=64, - n_pages_per_block=32, - tags=[TDATA, TTRANS]) + n_channels=8, + n_blocks_per_channel=64, + n_pages_per_block=32, + tags=[TDATA, TTRANS], + ) - ppns = pool._next_ppns_in_channel( - channel_id=0, n=1, tag=TDATA, block_index=0) + ppns = pool._next_ppns_in_channel(channel_id=0, n=1, tag=TDATA, block_index=0) self.assertEqual(len(ppns), 1) other_ppns = pool._next_ppns_in_channel( - channel_id=0, n=10, tag=TTRANS, block_index=0) + channel_id=0, n=10, tag=TTRANS, block_index=0 + ) self.assertEqual(len(other_ppns), 10) # not in the same block - self.assertNotEqual(ppns[0]/32, other_ppns[0]/32) + self.assertNotEqual(old_div(ppns[0], 32), old_div(other_ppns[0], 32)) def test_next_ppns(self): pool = MultiChannelBlockPool( - n_channels=8, - n_blocks_per_channel=64, - n_pages_per_block=32, - tags=[TDATA, TTRANS]) + n_channels=8, + n_blocks_per_channel=64, + n_pages_per_block=32, + tags=[TDATA, TTRANS], + ) # all from one channel ppns = pool.next_ppns(n=2, tag=TDATA, block_index=0, stripe_size=2) @@ -178,14 +193,15 @@ def test_next_ppns(self): cur_block_num = cur_blocks[0] for ppn in ppns: - self.assertEqual(ppn / 32, cur_block_num) + self.assertEqual(old_div(ppn, 32), cur_block_num) def test_next_ppns_across_channel(self): pool = MultiChannelBlockPool( - n_channels=8, - n_blocks_per_channel=64, - n_pages_per_block=32, - tags=[TDATA, TTRANS]) + n_channels=8, + n_blocks_per_channel=64, + n_pages_per_block=32, + tags=[TDATA, TTRANS], + ) # should use 3 channels ppns = pool.next_ppns(n=6, tag=TDATA, block_index=0, stripe_size=2) @@ -200,18 +216,19 @@ def test_next_ppns_across_channel(self): # ppns are from the current blocks for ppn in ppns: - self.assertIn(ppn / 32, cur_blocks) + self.assertIn(old_div(ppn, 32), cur_blocks) def test_remove_full_cur_blocks(self): pool = MultiChannelBlockPool( - n_channels=8, - n_blocks_per_channel=64, - n_pages_per_block=32, - tags=[TDATA, TTRANS]) + n_channels=8, + n_blocks_per_channel=64, + n_pages_per_block=32, + tags=[TDATA, TTRANS], + ) # should use 1 block in each channel - ppns = pool.next_ppns(n=8*32, tag=TDATA, block_index=0, stripe_size=1) - self.assertEqual(len(ppns), 8*32) + ppns = pool.next_ppns(n=8 * 32, tag=TDATA, block_index=0, stripe_size=1) + self.assertEqual(len(ppns), 8 * 32) cur_blocks = pool.current_blocks() channels = set() @@ -222,17 +239,18 @@ def test_remove_full_cur_blocks(self): # ppns are from the current blocks for ppn in ppns: - self.assertIn(ppn / 32, cur_blocks) + self.assertIn(old_div(ppn, 32), cur_blocks) pool.remove_full_cur_blocks() self.assertEqual(len(pool.current_blocks()), 0) def test_next_ppns_wrap_around(self): pool = MultiChannelBlockPool( - n_channels=8, - n_blocks_per_channel=64, - n_pages_per_block=32, - tags=[TDATA, TTRANS]) + n_channels=8, + n_blocks_per_channel=64, + n_pages_per_block=32, + tags=[TDATA, TTRANS], + ) ppns = pool.next_ppns(n=10, tag=TDATA, block_index=0, stripe_size=1) self.assertEqual(len(ppns), 10) @@ -246,24 +264,26 @@ def test_next_ppns_wrap_around(self): self.assertEqual(len(channels), 8) for ppn in ppns: - self.assertIn(ppn / 32, cur_blocks) + self.assertIn(old_div(ppn, 32), cur_blocks) def test_next_ppns_wrap_around(self): pool = MultiChannelBlockPool( - n_channels=8, - n_blocks_per_channel=64, - n_pages_per_block=32, - tags=[TDATA, TTRANS]) + n_channels=8, + n_blocks_per_channel=64, + n_pages_per_block=32, + tags=[TDATA, TTRANS], + ) ppns = pool.next_ppns(n=10, tag=TDATA, block_index=0, stripe_size=1) self.assertEqual(len(set(ppns)), 10) def test_block_conversion(self): pool = MultiChannelBlockPool( - n_channels=8, - n_blocks_per_channel=64, - n_pages_per_block=32, - tags=[TDATA, TTRANS]) + n_channels=8, + n_blocks_per_channel=64, + n_pages_per_block=32, + tags=[TDATA, TTRANS], + ) self.assertEqual(pool._channel_to_global(0, 0), 0) channel_id, block_off = pool._global_to_channel(0) @@ -275,26 +295,28 @@ def test_block_conversion(self): self.assertEqual(channel_id, 1) self.assertEqual(block_off, 0) - self.assertEqual(pool._channel_to_global(3, 4), 3*64+4) - channel_id, block_off = pool._global_to_channel(3*64+4) + self.assertEqual(pool._channel_to_global(3, 4), 3 * 64 + 4) + channel_id, block_off = pool._global_to_channel(3 * 64 + 4) self.assertEqual(channel_id, 3) self.assertEqual(block_off, 4) def test_ppn_conversion(self): pool = MultiChannelBlockPool( - n_channels=8, - n_blocks_per_channel=64, - n_pages_per_block=32, - tags=[TDATA, TTRANS]) + n_channels=8, + n_blocks_per_channel=64, + n_pages_per_block=32, + tags=[TDATA, TTRANS], + ) ppn = pool._ppn_channel_to_global(1, 2) - self.assertEqual(ppn, 64*32 + 2) + self.assertEqual(ppn, 64 * 32 + 2) def test_ppn_conversion2(self): pool = MultiChannelBlockPool( - n_channels=8, - n_blocks_per_channel=64, - n_pages_per_block=32, - tags=[TDATA, TTRANS]) + n_channels=8, + n_blocks_per_channel=64, + n_pages_per_block=32, + tags=[TDATA, TTRANS], + ) # should use 3 channels ppns = pool.next_ppns(n=6, tag=TDATA, block_index=0, stripe_size=2) @@ -303,24 +325,25 @@ def test_ppn_conversion2(self): # ppns should from 3 channels channel_set = set() for ppn in ppns: - channel = ppn / (64*32) + channel = old_div(ppn, (64 * 32)) channel_set.add(channel) self.assertEqual(len(channel_set), 3) def test_block_conversion2(self): pool = MultiChannelBlockPool( - n_channels=8, - n_blocks_per_channel=64, - n_pages_per_block=32, - tags=[TDATA, TTRANS]) + n_channels=8, + n_blocks_per_channel=64, + n_pages_per_block=32, + tags=[TDATA, TTRANS], + ) pool._next_channel = 0 block0 = pool.pick_and_move(src=TFREE, dst=TDATA) block1 = pool.pick_and_move(src=TFREE, dst=TDATA) - channel0 = block0 / pool.n_blocks_per_channel - channel1 = block1 / pool.n_blocks_per_channel + channel0 = old_div(block0, pool.n_blocks_per_channel) + channel1 = old_div(block1, pool.n_blocks_per_channel) self.assertEqual(channel0, 0) self.assertEqual(channel1, 1) @@ -329,22 +352,23 @@ def test_block_conversion2(self): class TestGettingDistribution(unittest.TestCase): def test(self): pool = MultiChannelBlockPool( - n_channels=8, - n_blocks_per_channel=64, - n_pages_per_block=32, - tags=[TDATA, TTRANS]) + n_channels=8, + n_blocks_per_channel=64, + n_pages_per_block=32, + tags=[TDATA, TTRANS], + ) dist = pool.get_erasure_count_dist() - self.assertEqual(dist[0], 64*8) + self.assertEqual(dist[0], 64 * 8) def test_erase_some(self): pool = MultiChannelBlockPool( - n_channels=8, - n_blocks_per_channel=64, - n_pages_per_block=32, - tags=[TDATA, TTRANS]) - + n_channels=8, + n_blocks_per_channel=64, + n_pages_per_block=32, + tags=[TDATA, TTRANS], + ) used_blocks = [] for i in range(5): @@ -355,52 +379,50 @@ def test_erase_some(self): pool.change_tag(block, TDATA, TFREE) dist = pool.get_erasure_count_dist() - self.assertEqual(dist[0], 64*8-5) + self.assertEqual(dist[0], 64 * 8 - 5) self.assertEqual(dist[1], 5) class TestWearLeveling(unittest.TestCase): def test_calulator(self): - c = Counter({1:3, 10: 3, 5: 3}) - self.assertListEqual( - list(utils.top_or_bottom_total(c, 3, 'top')), - [30, 3]) - self.assertListEqual( - list(utils.top_or_bottom_total(c, 3, 'bottom')), - [3, 3]) + c = Counter({1: 3, 10: 3, 5: 3}) + self.assertListEqual(list(utils.top_or_bottom_total(c, 3, "top")), [30, 3]) + self.assertListEqual(list(utils.top_or_bottom_total(c, 3, "bottom")), [3, 3]) def test_calulator_with_0(self): - c = Counter({0:3, 1:3, 5:3, 10:3}) - self.assertListEqual( - list(utils.top_or_bottom_total(c, 3, 'bottom')), - [0, 3]) + c = Counter({0: 3, 1: 3, 5: 3, 10: 3}) + self.assertListEqual(list(utils.top_or_bottom_total(c, 3, "bottom")), [0, 3]) def test_bottom_10_average_blockpool(self): pool = MultiChannelBlockPool( - n_channels=8, - n_blocks_per_channel=64, - n_pages_per_block=32, - tags=[TDATA, TTRANS]) + n_channels=8, + n_blocks_per_channel=64, + n_pages_per_block=32, + tags=[TDATA, TTRANS], + ) pool._channel_pool[1]._erasure_cnt[0] = 8 pool._channel_pool[1]._erasure_cnt[1] = 7 - erase_cnt, block_cnt = pool.get_top_or_bottom_erasure_total(choice='top', - need_nblocks=2) + erase_cnt, block_cnt = pool.get_top_or_bottom_erasure_total( + choice="top", need_nblocks=2 + ) self.assertEqual(erase_cnt, 15) self.assertEqual(block_cnt, 2) - erase_cnt, block_cnt = pool.get_top_or_bottom_erasure_total(choice='bottom', - need_nblocks=2) + erase_cnt, block_cnt = pool.get_top_or_bottom_erasure_total( + choice="bottom", need_nblocks=2 + ) self.assertEqual(erase_cnt, 0) self.assertEqual(block_cnt, 2) def test_wearleveling_trigger(self): pool = MultiChannelBlockPool( - n_channels=1, - n_blocks_per_channel=100, - n_pages_per_block=32, - tags=[TDATA, TTRANS]) + n_channels=1, + n_blocks_per_channel=100, + n_pages_per_block=32, + tags=[TDATA, TTRANS], + ) pool.leveling_factor = 2 pool.leveling_diff = 10 @@ -412,10 +434,11 @@ def test_wearleveling_trigger(self): def test_wearleveling_trigger2(self): pool = MultiChannelBlockPool( - n_channels=1, - n_blocks_per_channel=100, - n_pages_per_block=32, - tags=[TDATA, TTRANS]) + n_channels=1, + n_blocks_per_channel=100, + n_pages_per_block=32, + tags=[TDATA, TTRANS], + ) pool.leveling_factor = 2 pool.leveling_diff = 10 @@ -430,10 +453,11 @@ def test_wearleveling_trigger2(self): def test_wearleveling_trigger3(self): pool = MultiChannelBlockPool( - n_channels=1, - n_blocks_per_channel=100, - n_pages_per_block=32, - tags=[TDATA, TTRANS]) + n_channels=1, + n_blocks_per_channel=100, + n_pages_per_block=32, + tags=[TDATA, TTRANS], + ) pool.leveling_factor = 2 pool.leveling_diff = 10 @@ -448,18 +472,20 @@ def test_wearleveling_trigger3(self): def test_least_or_most_erased_blocks(self): pool = MultiChannelBlockPool( - n_channels=2, - n_blocks_per_channel=10, - n_pages_per_block=32, - tags=[TDATA, TTRANS]) + n_channels=2, + n_blocks_per_channel=10, + n_pages_per_block=32, + tags=[TDATA, TTRANS], + ) # block num 0 ... 9 in channel 0 # block num 10 ... 19 in channel 1 for i in range(0, 3): pool._channel_pool[1]._erasure_cnt[i] = 18 - blocks = pool.get_least_or_most_erased_blocks(tag=TFREE, - choice=MOST_ERASED, nblocks=3) + blocks = pool.get_least_or_most_erased_blocks( + tag=TFREE, choice=MOST_ERASED, nblocks=3 + ) self.assertEqual(sorted(blocks), [10, 11, 12]) def test_get_bottom_10_used_blocks(self): @@ -469,12 +495,9 @@ def test_write_to_most_used_blocks(self): pass - def main(): unittest.main() -if __name__ == '__main__': - main() - - +if __name__ == "__main__": + main() diff --git a/wiscsee/tests/test_dftldes.py b/wiscsee/tests/test_dftldes.py index a43f19a..262be2a 100644 --- a/wiscsee/tests/test_dftldes.py +++ b/wiscsee/tests/test_dftldes.py @@ -1,3 +1,4 @@ +from past.utils import old_div import unittest import random import simpy @@ -7,14 +8,14 @@ from utilities import utils import wiscsim from wiscsim.ftlsim_commons import Extent -from wiscsim.dftldes import LpnTable, LpnTableMvpn, UNINITIATED, \ - split_ext_by_segment +from wiscsim.dftldes import LpnTable, LpnTableMvpn, UNINITIATED, split_ext_by_segment from config import WLRUNNER, LBAGENERATOR, LBAMULTIPROC from commons import * from utilities.utils import get_expname import collections from workflow import run_workflow + class FtlTest(wiscsim.dftldes.Ftl): def get_mappings(self): return self._mappings @@ -28,23 +29,23 @@ def get_cleaner(self): def create_config(): conf = wiscsim.dftldes.Config() - conf['SSDFramework']['ncq_depth'] = 1 + conf["SSDFramework"]["ncq_depth"] = 1 - conf['flash_config']['n_pages_per_block'] = 64 - conf['flash_config']['n_blocks_per_plane'] = 2 - conf['flash_config']['n_planes_per_chip'] = 1 - conf['flash_config']['n_chips_per_package'] = 1 - conf['flash_config']['n_packages_per_channel'] = 1 - conf['flash_config']['n_channels_per_dev'] = 4 + conf["flash_config"]["n_pages_per_block"] = 64 + conf["flash_config"]["n_blocks_per_plane"] = 2 + conf["flash_config"]["n_planes_per_chip"] = 1 + conf["flash_config"]["n_chips_per_package"] = 1 + conf["flash_config"]["n_packages_per_channel"] = 1 + conf["flash_config"]["n_channels_per_dev"] = 4 - conf['max_victim_valid_ratio'] = 1 + conf["max_victim_valid_ratio"] = 1 - utils.set_exp_metadata(conf, save_data = False, - expname = 'test_expname', - subexpname = 'test_subexpname') + utils.set_exp_metadata( + conf, save_data=False, expname="test_expname", subexpname="test_subexpname" + ) - conf['ftl_type'] = 'dftldes' - conf['simulator_class'] = 'SimulatorDESSync' + conf["ftl_type"] = "dftldes" + conf["simulator_class"] = "SimulatorDESSync" logicsize_mb = 64 conf.n_cache_entries = conf.n_mapping_entries_per_page @@ -56,33 +57,41 @@ def create_config(): def create_recorder(conf): - rec = wiscsim.recorder.Recorder(output_target = conf['output_target'], - output_directory = conf['result_dir'], - verbose_level = conf['verbose_level'], - print_when_finished = conf['print_when_finished'] - ) + rec = wiscsim.recorder.Recorder( + output_target=conf["output_target"], + output_directory=conf["result_dir"], + verbose_level=conf["verbose_level"], + print_when_finished=conf["print_when_finished"], + ) rec.disable() return rec + def create_oob(conf): oob = wiscsim.dftldes.OutOfBandAreas(conf) return oob + def create_blockpool(conf): return wiscsim.dftldes.BlockPool(conf) + def create_flashcontrolelr(conf, env, rec): return wiscsim.controller.Controller3(env, conf, rec) + def create_simpy_env(): return simpy.Environment() + def create_translation_directory(conf, oob, block_pool): return wiscsim.dftldes.GlobalTranslationDirectory(conf, oob, block_pool) + def create_mapping_on_flash(conf): return wiscsim.dftldes.MappingOnFlash(conf) + def create_victimblocks(): conf = create_config() block_pool = create_blockpool(conf) @@ -92,6 +101,7 @@ def create_victimblocks(): return vbs + def create_wearlevelingvictimblocks(): conf = create_config() block_pool = create_blockpool(conf) @@ -101,6 +111,7 @@ def create_wearlevelingvictimblocks(): return vbs + def create_obj_set(conf): rec = create_recorder(conf) oob = create_oob(conf) @@ -112,46 +123,55 @@ def create_obj_set(conf): victimblocks = wiscsim.dftldes.VictimBlocks(conf, block_pool, oob) trans_page_locks = wiscsim.dftldes.LockPool(env) - return {'conf':conf, 'rec':rec, 'oob':oob, 'block_pool':block_pool, - 'flash_controller':flash_controller, 'env':env, 'directory':directory, - 'mapping_on_flash':gmt, 'victimblocks':victimblocks, - 'trans_page_locks':trans_page_locks - } + return { + "conf": conf, + "rec": rec, + "oob": oob, + "block_pool": block_pool, + "flash_controller": flash_controller, + "env": env, + "directory": directory, + "mapping_on_flash": gmt, + "victimblocks": victimblocks, + "trans_page_locks": trans_page_locks, + } def create_mapping_cache(objs): mapping_cache = wiscsim.dftldes.MappingCache( - confobj = objs['conf'], - block_pool = objs['block_pool'], - flashobj = objs['flash_controller'], - oobobj = objs['oob'], - recorderobj = objs['rec'], - envobj = objs['env'], - directory = objs['directory'], - mapping_on_flash = objs['mapping_on_flash'], - trans_page_locks = objs['trans_page_locks'] - ) + confobj=objs["conf"], + block_pool=objs["block_pool"], + flashobj=objs["flash_controller"], + oobobj=objs["oob"], + recorderobj=objs["rec"], + envobj=objs["env"], + directory=objs["directory"], + mapping_on_flash=objs["mapping_on_flash"], + trans_page_locks=objs["trans_page_locks"], + ) return mapping_cache + def create_datablockcleaner(objs): mappings = create_mapping_cache(objs) datablockcleaner = wiscsim.dftldes.DataBlockCleaner( - conf = objs['conf'], - flash = objs['flash_controller'], - oob = objs['oob'], - block_pool = objs['block_pool'], - mappings = mappings, - rec = objs['rec'], - env = objs['env']) + conf=objs["conf"], + flash=objs["flash_controller"], + oob=objs["oob"], + block_pool=objs["block_pool"], + mappings=mappings, + rec=objs["rec"], + env=objs["env"], + ) return datablockcleaner class TestMappingCache(unittest.TestCase): def update_m_vpn(self, objs, mapping_cache, m_vpn): - conf = objs['conf'] - env = objs['env'] - lpns = conf.m_vpn_to_lpns(m_vpn) + conf = objs["conf"] + env = objs["env"] + lpns = conf.m_vpn_to_lpns(m_vpn) for lpn in lpns: ppn = lpn * 1000 yield env.process(mapping_cache.update(lpn, ppn)) @@ -159,25 +179,26 @@ def update_m_vpn(self, objs, mapping_cache, m_vpn): self.assertEqual(ppn, ppn_in_cache) def update_proc(self, objs, mapping_cache): - conf = objs['conf'] - env = objs['env'] - time_read_page = objs['flash_controller'].channels[0].read_time - time_program_page = objs['flash_controller'].channels[0].program_time + conf = objs["conf"] + env = objs["env"] + time_read_page = objs["flash_controller"].channels[0].read_time + time_program_page = objs["flash_controller"].channels[0].program_time - yield env.process(self.update_m_vpn(objs, mapping_cache, m_vpn = 3)) + yield env.process(self.update_m_vpn(objs, mapping_cache, m_vpn=3)) # it should not take any time self.assertEqual(env.now, 0) - yield env.process(self.update_m_vpn(objs, mapping_cache, m_vpn = 4)) + yield env.process(self.update_m_vpn(objs, mapping_cache, m_vpn=4)) # it should only write one flash page (m_vpn=3) back to flash self.assertEqual(env.now, time_program_page) - lpns = conf.m_vpn_to_lpns(3) + lpns = conf.m_vpn_to_lpns(3) lpn = lpns[0] ppn = yield env.process(mapping_cache.lpn_to_ppn(lpn)) self.assertEqual(ppn, lpn * 1000) - self.assertEqual(env.now, time_program_page + time_program_page + \ - time_read_page) + self.assertEqual( + env.now, time_program_page + time_program_page + time_read_page + ) def test_update(self): conf = create_config() @@ -186,7 +207,7 @@ def test_update(self): mapping_cache = create_mapping_cache(objs) - env = objs['env'] + env = objs["env"] env.process(self.update_proc(objs, mapping_cache)) env.run() @@ -194,7 +215,7 @@ def test_update(self): class TestMappingCacheParallel(unittest.TestCase): def update_random(self, conf, env, mapping_cache): n = conf.total_num_pages() - print 'n tp pages', conf.total_translation_pages() + print(("n tp pages", conf.total_translation_pages())) lpns = list(range(n)) random.shuffle(lpns) lpns = lpns[:128] @@ -212,7 +233,7 @@ def update_random(self, conf, env, mapping_cache): ppn = yield env.process(mapping_cache.lpn_to_ppn(lpn=lpn, tag=None)) self.assertEqual(ppn, mappings[lpn]) - print "i am finished" + print("i am finished") def test_update(self): conf = create_config() @@ -221,7 +242,7 @@ def test_update(self): mapping_cache = create_mapping_cache(objs) - env = objs['env'] + env = objs["env"] env.process(self.update_random(conf, env, mapping_cache)) env.run() @@ -240,19 +261,19 @@ def load(self, conf, env, mapping_cache): recorder.enable() # each tp has 512 entries - self.assertEqual(lpntable.n_free_rows(), 4*512) + self.assertEqual(lpntable.n_free_rows(), 4 * 512) - for offset in [0*MB, 1*MB, 2*MB, 3*MB]: - lpn = offset / (2*KB) + for offset in [0 * MB, 1 * MB, 2 * MB, 3 * MB]: + lpn = old_div(offset, (2 * KB)) ppn = yield env.process(mapping_cache.lpn_to_ppn(lpn=lpn, tag=None)) self.assertEqual(ppn, UNINITIATED) self.assertEqual(lpntable.n_free_rows(), 0) - self.assertEqual(recorder.get_count_me('translation', - 'read-trans-for-load'), 4) - self.assertEqual(recorder.get_count_me('translation', - 'delete-lpn-in-table-for-load'), 0) + self.assertEqual(recorder.get_count_me("translation", "read-trans-for-load"), 4) + self.assertEqual( + recorder.get_count_me("translation", "delete-lpn-in-table-for-load"), 0 + ) def test_update(self): conf = create_config() @@ -261,7 +282,7 @@ def test_update(self): mapping_cache = create_mapping_cache(objs) - env = objs['env'] + env = objs["env"] env.process(self.load(conf, env, mapping_cache)) env.run() @@ -273,31 +294,35 @@ def load(self, conf, env, mapping_cache): recorder = mapping_cache.recorder recorder.enable() - for offset in [0*MB, 1*MB, 2*MB, 3*MB]: - lpn = offset / (2*KB) + for offset in [0 * MB, 1 * MB, 2 * MB, 3 * MB]: + lpn = old_div(offset, (2 * KB)) ppn = yield env.process(mapping_cache.lpn_to_ppn(lpn=lpn, tag=None)) self.assertEqual(ppn, UNINITIATED) # modify mapping - for offset in [0*MB, 1*MB, 2*MB, 3*MB]: - lpn = offset / (2*KB) - ppn = yield env.process(mapping_cache.update(lpn=lpn, ppn=lpn+1, tag=None)) + for offset in [0 * MB, 1 * MB, 2 * MB, 3 * MB]: + lpn = old_div(offset, (2 * KB)) + ppn = yield env.process( + mapping_cache.update(lpn=lpn, ppn=lpn + 1, tag=None) + ) - self.assertEqual( - recorder.get_count_me('translation', 'overwrite-in-cache'), 4) + self.assertEqual(recorder.get_count_me("translation", "overwrite-in-cache"), 4) # modify mapping out of cache, this should only evict one entry # because we only need to insert a new update - for offset in [4*MB]: - lpn = offset / (2*KB) - ppn = yield env.process(mapping_cache.update(lpn=lpn, ppn=lpn+1, tag=None)) + for offset in [4 * MB]: + lpn = old_div(offset, (2 * KB)) + ppn = yield env.process( + mapping_cache.update(lpn=lpn, ppn=lpn + 1, tag=None) + ) + self.assertEqual(recorder.get_count_me("translation", "overwrite-in-cache"), 4) self.assertEqual( - recorder.get_count_me('translation', 'overwrite-in-cache'), 4) - self.assertEqual( - recorder.get_count_me('translation', 'write-back-dirty-for-insert'), 0) + recorder.get_count_me("translation", "write-back-dirty-for-insert"), 0 + ) self.assertEqual( - recorder.get_count_me('translation', 'delete-lpn-in-table-for-insert'), 1) + recorder.get_count_me("translation", "delete-lpn-in-table-for-insert"), 1 + ) def test_update(self): conf = create_config() @@ -306,7 +331,7 @@ def test_update(self): mapping_cache = create_mapping_cache(objs) - env = objs['env'] + env = objs["env"] env.process(self.load(conf, env, mapping_cache)) env.run() @@ -319,33 +344,41 @@ def load(self, conf, env, mapping_cache): recorder.enable() # load to cache - for offset in [0*MB, 1*MB, 2*MB, 3*MB]: - lpn = offset / (2*KB) + for offset in [0 * MB, 1 * MB, 2 * MB, 3 * MB]: + lpn = old_div(offset, (2 * KB)) ppn = yield env.process(mapping_cache.lpn_to_ppn(lpn=lpn, tag=None)) self.assertEqual(ppn, UNINITIATED) # modify all mapping - for offset in range(0, 4*MB, 2*KB): - lpn = offset / (2*KB) - ppn = yield env.process(mapping_cache.update(lpn=lpn, ppn=lpn+1, tag=None)) + for offset in range(0, 4 * MB, 2 * KB): + lpn = old_div(offset, (2 * KB)) + ppn = yield env.process( + mapping_cache.update(lpn=lpn, ppn=lpn + 1, tag=None) + ) self.assertEqual( - recorder.get_count_me('translation', 'overwrite-in-cache'), - 4*MB/(2*KB)) + recorder.get_count_me("translation", "overwrite-in-cache"), + old_div(4 * MB, (2 * KB)), + ) # modify mapping out of cache, this should write back a whole TP and # and delete one entry in memory - for offset in [4*MB]: - lpn = offset / (2*KB) - ppn = yield env.process(mapping_cache.update(lpn=lpn, ppn=lpn+1, tag=None)) + for offset in [4 * MB]: + lpn = old_div(offset, (2 * KB)) + ppn = yield env.process( + mapping_cache.update(lpn=lpn, ppn=lpn + 1, tag=None) + ) self.assertEqual( - recorder.get_count_me('translation', 'overwrite-in-cache'), - 4*MB/(2*KB)) + recorder.get_count_me("translation", "overwrite-in-cache"), + old_div(4 * MB, (2 * KB)), + ) self.assertEqual( - recorder.get_count_me('translation', 'write-back-dirty-for-insert'), 1) + recorder.get_count_me("translation", "write-back-dirty-for-insert"), 1 + ) self.assertEqual( - recorder.get_count_me('translation', 'delete-lpn-in-table-for-insert'), 1) + recorder.get_count_me("translation", "delete-lpn-in-table-for-insert"), 1 + ) def test_update(self): conf = create_config() @@ -354,7 +387,7 @@ def test_update(self): mapping_cache = create_mapping_cache(objs) - env = objs['env'] + env = objs["env"] env.process(self.load(conf, env, mapping_cache)) env.run() @@ -367,32 +400,38 @@ def load(self, conf, env, mapping_cache): recorder.enable() # load to cache - for offset in [0*MB, 1*MB, 2*MB, 3*MB]: - lpn = offset / (2*KB) + for offset in [0 * MB, 1 * MB, 2 * MB, 3 * MB]: + lpn = old_div(offset, (2 * KB)) ppn = yield env.process(mapping_cache.lpn_to_ppn(lpn=lpn, tag=None)) self.assertEqual(ppn, UNINITIATED) # modify all mapping - for offset in range(0, 4*MB, 2*KB): - lpn = offset / (2*KB) - ppn = yield env.process(mapping_cache.update(lpn=lpn, ppn=lpn+1, tag=None)) + for offset in range(0, 4 * MB, 2 * KB): + lpn = old_div(offset, (2 * KB)) + ppn = yield env.process( + mapping_cache.update(lpn=lpn, ppn=lpn + 1, tag=None) + ) self.assertEqual( - recorder.get_count_me('translation', 'overwrite-in-cache'), - 4*MB/(2*KB)) + recorder.get_count_me("translation", "overwrite-in-cache"), + old_div(4 * MB, (2 * KB)), + ) # translate a new one - for offset in [4*MB]: - lpn = offset / (2*KB) + for offset in [4 * MB]: + lpn = old_div(offset, (2 * KB)) ppn = yield env.process(mapping_cache.lpn_to_ppn(lpn=lpn, tag=None)) self.assertEqual( - recorder.get_count_me('translation', 'overwrite-in-cache'), - 4*MB/(2*KB)) + recorder.get_count_me("translation", "overwrite-in-cache"), + old_div(4 * MB, (2 * KB)), + ) self.assertEqual( - recorder.get_count_me('translation', 'write-back-dirty-for-load'), 1) + recorder.get_count_me("translation", "write-back-dirty-for-load"), 1 + ) self.assertEqual( - recorder.get_count_me('translation', 'delete-lpn-in-table-for-load'), 512) + recorder.get_count_me("translation", "delete-lpn-in-table-for-load"), 512 + ) def test_update(self): conf = create_config() @@ -401,11 +440,12 @@ def test_update(self): mapping_cache = create_mapping_cache(objs) - env = objs['env'] + env = objs["env"] env.process(self.load(conf, env, mapping_cache)) env.run() + class TestMappingCacheFlush(unittest.TestCase): def load(self, conf, env, mapping_cache): """ @@ -419,10 +459,10 @@ def load(self, conf, env, mapping_cache): recorder.enable() # each tp has 512 entries - self.assertEqual(lpntable.n_free_rows(), 4*512) + self.assertEqual(lpntable.n_free_rows(), 4 * 512) - for offset in [0*MB, 1*MB, 2*MB, 3*MB]: - lpn = offset / (2*KB) + for offset in [0 * MB, 1 * MB, 2 * MB, 3 * MB]: + lpn = old_div(offset, (2 * KB)) ppn = yield env.process(mapping_cache.lpn_to_ppn(lpn=lpn, tag=None)) self.assertEqual(lpntable.n_free_rows(), 0) @@ -436,10 +476,9 @@ def load(self, conf, env, mapping_cache): self.assertEqual(lpntable.n_free_rows(), 0) mapping_cache.drop() - self.assertEqual(lpntable.n_free_rows(), 4*512) + self.assertEqual(lpntable.n_free_rows(), 4 * 512) - self.assertEqual( - recorder.get_count_me('translation', 'overwrite-in-cache'), 0) + self.assertEqual(recorder.get_count_me("translation", "overwrite-in-cache"), 0) def test_update(self): conf = create_config() @@ -448,40 +487,44 @@ def test_update(self): mapping_cache = create_mapping_cache(objs) - env = objs['env'] + env = objs["env"] env.process(self.load(conf, env, mapping_cache)) env.run() + class TestMappingCacheFlushWithEviction(unittest.TestCase): def load(self, conf, env, mapping_cache): lpntable = mapping_cache._lpn_table recorder = mapping_cache.recorder recorder.enable() - for offset in [0*MB, 1*MB, 2*MB, 3*MB]: - lpn = offset / (2*KB) + for offset in [0 * MB, 1 * MB, 2 * MB, 3 * MB]: + lpn = old_div(offset, (2 * KB)) ppn = yield env.process(mapping_cache.lpn_to_ppn(lpn=lpn, tag=None)) self.assertEqual(ppn, UNINITIATED) # modify mapping - for offset in [0*MB, 1*MB, 2*MB, 3*MB]: - lpn = offset / (2*KB) - ppn = yield env.process(mapping_cache.update(lpn=lpn, ppn=lpn+1, tag=None)) + for offset in [0 * MB, 1 * MB, 2 * MB, 3 * MB]: + lpn = old_div(offset, (2 * KB)) + ppn = yield env.process( + mapping_cache.update(lpn=lpn, ppn=lpn + 1, tag=None) + ) - self.assertEqual( - recorder.get_count_me('translation', 'overwrite-in-cache'), 4) + self.assertEqual(recorder.get_count_me("translation", "overwrite-in-cache"), 4) yield env.process(mapping_cache.flush()) + self.assertEqual(recorder.get_count_me("translation", "overwrite-in-cache"), 4) self.assertEqual( - recorder.get_count_me('translation', 'overwrite-in-cache'), 4) - self.assertEqual( - recorder.get_count_me('translation', 'write-back-dirty-for-flush'), 4) + recorder.get_count_me("translation", "write-back-dirty-for-flush"), 4 + ) mapping_cache.drop() self.assertEqual( - recorder.get_count_me('translation', 'delete-lpn-in-table-for-drop'), 512*4) + recorder.get_count_me("translation", "delete-lpn-in-table-for-drop"), + 512 * 4, + ) def test_update(self): conf = create_config() @@ -490,7 +533,7 @@ def test_update(self): mapping_cache = create_mapping_cache(objs) - env = objs['env'] + env = objs["env"] env.process(self.load(conf, env, mapping_cache)) env.run() @@ -502,42 +545,44 @@ def load(self, conf, env, mapping_cache): recorder = mapping_cache.recorder recorder.enable() - for offset in [0*MB, 1*MB, 2*MB, 3*MB]: - lpn = offset / (2*KB) + for offset in [0 * MB, 1 * MB, 2 * MB, 3 * MB]: + lpn = old_div(offset, (2 * KB)) ppn = yield env.process(mapping_cache.lpn_to_ppn(lpn=lpn, tag=None)) self.assertEqual(ppn, UNINITIATED) - self.assertEqual(recorder.get_count_me('translation', - 'read-trans-for-load'), 4) + self.assertEqual(recorder.get_count_me("translation", "read-trans-for-load"), 4) # modify mapping - for offset in [0*MB, 1*MB, 2*MB, 3*MB]: - lpn = offset / (2*KB) - ppn = yield env.process(mapping_cache.update(lpn=lpn, ppn=lpn+1, tag=None)) + for offset in [0 * MB, 1 * MB, 2 * MB, 3 * MB]: + lpn = old_div(offset, (2 * KB)) + ppn = yield env.process( + mapping_cache.update(lpn=lpn, ppn=lpn + 1, tag=None) + ) - self.assertEqual(recorder.get_count_me('translation', - 'read-trans-for-load'), 4) - self.assertEqual( - recorder.get_count_me('translation', 'overwrite-in-cache'), 4) + self.assertEqual(recorder.get_count_me("translation", "read-trans-for-load"), 4) + self.assertEqual(recorder.get_count_me("translation", "overwrite-in-cache"), 4) yield env.process(mapping_cache.flush()) + self.assertEqual(recorder.get_count_me("translation", "overwrite-in-cache"), 4) self.assertEqual( - recorder.get_count_me('translation', 'overwrite-in-cache'), 4) - self.assertEqual( - recorder.get_count_me('translation', 'write-back-dirty-for-flush'), 4) + recorder.get_count_me("translation", "write-back-dirty-for-flush"), 4 + ) mapping_cache.drop() self.assertEqual( - recorder.get_count_me('translation', 'delete-lpn-in-table-for-drop'), 512*4) + recorder.get_count_me("translation", "delete-lpn-in-table-for-drop"), + 512 * 4, + ) - for offset in [0*MB, 1*MB, 2*MB, 3*MB]: - lpn = offset / (2*KB) + for offset in [0 * MB, 1 * MB, 2 * MB, 3 * MB]: + lpn = old_div(offset, (2 * KB)) ppn = yield env.process(mapping_cache.lpn_to_ppn(lpn=lpn, tag=None)) - self.assertEqual(ppn, lpn+1) #stored previously + self.assertEqual(ppn, lpn + 1) # stored previously - self.assertEqual(recorder.get_count_me('translation', - 'read-trans-for-load'), 4+4) + self.assertEqual( + recorder.get_count_me("translation", "read-trans-for-load"), 4 + 4 + ) def test_update(self): conf = create_config() @@ -546,23 +591,21 @@ def test_update(self): mapping_cache = create_mapping_cache(objs) - env = objs['env'] + env = objs["env"] env.process(self.load(conf, env, mapping_cache)) env.run() - - class TestMappingCacheSimpleTranslation(unittest.TestCase): def translate(self, conf, env, mapping_cache): ppn = yield env.process(mapping_cache.lpn_to_ppn(lpn=0, tag=None)) self.assertEqual(ppn, UNINITIATED) - env.exit('finished') + return "finished" def runme(self, conf, env, mapping_cache): ret = yield env.process(self.translate(conf, env, mapping_cache)) - self.assertEqual(ret, 'finished') + self.assertEqual(ret, "finished") def test_update(self): conf = create_config() @@ -571,7 +614,7 @@ def test_update(self): mapping_cache = create_mapping_cache(objs) - env = objs['env'] + env = objs["env"] env.process(self.runme(conf, env, mapping_cache)) env.run() @@ -606,7 +649,7 @@ def update_and_check(self, conf, env, mapping_cache): ppn = yield env.process(mapping_cache.lpn_to_ppn(lpn=lpn, tag=None)) self.assertEqual(ppn, mappings[lpn]) - print 'finished' + print("finished") def test_update(self): conf = create_config() @@ -615,7 +658,7 @@ def test_update(self): mapping_cache = create_mapping_cache(objs) - env = objs['env'] + env = objs["env"] env.process(self.update_and_check(conf, env, mapping_cache)) env.run() @@ -626,7 +669,7 @@ def translate(self, conf, env, mapping_cache): p1 = env.process(mapping_cache.lpn_to_ppn(lpn=0, tag=None)) p2 = env.process(mapping_cache.lpn_to_ppn(lpn=0, tag=None)) yield simpy.AllOf(env, [p1, p2]) - print 'finished' + print("finished") def test_update(self): conf = create_config() @@ -635,7 +678,7 @@ def test_update(self): mapping_cache = create_mapping_cache(objs) - env = objs['env'] + env = objs["env"] env.process(self.translate(conf, env, mapping_cache)) env.run() @@ -645,7 +688,7 @@ def translate(self, conf, env, mapping_cache): p1 = env.process(mapping_cache.update(lpn=0, ppn=1, tag=None)) p2 = env.process(mapping_cache.update(lpn=0, ppn=2, tag=None)) yield simpy.AllOf(env, [p1, p2]) - print 'finished' + print("finished") def test_update(self): conf = create_config() @@ -654,7 +697,7 @@ def test_update(self): mapping_cache = create_mapping_cache(objs) - env = objs['env'] + env = objs["env"] env.process(self.translate(conf, env, mapping_cache)) env.run() @@ -664,18 +707,18 @@ class TestMappingCacheSameLpnUpdateWEvict(unittest.TestCase): def translate(self, conf, env, mapping_cache): # dirty 4 translation pages to mem n = conf.n_mapping_entries_per_page - ext = Extent(lpn_start=0, lpn_count=4*n) + ext = Extent(lpn_start=0, lpn_count=4 * n) for lpn in ext.lpn_iter(): yield env.process(mapping_cache.update(lpn=lpn, ppn=3)) # update a lpn that is not in mem - ext = Extent(lpn_start=4*n, lpn_count=4*n) + ext = Extent(lpn_start=4 * n, lpn_count=4 * n) procs = [] for lpn in ext.lpn_iter(): p = env.process(mapping_cache.update(lpn=lpn, ppn=1, tag=None)) procs.append(p) yield simpy.AllOf(env, procs) - print 'finished' + print("finished") def test_update(self): conf = create_config() @@ -684,7 +727,7 @@ def test_update(self): mapping_cache = create_mapping_cache(objs) - env = objs['env'] + env = objs["env"] env.process(self.translate(conf, env, mapping_cache)) env.run() @@ -722,39 +765,42 @@ def test_parallel_translation(self): conf = create_config() objs = create_obj_set(conf) - dftl = wiscsim.dftldes.Ftl(objs['conf'], objs['rec'], - objs['flash_controller'], objs['env']) - + dftl = wiscsim.dftldes.Ftl( + objs["conf"], objs["rec"], objs["flash_controller"], objs["env"] + ) def test_write(self): conf = create_config() objs = create_obj_set(conf) - env = objs['env'] + env = objs["env"] - dftl = wiscsim.dftldes.Ftl(objs['conf'], objs['rec'], - objs['flash_controller'], objs['env']) + dftl = wiscsim.dftldes.Ftl( + objs["conf"], objs["rec"], objs["flash_controller"], objs["env"] + ) def proc_test_write(self, env, dftl): pass + class TestWrite(unittest.TestCase): def test_write(self): conf = create_config() objs = create_obj_set(conf) - env = objs['env'] + env = objs["env"] - dftl = wiscsim.dftldes.Ftl(objs['conf'], objs['rec'], - objs['flash_controller'], objs['env']) + dftl = wiscsim.dftldes.Ftl( + objs["conf"], objs["rec"], objs["flash_controller"], objs["env"] + ) env.process(self.proc_test_write(objs, dftl, Extent(0, 1))) env.run() def proc_test_write(self, objs, dftl, ext): - env = objs['env'] - time_read_page = objs['flash_controller'].channels[0].read_time - time_program_page = objs['flash_controller'].channels[0].program_time + env = objs["env"] + time_read_page = objs["flash_controller"].channels[0].read_time + time_program_page = objs["flash_controller"].channels[0].program_time - rec = objs['rec'] + rec = objs["rec"] rec.enable() yield env.process(dftl.write_ext(ext)) @@ -766,23 +812,24 @@ def proc_test_write(self, objs, dftl, ext): def test_write_larger(self): conf = create_config() - conf['stripe_size'] = 1 + conf["stripe_size"] = 1 objs = create_obj_set(conf) - env = objs['env'] + env = objs["env"] - dftl = wiscsim.dftldes.Ftl(objs['conf'], objs['rec'], - objs['flash_controller'], objs['env']) + dftl = wiscsim.dftldes.Ftl( + objs["conf"], objs["rec"], objs["flash_controller"], objs["env"] + ) env.process(self.proc_test_write_larger(objs, dftl, Extent(0, 2))) env.run() def proc_test_write_larger(self, objs, dftl, ext): - env = objs['env'] - time_read_page = objs['flash_controller'].channels[0].read_time - time_program_page = objs['flash_controller'].channels[0].program_time + env = objs["env"] + time_read_page = objs["flash_controller"].channels[0].read_time + time_program_page = objs["flash_controller"].channels[0].program_time - rec = objs['rec'] + rec = objs["rec"] rec.enable() yield env.process(dftl.write_ext(ext)) @@ -796,23 +843,23 @@ def proc_test_write_larger(self, objs, dftl, ext): def test_write_larger2(self): conf = create_config() - conf['stripe_size'] = 1 + conf["stripe_size"] = 1 objs = create_obj_set(conf) - env = objs['env'] + env = objs["env"] - dftl = wiscsim.dftldes.Ftl(objs['conf'], objs['rec'], - objs['flash_controller'], objs['env']) + dftl = wiscsim.dftldes.Ftl( + objs["conf"], objs["rec"], objs["flash_controller"], objs["env"] + ) - env.process(self.proc_test_write_larger2(objs, dftl, - Extent(0, 4))) + env.process(self.proc_test_write_larger2(objs, dftl, Extent(0, 4))) env.run() def proc_test_write_larger2(self, objs, dftl, ext): - env = objs['env'] - time_read_page = objs['flash_controller'].channels[0].read_time - time_program_page = objs['flash_controller'].channels[0].program_time + env = objs["env"] + time_read_page = objs["flash_controller"].channels[0].read_time + time_program_page = objs["flash_controller"].channels[0].program_time - rec = objs['rec'] + rec = objs["rec"] rec.enable() yield env.process(dftl.write_ext(ext)) @@ -827,55 +874,66 @@ def proc_test_write_larger2(self, objs, dftl, ext): def test_write_2vpn(self): conf = create_config() - conf['stripe_size'] = 1 + conf["stripe_size"] = 1 # make sure no cache miss in this test conf.n_cache_entries = conf.n_mapping_entries_per_page * 100 objs = create_obj_set(conf) - env = objs['env'] + env = objs["env"] - dftl = wiscsim.dftldes.Ftl(objs['conf'], objs['rec'], - objs['flash_controller'], objs['env']) + dftl = wiscsim.dftldes.Ftl( + objs["conf"], objs["rec"], objs["flash_controller"], objs["env"] + ) - env.process(self.proc_test_write_2vpn(objs, dftl, - Extent(0, conf.n_mapping_entries_per_page * 2))) + env.process( + self.proc_test_write_2vpn( + objs, dftl, Extent(0, conf.n_mapping_entries_per_page * 2) + ) + ) env.run() def proc_test_write_2vpn(self, objs, dftl, ext): - env = objs['env'] - time_read_page = objs['flash_controller'].channels[0].read_time - time_program_page = objs['flash_controller'].channels[0].program_time + env = objs["env"] + time_read_page = objs["flash_controller"].channels[0].read_time + time_program_page = objs["flash_controller"].channels[0].program_time - rec = objs['rec'] + rec = objs["rec"] rec.enable() yield env.process(dftl.write_ext(ext)) # read translation page, then write all 4 data pages at the same time - self.assertEqual(env.now, time_read_page + - (ext.lpn_count/4) * time_program_page) + self.assertEqual( + env.now, time_read_page + (old_div(ext.lpn_count, 4)) * time_program_page + ) self.assertEqual(rec.get_count_me("Mapping_Cache", "miss"), 2) - self.assertEqual(rec.get_count_me("Mapping_Cache", "hit"), - objs['conf'].n_mapping_entries_per_page * 2 - 2) + self.assertEqual( + rec.get_count_me("Mapping_Cache", "hit"), + objs["conf"].n_mapping_entries_per_page * 2 - 2, + ) def test_write_outofspace(self): conf = create_config() - conf['stripe_size'] = 1 + conf["stripe_size"] = 1 objs = create_obj_set(conf) - env = objs['env'] + env = objs["env"] - dftl = wiscsim.dftldes.Ftl(objs['conf'], objs['rec'], - objs['flash_controller'], objs['env']) + dftl = wiscsim.dftldes.Ftl( + objs["conf"], objs["rec"], objs["flash_controller"], objs["env"] + ) - env.process(self.proc_test_write_outofspace(objs, dftl, - Extent(0, conf.total_num_pages()))) + env.process( + self.proc_test_write_outofspace( + objs, dftl, Extent(0, conf.total_num_pages()) + ) + ) env.run() def proc_test_write_outofspace(self, objs, dftl, ext): - env = objs['env'] - time_read_page = objs['flash_controller'].channels[0].read_time - time_program_page = objs['flash_controller'].channels[0].program_time + env = objs["env"] + time_read_page = objs["flash_controller"].channels[0].read_time + time_program_page = objs["flash_controller"].channels[0].program_time with self.assertRaises(wiscsim.blkpool.OutOfSpaceError): yield env.process(dftl.write_ext(ext)) @@ -885,18 +943,19 @@ class TestRead(unittest.TestCase): def test_read(self): conf = create_config() objs = create_obj_set(conf) - env = objs['env'] + env = objs["env"] - dftl = wiscsim.dftldes.Ftl(objs['conf'], objs['rec'], - objs['flash_controller'], objs['env']) + dftl = wiscsim.dftldes.Ftl( + objs["conf"], objs["rec"], objs["flash_controller"], objs["env"] + ) env.process(self.proc_test_read(objs, dftl, Extent(0, 1))) env.run() def proc_test_read(self, objs, dftl, ext): - env = objs['env'] - time_read_page = objs['flash_controller'].channels[0].read_time - time_program_page = objs['flash_controller'].channels[0].program_time + env = objs["env"] + time_read_page = objs["flash_controller"].channels[0].read_time + time_program_page = objs["flash_controller"].channels[0].program_time yield env.process(dftl.read_ext(ext)) @@ -907,20 +966,24 @@ def proc_test_read(self, objs, dftl, ext): def test_read_larger(self): conf = create_config() objs = create_obj_set(conf) - env = objs['env'] + env = objs["env"] - dftl = wiscsim.dftldes.Ftl(objs['conf'], objs['rec'], - objs['flash_controller'], objs['env']) + dftl = wiscsim.dftldes.Ftl( + objs["conf"], objs["rec"], objs["flash_controller"], objs["env"] + ) # read a whole m_vpn's lpn - env.process(self.proc_test_read_larger(objs, dftl, - Extent(0, conf.n_mapping_entries_per_page))) + env.process( + self.proc_test_read_larger( + objs, dftl, Extent(0, conf.n_mapping_entries_per_page) + ) + ) env.run() def proc_test_read_larger(self, objs, dftl, ext): - env = objs['env'] - time_read_page = objs['flash_controller'].channels[0].read_time - time_program_page = objs['flash_controller'].channels[0].program_time + env = objs["env"] + time_read_page = objs["flash_controller"].channels[0].read_time + time_program_page = objs["flash_controller"].channels[0].program_time yield env.process(dftl.read_ext(ext)) @@ -936,20 +999,24 @@ def _test_read_2tp(self): """ conf = create_config() objs = create_obj_set(conf) - env = objs['env'] + env = objs["env"] - dftl = wiscsim.dftldes.Ftl(objs['conf'], objs['rec'], - objs['flash_controller'], objs['env']) + dftl = wiscsim.dftldes.Ftl( + objs["conf"], objs["rec"], objs["flash_controller"], objs["env"] + ) # read two m_vpn's lpns - env.process(self.proc_test_read_2tp(objs, dftl, - Extent(0, 2 * conf.n_mapping_entries_per_page))) + env.process( + self.proc_test_read_2tp( + objs, dftl, Extent(0, 2 * conf.n_mapping_entries_per_page) + ) + ) env.run() def proc_test_read_2tp(self, objs, dftl, ext): - env = objs['env'] - time_read_page = objs['flash_controller'].channels[0].read_time - time_program_page = objs['flash_controller'].channels[0].program_time + env = objs["env"] + time_read_page = objs["flash_controller"].channels[0].read_time + time_program_page = objs["flash_controller"].channels[0].program_time yield env.process(dftl.read_ext(ext)) @@ -981,18 +1048,19 @@ class TestDiscard(unittest.TestCase): def test_discard(self): conf = create_config() objs = create_obj_set(conf) - env = objs['env'] + env = objs["env"] - dftl = wiscsim.dftldes.Ftl(objs['conf'], objs['rec'], - objs['flash_controller'], objs['env']) + dftl = wiscsim.dftldes.Ftl( + objs["conf"], objs["rec"], objs["flash_controller"], objs["env"] + ) env.process(self.proc_test_discard(objs, dftl, Extent(0, 1))) env.run() def proc_test_discard(self, objs, dftl, ext): - env = objs['env'] - time_read_page = objs['flash_controller'].channels[0].read_time - time_program_page = objs['flash_controller'].channels[0].program_time + env = objs["env"] + time_read_page = objs["flash_controller"].channels[0].read_time + time_program_page = objs["flash_controller"].channels[0].program_time yield env.process(dftl.discard_ext(ext)) @@ -1035,28 +1103,28 @@ def test_ops(self): self.assertEqual(table.n_locked_free_rows(), 1) self.assertEqual(table.n_used_rows(), 0) - table.add_lpn(rowid = rowid, lpn = 8, ppn = 88, dirty = True) + table.add_lpn(rowid=rowid, lpn=8, ppn=88, dirty=True) self.assertEqual(table.lpn_to_ppn(8), 88) self.assertEqual(table.is_dirty(8), True) self.assertEqual(table.n_free_rows(), 7) self.assertEqual(table.n_locked_free_rows(), 0) self.assertEqual(table.n_used_rows(), 1) - table.overwrite_lpn(lpn = 8, ppn = 89, dirty = False) + table.overwrite_lpn(lpn=8, ppn=89, dirty=False) self.assertEqual(table.lpn_to_ppn(8), 89) self.assertEqual(table.is_dirty(8), False) self.assertEqual(table.n_free_rows(), 7) self.assertEqual(table.n_locked_free_rows(), 0) self.assertEqual(table.n_used_rows(), 1) - deleted_rowid = table.delete_lpn_and_lock(lpn = 8) + deleted_rowid = table.delete_lpn_and_lock(lpn=8) self.assertEqual(rowid, deleted_rowid) self.assertEqual(table.has_lpn(8), False) self.assertEqual(table.n_free_rows(), 7) self.assertEqual(table.n_locked_free_rows(), 1) self.assertEqual(table.n_used_rows(), 0) - table.unlock_free_row(rowid = deleted_rowid) + table.unlock_free_row(rowid=deleted_rowid) self.assertEqual(table.n_free_rows(), 8) self.assertEqual(table.n_locked_free_rows(), 0) self.assertEqual(table.n_used_rows(), 0) @@ -1075,7 +1143,7 @@ def test_multiple_adds(self): locked_rows = table.lock_free_rows(3) self.assertEqual(len(locked_rows), 3) - table.add_lpns(locked_rows, {1:11, 2:22, 3:33}, False) + table.add_lpns(locked_rows, {1: 11, 2: 22, 3: 33}, False) self.assertEqual(table.n_free_rows(), 5) self.assertEqual(table.n_locked_free_rows(), 0) @@ -1090,9 +1158,9 @@ def test_locking_lpn(self): locked_rows = table.lock_free_rows(3) self.assertEqual(len(locked_rows), 3) - self.assertListEqual(locked_rows, [0,1,2]) + self.assertListEqual(locked_rows, [0, 1, 2]) - table.add_lpns(locked_rows, {1:11, 2:22, 3:33}, False) + table.add_lpns(locked_rows, {1: 11, 2: 22, 3: 33}, False) table.lock_lpn(1) self.assertEqual(table.n_free_rows(), 5) @@ -1131,21 +1199,21 @@ def test_request(self): class TestParallelDFTL(unittest.TestCase): def setup_config(self): self.conf = wiscsim.dftldes.Config() - self.conf['SSDFramework']['ncq_depth'] = 1 + self.conf["SSDFramework"]["ncq_depth"] = 1 - self.conf['flash_config']['n_pages_per_block'] = 2 - self.conf['flash_config']['n_blocks_per_plane'] = 2 - self.conf['flash_config']['n_planes_per_chip'] = 1 - self.conf['flash_config']['n_chips_per_package'] = 1 - self.conf['flash_config']['n_packages_per_channel'] = 1 - self.conf['flash_config']['n_channels_per_dev'] = 4 + self.conf["flash_config"]["n_pages_per_block"] = 2 + self.conf["flash_config"]["n_blocks_per_plane"] = 2 + self.conf["flash_config"]["n_planes_per_chip"] = 1 + self.conf["flash_config"]["n_chips_per_package"] = 1 + self.conf["flash_config"]["n_packages_per_channel"] = 1 + self.conf["flash_config"]["n_channels_per_dev"] = 4 def setup_environment(self): - metadata_dic = utils.choose_exp_metadata(self.conf, interactive = False) + metadata_dic = utils.choose_exp_metadata(self.conf, interactive=False) self.conf.update(metadata_dic) - self.conf['enable_blktrace'] = True - self.conf['enable_simulation'] = True + self.conf["enable_blktrace"] = True + self.conf["enable_simulation"] = True def setup_workload(self): w = OP_WRITE @@ -1155,20 +1223,18 @@ def setup_workload(self): self.conf["workload_src"] = LBAGENERATOR self.conf["lba_workload_class"] = "ExtentTestWorkloadFLEX2" self.conf["lba_workload_configs"]["ExtentTestWorkloadFLEX2"] = { - "events": [ - (d, 1, 3), - (w, 1, 3), - (r, 1, 3) - ]} + "events": [(d, 1, 3), (w, 1, 3), (r, 1, 3)] + } self.conf["age_workload_class"] = "NoOp" def setup_ftl(self): - self.conf['ftl_type'] = 'dftldes' - self.conf['simulator_class'] = 'SimulatorDESNew' + self.conf["ftl_type"] = "dftldes" + self.conf["simulator_class"] = "SimulatorDESNew" logicsize_mb = 2 - self.conf.mapping_cache_bytes = self.conf.n_mapping_entries_per_page \ - * self.conf['cache_entry_bytes'] # 8 bytes (64bits) needed in mem + self.conf.mapping_cache_bytes = ( + self.conf.n_mapping_entries_per_page * self.conf["cache_entry_bytes"] + ) # 8 bytes (64bits) needed in mem self.conf.set_flash_num_blocks_by_bytes(int(logicsize_mb * 2**20 * 1.28)) def my_run(self): @@ -1186,48 +1252,57 @@ def test_main(self): class TestFTLwithMoreData(unittest.TestCase): def setup_config(self): self.conf = wiscsim.dftldes.Config() - self.conf['SSDFramework']['ncq_depth'] = 2 - - self.conf['flash_config']['page_size'] = 2048 - self.conf['flash_config']['n_pages_per_block'] = 4 - self.conf['flash_config']['n_blocks_per_plane'] = 8 - self.conf['flash_config']['n_planes_per_chip'] = 1 - self.conf['flash_config']['n_chips_per_package'] = 1 - self.conf['flash_config']['n_packages_per_channel'] = 1 - self.conf['flash_config']['n_channels_per_dev'] = 4 + self.conf["SSDFramework"]["ncq_depth"] = 2 + self.conf["flash_config"]["page_size"] = 2048 + self.conf["flash_config"]["n_pages_per_block"] = 4 + self.conf["flash_config"]["n_blocks_per_plane"] = 8 + self.conf["flash_config"]["n_planes_per_chip"] = 1 + self.conf["flash_config"]["n_chips_per_package"] = 1 + self.conf["flash_config"]["n_packages_per_channel"] = 1 + self.conf["flash_config"]["n_channels_per_dev"] = 4 def setup_environment(self): - metadata_dic = utils.choose_exp_metadata(self.conf, interactive = False) + metadata_dic = utils.choose_exp_metadata(self.conf, interactive=False) self.conf.update(metadata_dic) - self.conf['enable_blktrace'] = True - self.conf['enable_simulation'] = True + self.conf["enable_blktrace"] = True + self.conf["enable_simulation"] = True def setup_workload(self): self.conf["workload_src"] = LBAGENERATOR self.conf["lba_workload_class"] = "TestWorkloadFLEX3" - traffic = 2*MB - chunk_size = 32*KB - page_size = self.conf['flash_config']['page_size'] + traffic = 2 * MB + chunk_size = 32 * KB + page_size = self.conf["flash_config"]["page_size"] self.conf["lba_workload_configs"]["TestWorkloadFLEX3"] = { - "op_count": traffic/chunk_size, - "extent_size": chunk_size/page_size , - "ops": [OP_WRITE], 'mode': 'random'} - print self.conf['lba_workload_configs']['TestWorkloadFLEX3'] + "op_count": old_div(traffic, chunk_size), + "extent_size": old_div(chunk_size, page_size), + "ops": [OP_WRITE], + "mode": "random", + } + print(self.conf["lba_workload_configs"]["TestWorkloadFLEX3"]) self.conf["age_workload_class"] = "NoOp" def setup_ftl(self): - self.conf['ftl_type'] = 'dftldes' - self.conf['simulator_class'] = 'SimulatorDESNew' + self.conf["ftl_type"] = "dftldes" + self.conf["simulator_class"] = "SimulatorDESNew" logicsize_mb = 8 - entries_need = int(logicsize_mb * 2**20 * 0.03 / self.conf['flash_config']['page_size']) - self.conf.mapping_cache_bytes = 4 * self.conf.n_mapping_entries_per_page * self.conf['cache_entry_bytes'] # 8 bytes (64bits) needed in mem + entries_need = int( + logicsize_mb * 2**20 * 0.03 / self.conf["flash_config"]["page_size"] + ) + self.conf.mapping_cache_bytes = ( + 4 * self.conf.n_mapping_entries_per_page * self.conf["cache_entry_bytes"] + ) # 8 bytes (64bits) needed in mem self.conf.set_flash_num_blocks_by_bytes(int(logicsize_mb * 2**20 * 1.3)) - print "Current n_blocks_per_plane",\ - self.conf['flash_config']['n_blocks_per_plane'] + print( + ( + "Current n_blocks_per_plane", + self.conf["flash_config"]["n_blocks_per_plane"], + ) + ) def my_run(self): utils.runtime_update(self.conf) @@ -1244,216 +1319,299 @@ def test_main(self): class TestTranslationWithWrite(unittest.TestCase): def test(self): conf = create_config() - conf['flash_config']['n_channels_per_dev'] = 4 - conf['stripe_size'] = 1 + conf["flash_config"]["n_channels_per_dev"] = 4 + conf["stripe_size"] = 1 - conf.n_cache_entries = 4*conf.n_mapping_entries_per_page - conf.set_flash_num_blocks_by_bytes(128*MB) + conf.n_cache_entries = 4 * conf.n_mapping_entries_per_page + conf.set_flash_num_blocks_by_bytes(128 * MB) objs = create_obj_set(conf) - env = objs['env'] + env = objs["env"] - dftl = wiscsim.dftldes.Ftl(objs['conf'], objs['rec'], - objs['flash_controller'], objs['env']) + dftl = wiscsim.dftldes.Ftl( + objs["conf"], objs["rec"], objs["flash_controller"], objs["env"] + ) env.process(self.proc_test_write(objs, dftl)) env.run() def proc_test_write(self, objs, dftl): - env = objs['env'] - rec = objs['rec'] + env = objs["env"] + rec = objs["rec"] rec.enable() block_pool = dftl.block_pool oob = dftl.oob - conf = objs['conf'] - PAGE_SIZE = 2*KB + conf = objs["conf"] + PAGE_SIZE = 2 * KB # Write first 4*MB for i in range(4): - yield env.process(dftl.write_ext( - Extent((i*MB/PAGE_SIZE), 1*MB/PAGE_SIZE))) - - self.assertEqual(rec.get_count_me('translation', 'write-back-dirty-for-load'), 0) - self.assertEqual(rec.get_count_me('translation', 'write-back-dirty-for-insert'), 0) - self.assertEqual(rec.get_count_me('translation', 'delete-lpn-in-table-for-load'), 0) - self.assertEqual(rec.get_count_me('translation', 'delete-lpn-in-table-for-insert'), 0) - self.assertEqual(rec.get_count_me('translation', 'read-trans-for-load'), 4) - self.assertEqual(rec.get_count_me('translation', 'overwrite-in-cache'), 4*512) - self.assertEqual(rec.get_count_me('translation', 'insert-to-free'), 0) - - yield env.process(dftl.write_ext(Extent(4*MB/PAGE_SIZE, 1))) - - self.assertEqual(rec.get_count_me('translation', 'write-back-dirty-for-load'), 0+1) - self.assertEqual(rec.get_count_me('translation', 'write-back-dirty-for-insert'), 0) - self.assertEqual(rec.get_count_me('translation', 'delete-lpn-in-table-for-load'), 0+512) - self.assertEqual(rec.get_count_me('translation', 'delete-lpn-in-table-for-insert'), 0+0) - self.assertEqual(rec.get_count_me('translation', 'read-trans-for-load'), 4+1) - self.assertEqual(rec.get_count_me('translation', 'overwrite-in-cache'), 4*512+1) - self.assertEqual(rec.get_count_me('translation', 'insert-to-free'), 0+0) - - yield env.process(dftl.write_ext(Extent(5*MB/PAGE_SIZE, 1))) - - self.assertEqual(rec.get_count_me('translation', 'write-back-dirty-for-load'), 0+1+1) - self.assertEqual(rec.get_count_me('translation', 'write-back-dirty-for-insert'), 0) - self.assertEqual(rec.get_count_me('translation', 'delete-lpn-in-table-for-load'), 0+512+512) - self.assertEqual(rec.get_count_me('translation', 'delete-lpn-in-table-for-insert'), 0+0+0) - self.assertEqual(rec.get_count_me('translation', 'read-trans-for-load'), 4+1+1) - self.assertEqual(rec.get_count_me('translation', 'overwrite-in-cache'), 4*512+1+1) - self.assertEqual(rec.get_count_me('translation', 'insert-to-free'), 0+0+0) + yield env.process( + dftl.write_ext( + Extent((old_div(i * MB, PAGE_SIZE)), old_div(1 * MB, PAGE_SIZE)) + ) + ) + + self.assertEqual( + rec.get_count_me("translation", "write-back-dirty-for-load"), 0 + ) + self.assertEqual( + rec.get_count_me("translation", "write-back-dirty-for-insert"), 0 + ) + self.assertEqual( + rec.get_count_me("translation", "delete-lpn-in-table-for-load"), 0 + ) + self.assertEqual( + rec.get_count_me("translation", "delete-lpn-in-table-for-insert"), 0 + ) + self.assertEqual(rec.get_count_me("translation", "read-trans-for-load"), 4) + self.assertEqual(rec.get_count_me("translation", "overwrite-in-cache"), 4 * 512) + self.assertEqual(rec.get_count_me("translation", "insert-to-free"), 0) + + yield env.process(dftl.write_ext(Extent(old_div(4 * MB, PAGE_SIZE), 1))) + + self.assertEqual( + rec.get_count_me("translation", "write-back-dirty-for-load"), 0 + 1 + ) + self.assertEqual( + rec.get_count_me("translation", "write-back-dirty-for-insert"), 0 + ) + self.assertEqual( + rec.get_count_me("translation", "delete-lpn-in-table-for-load"), 0 + 512 + ) + self.assertEqual( + rec.get_count_me("translation", "delete-lpn-in-table-for-insert"), 0 + 0 + ) + self.assertEqual(rec.get_count_me("translation", "read-trans-for-load"), 4 + 1) + self.assertEqual( + rec.get_count_me("translation", "overwrite-in-cache"), 4 * 512 + 1 + ) + self.assertEqual(rec.get_count_me("translation", "insert-to-free"), 0 + 0) + + yield env.process(dftl.write_ext(Extent(old_div(5 * MB, PAGE_SIZE), 1))) + + self.assertEqual( + rec.get_count_me("translation", "write-back-dirty-for-load"), 0 + 1 + 1 + ) + self.assertEqual( + rec.get_count_me("translation", "write-back-dirty-for-insert"), 0 + ) + self.assertEqual( + rec.get_count_me("translation", "delete-lpn-in-table-for-load"), + 0 + 512 + 512, + ) + self.assertEqual( + rec.get_count_me("translation", "delete-lpn-in-table-for-insert"), 0 + 0 + 0 + ) + self.assertEqual( + rec.get_count_me("translation", "read-trans-for-load"), 4 + 1 + 1 + ) + self.assertEqual( + rec.get_count_me("translation", "overwrite-in-cache"), 4 * 512 + 1 + 1 + ) + self.assertEqual(rec.get_count_me("translation", "insert-to-free"), 0 + 0 + 0) class TestTranslationWithWriteTwice(unittest.TestCase): def test(self): conf = create_config() - conf['flash_config']['n_channels_per_dev'] = 4 - conf['stripe_size'] = 1 + conf["flash_config"]["n_channels_per_dev"] = 4 + conf["stripe_size"] = 1 - conf.n_cache_entries = 4*conf.n_mapping_entries_per_page - conf.set_flash_num_blocks_by_bytes(128*MB) + conf.n_cache_entries = 4 * conf.n_mapping_entries_per_page + conf.set_flash_num_blocks_by_bytes(128 * MB) objs = create_obj_set(conf) - env = objs['env'] + env = objs["env"] - dftl = wiscsim.dftldes.Ftl(objs['conf'], objs['rec'], - objs['flash_controller'], objs['env']) + dftl = wiscsim.dftldes.Ftl( + objs["conf"], objs["rec"], objs["flash_controller"], objs["env"] + ) env.process(self.proc_test_write(objs, dftl)) env.run() def proc_test_write(self, objs, dftl): - env = objs['env'] - rec = objs['rec'] + env = objs["env"] + rec = objs["rec"] rec.enable() block_pool = dftl.block_pool oob = dftl.oob - conf = objs['conf'] - PAGE_SIZE = 2*KB + conf = objs["conf"] + PAGE_SIZE = 2 * KB # Write first 4*MB for i in range(4): - yield env.process(dftl.write_ext( - Extent((i*MB/PAGE_SIZE), 1*MB/PAGE_SIZE))) + yield env.process( + dftl.write_ext( + Extent((old_div(i * MB, PAGE_SIZE)), old_div(1 * MB, PAGE_SIZE)) + ) + ) - self.assertEqual(rec.get_count_me('translation', 'write-back-dirty-for-load'), 0) + self.assertEqual( + rec.get_count_me("translation", "write-back-dirty-for-load"), 0 + ) for i in range(4, 8): - yield env.process(dftl.write_ext( - Extent((i*MB/PAGE_SIZE), 1*MB/PAGE_SIZE))) + yield env.process( + dftl.write_ext( + Extent((old_div(i * MB, PAGE_SIZE)), old_div(1 * MB, PAGE_SIZE)) + ) + ) # should have purged previous mappings - self.assertEqual(rec.get_count_me('translation', 'write-back-dirty-for-load'), 4) + self.assertEqual( + rec.get_count_me("translation", "write-back-dirty-for-load"), 4 + ) for i in range(4): - yield env.process(dftl.write_ext( - Extent((i*MB/PAGE_SIZE), 1*MB/PAGE_SIZE))) + yield env.process( + dftl.write_ext( + Extent((old_div(i * MB, PAGE_SIZE)), old_div(1 * MB, PAGE_SIZE)) + ) + ) - self.assertEqual(rec.get_count_me('translation', 'write-back-dirty-for-load'), 8) + self.assertEqual( + rec.get_count_me("translation", "write-back-dirty-for-load"), 8 + ) class TestTranslationWriteBacks(unittest.TestCase): def test(self): conf = create_config() - conf['flash_config']['n_channels_per_dev'] = 4 - conf['stripe_size'] = 1 + conf["flash_config"]["n_channels_per_dev"] = 4 + conf["stripe_size"] = 1 - conf.n_cache_entries = 4*conf.n_mapping_entries_per_page - conf.set_flash_num_blocks_by_bytes(1*GB) + conf.n_cache_entries = 4 * conf.n_mapping_entries_per_page + conf.set_flash_num_blocks_by_bytes(1 * GB) objs = create_obj_set(conf) - env = objs['env'] + env = objs["env"] - dftl = wiscsim.dftldes.Ftl(objs['conf'], objs['rec'], - objs['flash_controller'], objs['env']) + dftl = wiscsim.dftldes.Ftl( + objs["conf"], objs["rec"], objs["flash_controller"], objs["env"] + ) env.process(self.proc_test_write(objs, dftl)) env.run() def proc_test_write(self, objs, dftl): return - env = objs['env'] - rec = objs['rec'] + env = objs["env"] + rec = objs["rec"] rec.enable() block_pool = dftl.block_pool oob = dftl.oob - conf = objs['conf'] - PAGE_SIZE = 2*KB + conf = objs["conf"] + PAGE_SIZE = 2 * KB - n = 4*512 - 511 - npages = 32*MB/PAGE_SIZE - lpns = range(npages) + n = 4 * 512 - 511 + npages = old_div(32 * MB, PAGE_SIZE) + lpns = list(range(npages)) # random.shuffle(lpns) - print n - for i in range(4*512*2): + print(n) + for i in range(4 * 512 * 2): lpn = random.choice(lpns) yield env.process(dftl.write_ext(Extent(lpn, 1))) # yield env.process(dftl.write_ext(Extent(lpns[i], 1))) - print i, n, rec.get_count_me('translation', 'write-back-dirty-for-load') - - self.assertEqual(rec.get_count_me('translation', 'write-back-dirty-for-load'), 0) - + print((i, n, rec.get_count_me("translation", "write-back-dirty-for-load"))) + self.assertEqual( + rec.get_count_me("translation", "write-back-dirty-for-load"), 0 + ) class TestTranslationWithWriteNotAligned(unittest.TestCase): def test(self): conf = create_config() - conf['flash_config']['n_channels_per_dev'] = 4 - conf['stripe_size'] = 1 + conf["flash_config"]["n_channels_per_dev"] = 4 + conf["stripe_size"] = 1 - conf.n_cache_entries = 4*conf.n_mapping_entries_per_page - conf.set_flash_num_blocks_by_bytes(128*MB) + conf.n_cache_entries = 4 * conf.n_mapping_entries_per_page + conf.set_flash_num_blocks_by_bytes(128 * MB) objs = create_obj_set(conf) - env = objs['env'] + env = objs["env"] - dftl = wiscsim.dftldes.Ftl(objs['conf'], objs['rec'], - objs['flash_controller'], objs['env']) + dftl = wiscsim.dftldes.Ftl( + objs["conf"], objs["rec"], objs["flash_controller"], objs["env"] + ) env.process(self.proc_test_write(objs, dftl)) env.run() def proc_test_write(self, objs, dftl): - env = objs['env'] - rec = objs['rec'] + env = objs["env"] + rec = objs["rec"] rec.enable() block_pool = dftl.block_pool oob = dftl.oob - conf = objs['conf'] - PAGE_SIZE = 2*KB + conf = objs["conf"] + PAGE_SIZE = 2 * KB # Write 4 places for i in [8, 22, 33, 77]: - yield env.process(dftl.write_ext( - Extent((i*MB/PAGE_SIZE) + 1, 1))) - - self.assertEqual(rec.get_count_me('translation', 'write-back-dirty-for-load'), 0) - self.assertEqual(rec.get_count_me('translation', 'write-back-dirty-for-insert'), 0) - self.assertEqual(rec.get_count_me('translation', 'delete-lpn-in-table-for-load'), 0) - self.assertEqual(rec.get_count_me('translation', 'delete-lpn-in-table-for-insert'), 0) - self.assertEqual(rec.get_count_me('translation', 'read-trans-for-load'), 4) - self.assertEqual(rec.get_count_me('translation', 'overwrite-in-cache'), 4) - self.assertEqual(rec.get_count_me('translation', 'insert-to-free'), 0) - - yield env.process(dftl.write_ext(Extent(7*MB/PAGE_SIZE + 1, 1))) - - self.assertEqual(rec.get_count_me('translation', 'write-back-dirty-for-load'), 0+0) - self.assertEqual(rec.get_count_me('translation', 'write-back-dirty-for-insert'), 0) - self.assertEqual(rec.get_count_me('translation', 'delete-lpn-in-table-for-load'), 0+512) - self.assertEqual(rec.get_count_me('translation', 'delete-lpn-in-table-for-insert'), 0+0) - self.assertEqual(rec.get_count_me('translation', 'read-trans-for-load'), 4+1) - self.assertEqual(rec.get_count_me('translation', 'overwrite-in-cache'), 4+1) - self.assertEqual(rec.get_count_me('translation', 'insert-to-free'), 0+0) - - yield env.process(dftl.write_ext(Extent(5*MB/PAGE_SIZE, 1))) - - self.assertEqual(rec.get_count_me('translation', 'write-back-dirty-for-load'), 0) - self.assertEqual(rec.get_count_me('translation', 'write-back-dirty-for-insert'), 0) - self.assertEqual(rec.get_count_me('translation', 'delete-lpn-in-table-for-load'), 0+512+512) - self.assertEqual(rec.get_count_me('translation', 'delete-lpn-in-table-for-insert'), 0+0+0) - self.assertEqual(rec.get_count_me('translation', 'read-trans-for-load'), 4+1+1) - self.assertEqual(rec.get_count_me('translation', 'overwrite-in-cache'), 4+1+1) - self.assertEqual(rec.get_count_me('translation', 'insert-to-free'), 0+0+0) + yield env.process( + dftl.write_ext(Extent((old_div(i * MB, PAGE_SIZE)) + 1, 1)) + ) + + self.assertEqual( + rec.get_count_me("translation", "write-back-dirty-for-load"), 0 + ) + self.assertEqual( + rec.get_count_me("translation", "write-back-dirty-for-insert"), 0 + ) + self.assertEqual( + rec.get_count_me("translation", "delete-lpn-in-table-for-load"), 0 + ) + self.assertEqual( + rec.get_count_me("translation", "delete-lpn-in-table-for-insert"), 0 + ) + self.assertEqual(rec.get_count_me("translation", "read-trans-for-load"), 4) + self.assertEqual(rec.get_count_me("translation", "overwrite-in-cache"), 4) + self.assertEqual(rec.get_count_me("translation", "insert-to-free"), 0) + + yield env.process(dftl.write_ext(Extent(old_div(7 * MB, PAGE_SIZE) + 1, 1))) + + self.assertEqual( + rec.get_count_me("translation", "write-back-dirty-for-load"), 0 + 0 + ) + self.assertEqual( + rec.get_count_me("translation", "write-back-dirty-for-insert"), 0 + ) + self.assertEqual( + rec.get_count_me("translation", "delete-lpn-in-table-for-load"), 0 + 512 + ) + self.assertEqual( + rec.get_count_me("translation", "delete-lpn-in-table-for-insert"), 0 + 0 + ) + self.assertEqual(rec.get_count_me("translation", "read-trans-for-load"), 4 + 1) + self.assertEqual(rec.get_count_me("translation", "overwrite-in-cache"), 4 + 1) + self.assertEqual(rec.get_count_me("translation", "insert-to-free"), 0 + 0) + + yield env.process(dftl.write_ext(Extent(old_div(5 * MB, PAGE_SIZE), 1))) + + self.assertEqual( + rec.get_count_me("translation", "write-back-dirty-for-load"), 0 + ) + self.assertEqual( + rec.get_count_me("translation", "write-back-dirty-for-insert"), 0 + ) + self.assertEqual( + rec.get_count_me("translation", "delete-lpn-in-table-for-load"), + 0 + 512 + 512, + ) + self.assertEqual( + rec.get_count_me("translation", "delete-lpn-in-table-for-insert"), 0 + 0 + 0 + ) + self.assertEqual( + rec.get_count_me("translation", "read-trans-for-load"), 4 + 1 + 1 + ) + self.assertEqual( + rec.get_count_me("translation", "overwrite-in-cache"), 4 + 1 + 1 + ) + self.assertEqual(rec.get_count_me("translation", "insert-to-free"), 0 + 0 + 0) class TestWearLevelingVictimBlocks(unittest.TestCase): @@ -1489,7 +1647,7 @@ def test_some_used(self): vbs.n_victims = 3 victims = list(vbs.iterator_verbose()) - vblocks = [ blocknum for _, _, blocknum in victims ] + vblocks = [blocknum for _, _, blocknum in victims] self.assertListEqual(sorted(vblocks), sorted([block1, block2, block3])) @@ -1520,8 +1678,8 @@ def test_cur_blocks(self): def test_one_victim_candidate(self): conf = create_config() - conf['flash_config']['n_channels_per_dev'] = 1 - conf['stripe_size'] = 'infinity' + conf["flash_config"]["n_channels_per_dev"] = 1 + conf["stripe_size"] = "infinity" block_pool = create_blockpool(conf) oob = create_oob(conf) @@ -1540,8 +1698,8 @@ def test_one_victim_candidate(self): def test_3_victim_candidates(self): conf = create_config() - conf['flash_config']['n_channels_per_dev'] = 1 - conf['stripe_size'] = 'infinity' + conf["flash_config"]["n_channels_per_dev"] = 1 + conf["stripe_size"] = "infinity" block_pool = create_blockpool(conf) oob = create_oob(conf) @@ -1586,20 +1744,20 @@ def test_valid_ratio_stats(self): ratios = [] self.validate_n_pages_in_block(conf, blockpool, oob, 0) self.validate_n_pages_in_block(conf, blockpool, oob, 0) - ratios.append( 0 / n ) + ratios.append(old_div(0, n)) self.validate_n_pages_in_block(conf, blockpool, oob, 2) self.validate_n_pages_in_block(conf, blockpool, oob, 2) - ratios.append( 2 / n ) + ratios.append(old_div(2, n)) self.validate_n_pages_in_block(conf, blockpool, oob, 3) self.validate_n_pages_in_block(conf, blockpool, oob, 3) - ratios.append( 3 / n ) + ratios.append(old_div(3, n)) self.validate_n_pages_in_block(conf, blockpool, oob, n) self.validate_n_pages_in_block(conf, blockpool, oob, n) - ratios.append( n / n ) + ratios.append(old_div(n, n)) counter = vbs.get_valid_ratio_counter_of_used_blocks() for valid_ratio in ratios: - ratio_str = "{0:.2f}".format(valid_ratio) + ratio_str = f"{valid_ratio:.2f}" self.assertEqual(counter[ratio_str], 2) def validate_n_pages_in_block(self, conf, block_pool, oob, n): @@ -1609,37 +1767,36 @@ def validate_n_pages_in_block(self, conf, block_pool, oob, n): oob.states.validate_page(ppn) - class TestGC(unittest.TestCase): def test_valid_ratio(self): conf = create_config() - conf['flash_config']['n_channels_per_dev'] = 1 - conf['stripe_size'] = 'infinity' - conf.set_flash_num_blocks_by_bytes(128*MB) + conf["flash_config"]["n_channels_per_dev"] = 1 + conf["stripe_size"] = "infinity" + conf.set_flash_num_blocks_by_bytes(128 * MB) objs = create_obj_set(conf) - env = objs['env'] + env = objs["env"] - dftl = wiscsim.dftldes.Ftl(objs['conf'], objs['rec'], - objs['flash_controller'], objs['env']) + dftl = wiscsim.dftldes.Ftl( + objs["conf"], objs["rec"], objs["flash_controller"], objs["env"] + ) env.process(self.proc_test_write(objs, dftl)) env.run() def proc_test_write(self, objs, dftl): - env = objs['env'] - time_read_page = objs['flash_controller'].channels[0].read_time - time_program_page = objs['flash_controller'].channels[0].program_time + env = objs["env"] + time_read_page = objs["flash_controller"].channels[0].read_time + time_program_page = objs["flash_controller"].channels[0].program_time - rec = objs['rec'] + rec = objs["rec"] rec.enable() block_pool = dftl.block_pool oob = dftl.oob - victims = wiscsim.dftldes.VictimBlocks(objs['conf'], block_pool, oob) - + victims = wiscsim.dftldes.VictimBlocks(objs["conf"], block_pool, oob) - conf = objs['conf'] + conf = objs["conf"] n = conf.n_pages_per_block yield env.process(dftl.write_ext(Extent(0, n))) self.assertEqual(len(list(victims.iterator_verbose())), 0) @@ -1648,7 +1805,7 @@ def proc_test_write(self, objs, dftl): victim_blocks = list(victims.iterator_verbose()) valid_ratio, block_type, block_num = victim_blocks[0] - self.assertEqual(valid_ratio, (n-1.0)/n) + self.assertEqual(valid_ratio, old_div((n - 1.0), n)) yield env.process(dftl.write_ext(Extent(0, n))) @@ -1660,41 +1817,41 @@ def proc_test_write(self, objs, dftl): class TestDataBlockCleaner(unittest.TestCase): def test(self): conf = create_config() - conf['flash_config']['n_channels_per_dev'] = 1 - conf['stripe_size'] = 'infinity' - conf.set_flash_num_blocks_by_bytes(128*MB) + conf["flash_config"]["n_channels_per_dev"] = 1 + conf["stripe_size"] = "infinity" + conf.set_flash_num_blocks_by_bytes(128 * MB) objs = create_obj_set(conf) - env = objs['env'] + env = objs["env"] - dftl = FtlTest(objs['conf'], objs['rec'], - objs['flash_controller'], objs['env']) + dftl = FtlTest(objs["conf"], objs["rec"], objs["flash_controller"], objs["env"]) env.process(self.proc_test_write(objs, dftl)) env.run() def proc_test_write(self, objs, dftl): - conf = objs['conf'] - env = objs['env'] - rec = objs['rec'] + conf = objs["conf"] + env = objs["env"] + rec = objs["rec"] rec.enable() - time_read_page = objs['flash_controller'].channels[0].read_time - time_program_page = objs['flash_controller'].channels[0].program_time - time_erase_block = objs['flash_controller'].channels[0].erase_time + time_read_page = objs["flash_controller"].channels[0].read_time + time_program_page = objs["flash_controller"].channels[0].program_time + time_erase_block = objs["flash_controller"].channels[0].erase_time block_pool = dftl.block_pool oob = dftl.oob mappings = dftl.get_mappings() - victims = wiscsim.dftldes.VictimBlocks(objs['conf'], block_pool, oob) + victims = wiscsim.dftldes.VictimBlocks(objs["conf"], block_pool, oob) datablockcleaner = wiscsim.dftldes.DataBlockCleaner( - conf = objs['conf'], - flash = objs['flash_controller'], - oob = oob, - block_pool = block_pool, - mappings = mappings, - rec = objs['rec'], - env = objs['env']) + conf=objs["conf"], + flash=objs["flash_controller"], + oob=oob, + block_pool=block_pool, + mappings=mappings, + rec=objs["rec"], + env=objs["env"], + ) n = conf.n_pages_per_block yield env.process(dftl.write_ext(Extent(0, n))) @@ -1702,14 +1859,18 @@ def proc_test_write(self, objs, dftl): victim_blocks = list(victims.iterator_verbose()) valid_ratio, block_type, victim_block = victim_blocks[0] - self.assertEqual(oob.states.block_valid_ratio(victim_block), (n-1.0)/n) + self.assertEqual( + oob.states.block_valid_ratio(victim_block), old_div((n - 1.0), n) + ) s = env.now yield env.process(datablockcleaner.clean(victim_block)) # check the time to move n-1 valid pages - self.assertEqual(env.now, - s + (n-1)*(time_read_page+time_program_page) + time_erase_block) + self.assertEqual( + env.now, + s + (n - 1) * (time_read_page + time_program_page) + time_erase_block, + ) # check validation self.assertEqual(oob.states.block_valid_ratio(victim_block), 0) @@ -1731,21 +1892,19 @@ def proc_test_write(self, objs, dftl): class TestTransBlockCleaner(unittest.TestCase): def test(self): conf = create_config() - conf['flash_config']['n_channels_per_dev'] = 1 - conf['stripe_size'] = 'infinity' - conf.set_flash_num_blocks_by_bytes(128*MB) + conf["flash_config"]["n_channels_per_dev"] = 1 + conf["stripe_size"] = "infinity" + conf.set_flash_num_blocks_by_bytes(128 * MB) # let cache just enough to hold one translation page conf.n_cache_entries = conf.n_mapping_entries_per_page objs = create_obj_set(conf) - env = objs['env'] + env = objs["env"] - dftl = FtlTest(objs['conf'], objs['rec'], - objs['flash_controller'], objs['env']) + dftl = FtlTest(objs["conf"], objs["rec"], objs["flash_controller"], objs["env"]) env.process(self.proc_test_write(objs, dftl)) env.run() - def evict(self, env, mappings, lpn0, lpn1, n_evictions): """ lpn0 and lpn1 are in two different m_vpns @@ -1756,21 +1915,20 @@ def evict(self, env, mappings, lpn0, lpn1, n_evictions): lpns = [lpn1, lpn0] * n_evictions lpns = lpns[:n_evictions] - for (i, lpn) in enumerate(lpns): + for i, lpn in enumerate(lpns): # every iteration will evict one translation page yield env.process(mappings.lpn_to_ppn(lpn)) yield env.process(mappings.update(lpn=lpn, ppn=i)) - def proc_test_write(self, objs, dftl): - conf = objs['conf'] - env = objs['env'] - rec = objs['rec'] + conf = objs["conf"] + env = objs["env"] + rec = objs["rec"] rec.enable() - time_read_page = objs['flash_controller'].channels[0].read_time - time_program_page = objs['flash_controller'].channels[0].program_time - time_erase_block = objs['flash_controller'].channels[0].erase_time + time_read_page = objs["flash_controller"].channels[0].read_time + time_program_page = objs["flash_controller"].channels[0].program_time + time_erase_block = objs["flash_controller"].channels[0].erase_time block_pool = dftl.block_pool oob = dftl.oob @@ -1780,39 +1938,44 @@ def proc_test_write(self, objs, dftl): # there should be a current translation block self.assertGreater(len(block_pool.current_blocks()), 0) - victims = wiscsim.dftldes.VictimBlocks(objs['conf'], block_pool, oob) + victims = wiscsim.dftldes.VictimBlocks(objs["conf"], block_pool, oob) transblockcleaner = wiscsim.dftldes.TransBlockCleaner( - conf = objs['conf'], - flash = objs['flash_controller'], - oob = oob, - block_pool = block_pool, - mappings = mappings, - directory = objs['directory'], - rec = objs['rec'], - env = objs['env'], - trans_page_locks = objs['trans_page_locks'] - ) + conf=objs["conf"], + flash=objs["flash_controller"], + oob=oob, + block_pool=block_pool, + mappings=mappings, + directory=objs["directory"], + rec=objs["rec"], + env=objs["env"], + trans_page_locks=objs["trans_page_locks"], + ) k = conf.n_mapping_entries_per_page n = conf.n_pages_per_block n_tp = conf.total_translation_pages() # try to evict many so we have a used translation block - yield env.process(self.evict(env, mappings, 0, k, n/2)) + yield env.process(self.evict(env, mappings, 0, k, old_div(n, 2))) victim_blocks = list(victims.iterator_verbose()) valid_ratio, block_type, victim_block = victim_blocks[0] - self.assertEqual(valid_ratio, - (min(n_tp, conf.n_pages_per_block) - 2.0) / n) + self.assertEqual( + valid_ratio, old_div((min(n_tp, conf.n_pages_per_block) - 2.0), n) + ) s = env.now yield env.process(transblockcleaner.clean(victim_block)) # check the time to move n-1 valid pages - self.assertEqual(env.now, - s + (min(n_tp, conf.n_pages_per_block)-2)*\ - (time_read_page+time_program_page) + time_erase_block) + self.assertEqual( + env.now, + s + + (min(n_tp, conf.n_pages_per_block) - 2) + * (time_read_page + time_program_page) + + time_erase_block, + ) # check validation self.assertEqual(oob.states.block_valid_ratio(victim_block), 0) @@ -1826,8 +1989,10 @@ def proc_test_write(self, objs, dftl): self.assertIn(victim_block, block_pool.freeblocks) # check directory - self.assertNotEqual(conf.page_to_block_off(directory.m_vpn_to_m_ppn(0))[0], - victim_block) + self.assertNotEqual( + conf.page_to_block_off(directory.m_vpn_to_m_ppn(0))[0], victim_block + ) + class TestGlobalGC(unittest.TestCase): def test_used_ratio(self): @@ -1841,85 +2006,84 @@ def test_used_ratio(self): # use one block ppn = block_pool.next_n_data_pages_to_program_striped(1) - self.assertEqual(block_pool.used_ratio(), 1.0/n) + self.assertEqual(block_pool.used_ratio(), 1.0 / n) def test_threshold_sanity(self): conf = create_config() - conf.set_flash_num_blocks_by_bytes(128*MB) + conf.set_flash_num_blocks_by_bytes(128 * MB) conf.GC_high_threshold_ratio = 0.9 conf.GC_low_threshold_ratio = 0.5 conf.over_provisioning = 1.28 objs = create_obj_set(conf) - env = objs['env'] - dftl = FtlTest(objs['conf'], objs['rec'], - objs['flash_controller'], objs['env']) + env = objs["env"] + dftl = FtlTest(objs["conf"], objs["rec"], objs["flash_controller"], objs["env"]) def test_threshold_sanity2(self): conf = create_config() - conf.set_flash_num_blocks_by_bytes(1*MB) + conf.set_flash_num_blocks_by_bytes(1 * MB) conf.GC_high_threshold_ratio = 0.9 conf.GC_low_threshold_ratio = 0.5 conf.over_provisioning = 1.28 objs = create_obj_set(conf) - env = objs['env'] - with self.assertRaisesRegexp(RuntimeError, 'spare blocks'): - dftl = FtlTest(objs['conf'], objs['rec'], - objs['flash_controller'], objs['env']) + env = objs["env"] + with self.assertRaisesRegex(RuntimeError, "spare blocks"): + dftl = FtlTest( + objs["conf"], objs["rec"], objs["flash_controller"], objs["env"] + ) def test_gc_triggering(self): conf = create_config() - conf.set_flash_num_blocks_by_bytes(128*MB) + conf.set_flash_num_blocks_by_bytes(128 * MB) conf.GC_high_threshold_ratio = 0.9 conf.GC_low_threshold_ratio = 0.5 objs = create_obj_set(conf) - env = objs['env'] - dftl = FtlTest(objs['conf'], objs['rec'], - objs['flash_controller'], objs['env']) + env = objs["env"] + dftl = FtlTest(objs["conf"], objs["rec"], objs["flash_controller"], objs["env"]) # cleaner = dftl.get_cleaner() class TestCleaningDataBlocksByCleaner(unittest.TestCase): def test(self): conf = create_config() - conf['flash_config']['n_channels_per_dev'] = 1 - conf['stripe_size'] = 'infinity' - conf.set_flash_num_blocks_by_bytes(128*MB) + conf["flash_config"]["n_channels_per_dev"] = 1 + conf["stripe_size"] = "infinity" + conf.set_flash_num_blocks_by_bytes(128 * MB) conf.GC_low_threshold_ratio = 0 objs = create_obj_set(conf) - env = objs['env'] + env = objs["env"] - dftl = FtlTest(objs['conf'], objs['rec'], - objs['flash_controller'], objs['env']) + dftl = FtlTest(objs["conf"], objs["rec"], objs["flash_controller"], objs["env"]) env.process(self.proc_test_write(objs, dftl)) env.run() def proc_test_write(self, objs, dftl): - conf = objs['conf'] - env = objs['env'] - rec = objs['rec'] + conf = objs["conf"] + env = objs["env"] + rec = objs["rec"] rec.enable() - time_read_page = objs['flash_controller'].channels[0].read_time - time_program_page = objs['flash_controller'].channels[0].program_time - time_erase_block = objs['flash_controller'].channels[0].erase_time + time_read_page = objs["flash_controller"].channels[0].read_time + time_program_page = objs["flash_controller"].channels[0].program_time + time_erase_block = objs["flash_controller"].channels[0].erase_time block_pool = dftl.block_pool oob = dftl.oob mappings = dftl.get_mappings() cleaner = dftl.get_cleaner() - victims = wiscsim.dftldes.VictimBlocks(objs['conf'], block_pool, oob) + victims = wiscsim.dftldes.VictimBlocks(objs["conf"], block_pool, oob) datablockcleaner = wiscsim.dftldes.DataBlockCleaner( - conf = objs['conf'], - flash = objs['flash_controller'], - oob = oob, - block_pool = block_pool, - mappings = mappings, - rec = objs['rec'], - env = objs['env']) + conf=objs["conf"], + flash=objs["flash_controller"], + oob=oob, + block_pool=block_pool, + mappings=mappings, + rec=objs["rec"], + env=objs["env"], + ) n = conf.n_pages_per_block yield env.process(dftl.write_ext(Extent(0, n))) @@ -1931,14 +2095,18 @@ def proc_test_write(self, objs, dftl): victim_blocks = list(victims.iterator_verbose()) valid_ratio, block_type, victim_block = victim_blocks[0] - self.assertEqual(oob.states.block_valid_ratio(victim_block), (n-1.0)/n) + self.assertEqual( + oob.states.block_valid_ratio(victim_block), old_div((n - 1.0), n) + ) s = env.now yield env.process(cleaner.clean()) # check the time to move n-1 valid pages - self.assertEqual(env.now, - s + (n-1)*(time_read_page+time_program_page) + time_erase_block) + self.assertEqual( + env.now, + s + (n - 1) * (time_read_page + time_program_page) + time_erase_block, + ) # check validation self.assertEqual(oob.states.block_valid_ratio(victim_block), 0) @@ -1960,23 +2128,22 @@ def proc_test_write(self, objs, dftl): class TestLevelingWear(unittest.TestCase): def test(self): conf = create_config() - conf['flash_config']['n_channels_per_dev'] = 1 - conf['stripe_size'] = 'infinity' - conf.set_flash_num_blocks_by_bytes(128*MB) + conf["flash_config"]["n_channels_per_dev"] = 1 + conf["stripe_size"] = "infinity" + conf.set_flash_num_blocks_by_bytes(128 * MB) conf.GC_low_threshold_ratio = 0 objs = create_obj_set(conf) - env = objs['env'] + env = objs["env"] - dftl = FtlTest(objs['conf'], objs['rec'], - objs['flash_controller'], objs['env']) + dftl = FtlTest(objs["conf"], objs["rec"], objs["flash_controller"], objs["env"]) env.process(self.proc_test_write(objs, dftl)) env.run() def proc_test_write(self, objs, dftl): - conf = objs['conf'] - env = objs['env'] - rec = objs['rec'] + conf = objs["conf"] + env = objs["env"] + rec = objs["rec"] rec.enable() block_pool = dftl.block_pool @@ -1984,16 +2151,18 @@ def proc_test_write(self, objs, dftl): mappings = dftl.get_mappings() cleaner = dftl.get_cleaner() - victims = wiscsim.dftldes.WearLevelingVictimBlocks(objs['conf'], - block_pool, oob, 1) + victims = wiscsim.dftldes.WearLevelingVictimBlocks( + objs["conf"], block_pool, oob, 1 + ) datablockcleaner = wiscsim.dftldes.DataBlockCleaner( - conf = objs['conf'], - flash = objs['flash_controller'], - oob = oob, - block_pool = block_pool, - mappings = mappings, - rec = objs['rec'], - env = objs['env']) + conf=objs["conf"], + flash=objs["flash_controller"], + oob=oob, + block_pool=block_pool, + mappings=mappings, + rec=objs["rec"], + env=objs["env"], + ) n = conf.n_pages_per_block yield env.process(dftl.write_ext(Extent(0, n))) @@ -2017,7 +2186,7 @@ def proc_test_write(self, objs, dftl): # will immediately become most used after moving data. # ppn_s, ppn_e = conf.block_to_page_range(victim_block) # for ppn in range(ppn_s, ppn_e): - # self.assertEqual(oob.states.is_page_erased(ppn), True) + # self.assertEqual(oob.states.is_page_erased(ppn), True) # check blockpool # self.assertNotIn(victim_block, block_pool.current_blocks()) @@ -2033,17 +2202,16 @@ def proc_test_write(self, objs, dftl): class TestCleaningTransBlocksByCleaner(unittest.TestCase): def test(self): conf = create_config() - conf['flash_config']['n_channels_per_dev'] = 1 - conf['stripe_size'] = 'infinity' - conf.set_flash_num_blocks_by_bytes(128*MB) + conf["flash_config"]["n_channels_per_dev"] = 1 + conf["stripe_size"] = "infinity" + conf.set_flash_num_blocks_by_bytes(128 * MB) conf.GC_low_threshold_ratio = 0 # let cache just enough to hold one translation page conf.n_cache_entries = conf.n_mapping_entries_per_page objs = create_obj_set(conf) - env = objs['env'] + env = objs["env"] - dftl = FtlTest(objs['conf'], objs['rec'], - objs['flash_controller'], objs['env']) + dftl = FtlTest(objs["conf"], objs["rec"], objs["flash_controller"], objs["env"]) env.process(self.proc_test_write(objs, dftl)) env.run() @@ -2058,20 +2226,20 @@ def evict(self, env, mappings, lpn0, lpn1, n_evictions): lpns = [lpn1, lpn0] * n_evictions lpns = lpns[:n_evictions] - for (i, lpn) in enumerate(lpns): + for i, lpn in enumerate(lpns): # every iteration will evict one translation page yield env.process(mappings.lpn_to_ppn(lpn)) yield env.process(mappings.update(lpn=lpn, ppn=i)) def proc_test_write(self, objs, dftl): - conf = objs['conf'] - env = objs['env'] - rec = objs['rec'] + conf = objs["conf"] + env = objs["env"] + rec = objs["rec"] rec.enable() - time_read_page = objs['flash_controller'].channels[0].read_time - time_program_page = objs['flash_controller'].channels[0].program_time - time_erase_block = objs['flash_controller'].channels[0].erase_time + time_read_page = objs["flash_controller"].channels[0].read_time + time_program_page = objs["flash_controller"].channels[0].program_time + time_erase_block = objs["flash_controller"].channels[0].erase_time block_pool = dftl.block_pool oob = dftl.oob @@ -2082,39 +2250,44 @@ def proc_test_write(self, objs, dftl): # there should be a current translation block self.assertGreater(len(block_pool.current_blocks()), 0) - victims = wiscsim.dftldes.VictimBlocks(objs['conf'], block_pool, oob) + victims = wiscsim.dftldes.VictimBlocks(objs["conf"], block_pool, oob) transblockcleaner = wiscsim.dftldes.TransBlockCleaner( - conf = objs['conf'], - flash = objs['flash_controller'], - oob = oob, - block_pool = block_pool, - mappings = mappings, - directory = objs['directory'], - rec = objs['rec'], - env = objs['env'], - trans_page_locks = objs['trans_page_locks'] - ) + conf=objs["conf"], + flash=objs["flash_controller"], + oob=oob, + block_pool=block_pool, + mappings=mappings, + directory=objs["directory"], + rec=objs["rec"], + env=objs["env"], + trans_page_locks=objs["trans_page_locks"], + ) k = conf.n_mapping_entries_per_page n = conf.n_pages_per_block n_tp = conf.total_translation_pages() # try to evict many so we have a used translation block - yield env.process(self.evict(env, mappings, 0, k, n/2)) + yield env.process(self.evict(env, mappings, 0, k, old_div(n, 2))) victim_blocks = list(victims.iterator_verbose()) valid_ratio, block_type, victim_block = victim_blocks[0] - self.assertEqual(valid_ratio, - (min(n_tp, conf.n_pages_per_block) - 2.0) / n) + self.assertEqual( + valid_ratio, old_div((min(n_tp, conf.n_pages_per_block) - 2.0), n) + ) s = env.now yield env.process(cleaner.clean()) # check the time to move n-1 valid pages - self.assertEqual(env.now, - s + (min(n_tp, conf.n_pages_per_block)-2)*\ - (time_read_page+time_program_page)+time_erase_block) + self.assertEqual( + env.now, + s + + (min(n_tp, conf.n_pages_per_block) - 2) + * (time_read_page + time_program_page) + + time_erase_block, + ) # check validation self.assertEqual(oob.states.block_valid_ratio(victim_block), 0) @@ -2128,24 +2301,25 @@ def proc_test_write(self, objs, dftl): self.assertIn(victim_block, block_pool.freeblocks) # check directory - self.assertNotEqual(conf.page_to_block_off(directory.m_vpn_to_m_ppn(0))[0], - victim_block) + self.assertNotEqual( + conf.page_to_block_off(directory.m_vpn_to_m_ppn(0))[0], victim_block + ) + @unittest.skipUnless(TESTALL == True, "Skip unless we want to test all") class TestCleaningTransBlocksByCleaner4Channel(unittest.TestCase): def test(self): conf = create_config() - conf['flash_config']['n_channels_per_dev'] = 4 - conf['stripe_size'] = 'infinity' - conf.set_flash_num_blocks_by_bytes(128*MB) + conf["flash_config"]["n_channels_per_dev"] = 4 + conf["stripe_size"] = "infinity" + conf.set_flash_num_blocks_by_bytes(128 * MB) conf.GC_low_threshold_ratio = 0 # let cache just enough to hold one translation page conf.n_cache_entries = conf.n_mapping_entries_per_page objs = create_obj_set(conf) - env = objs['env'] + env = objs["env"] - dftl = FtlTest(objs['conf'], objs['rec'], - objs['flash_controller'], objs['env']) + dftl = FtlTest(objs["conf"], objs["rec"], objs["flash_controller"], objs["env"]) env.process(self.proc_test_write(objs, dftl)) env.run() @@ -2160,20 +2334,20 @@ def evict(self, env, mappings, lpn0, lpn1, n_evictions): lpns = [lpn1, lpn0] * n_evictions lpns = lpns[:n_evictions] - for (i, lpn) in enumerate(lpns): + for i, lpn in enumerate(lpns): # every iteration will evict one translation page yield env.process(mappings.lpn_to_ppn(lpn)) yield env.process(mappings.update(lpn=lpn, ppn=i)) def proc_test_write(self, objs, dftl): - conf = objs['conf'] - env = objs['env'] - rec = objs['rec'] + conf = objs["conf"] + env = objs["env"] + rec = objs["rec"] rec.enable() - time_read_page = objs['flash_controller'].channels[0].read_time - time_program_page = objs['flash_controller'].channels[0].program_time - time_erase_block = objs['flash_controller'].channels[0].erase_time + time_read_page = objs["flash_controller"].channels[0].read_time + time_program_page = objs["flash_controller"].channels[0].program_time + time_erase_block = objs["flash_controller"].channels[0].erase_time block_pool = dftl.block_pool oob = dftl.oob @@ -2184,39 +2358,39 @@ def proc_test_write(self, objs, dftl): # there should be a current translation block self.assertGreater(len(block_pool.current_blocks()), 0) - victims = wiscsim.dftldes.VictimBlocks(objs['conf'], block_pool, oob) + victims = wiscsim.dftldes.VictimBlocks(objs["conf"], block_pool, oob) transblockcleaner = wiscsim.dftldes.TransBlockCleaner( - conf = objs['conf'], - flash = objs['flash_controller'], - oob = oob, - block_pool = block_pool, - mappings = mappings, - directory = objs['directory'], - rec = objs['rec'], - env = objs['env'], - trans_page_locks = objs['trans_page_locks'] - ) + conf=objs["conf"], + flash=objs["flash_controller"], + oob=oob, + block_pool=block_pool, + mappings=mappings, + directory=objs["directory"], + rec=objs["rec"], + env=objs["env"], + trans_page_locks=objs["trans_page_locks"], + ) k = conf.n_mapping_entries_per_page n = conf.n_pages_per_block n_tp = conf.total_translation_pages() # try to evict many so we have a used translation block - yield env.process(self.evict(env, mappings, 0, k, 4*n)) + yield env.process(self.evict(env, mappings, 0, k, 4 * n)) victim_blocks = list(victims.iterator_verbose()) valid_ratio, block_type, victim_block = victim_blocks[0] - init_n_tp_per_block = n_tp / 4 - self.assertEqual(valid_ratio, (init_n_tp_per_block - 1.0) / n) + init_n_tp_per_block = old_div(n_tp, 4) + self.assertEqual(valid_ratio, old_div((init_n_tp_per_block - 1.0), n)) s = env.now yield env.process(cleaner.clean()) # check the time to move init_n_tp_per_block valid pages # self.assertEqual(env.now, - # s + init_n_tp_per_block*\ - # (time_read_page+time_program_page)+time_erase_block) + # s + init_n_tp_per_block*\ + # (time_read_page+time_program_page)+time_erase_block) # check validation for _, _, victim_blocknum in victim_blocks: @@ -2231,51 +2405,51 @@ def proc_test_write(self, objs, dftl): self.assertIn(victim_blocknum, block_pool.freeblocks) # check directory - self.assertNotEqual(conf.page_to_block_off(directory.m_vpn_to_m_ppn(0))[0], - victim_blocknum) - + self.assertNotEqual( + conf.page_to_block_off(directory.m_vpn_to_m_ppn(0))[0], victim_blocknum + ) class TestCleaning(unittest.TestCase): def test(self): conf = create_config() - conf['flash_config']['n_channels_per_dev'] = 1 - conf['stripe_size'] = 'infinity' - conf.set_flash_num_blocks_by_bytes(128*MB) + conf["flash_config"]["n_channels_per_dev"] = 1 + conf["stripe_size"] = "infinity" + conf.set_flash_num_blocks_by_bytes(128 * MB) conf.GC_low_threshold_ratio = 0 objs = create_obj_set(conf) - env = objs['env'] + env = objs["env"] - dftl = FtlTest(objs['conf'], objs['rec'], - objs['flash_controller'], objs['env']) + dftl = FtlTest(objs["conf"], objs["rec"], objs["flash_controller"], objs["env"]) env.process(self.proc_test_write(objs, dftl)) env.run() def proc_test_write(self, objs, dftl): - conf = objs['conf'] - env = objs['env'] - rec = objs['rec'] + conf = objs["conf"] + env = objs["env"] + rec = objs["rec"] rec.enable() - time_read_page = objs['flash_controller'].channels[0].read_time - time_program_page = objs['flash_controller'].channels[0].program_time - time_erase_block = objs['flash_controller'].channels[0].erase_time + time_read_page = objs["flash_controller"].channels[0].read_time + time_program_page = objs["flash_controller"].channels[0].program_time + time_erase_block = objs["flash_controller"].channels[0].erase_time block_pool = dftl.block_pool oob = dftl.oob mappings = dftl.get_mappings() cleaner = dftl.get_cleaner() - victims = wiscsim.dftldes.VictimBlocks(objs['conf'], block_pool, oob) + victims = wiscsim.dftldes.VictimBlocks(objs["conf"], block_pool, oob) datablockcleaner = wiscsim.dftldes.DataBlockCleaner( - conf = objs['conf'], - flash = objs['flash_controller'], - oob = oob, - block_pool = block_pool, - mappings = mappings, - rec = objs['rec'], - env = objs['env']) + conf=objs["conf"], + flash=objs["flash_controller"], + oob=oob, + block_pool=block_pool, + mappings=mappings, + rec=objs["rec"], + env=objs["env"], + ) n_data_used_blocks = len(block_pool.data_usedblocks) @@ -2293,7 +2467,7 @@ def proc_test_write(self, objs, dftl): # yield env.process(datablockcleaner.clean(victim_block)) yield env.process(cleaner.clean()) # check the time to move 0 valid pages and erase 3 blocks - self.assertEqual(env.now, s + 3*time_erase_block) + self.assertEqual(env.now, s + 3 * time_erase_block) # check validation self.assertEqual(oob.states.block_valid_ratio(victim_block), 0) @@ -2311,38 +2485,38 @@ def proc_test_write(self, objs, dftl): block, _ = conf.page_to_block_off(ppn) self.assertNotEqual(block, victim_block) + class TestCleaning4Channel(unittest.TestCase): def test(self): conf = create_config() - conf['flash_config']['n_channels_per_dev'] = 4 - conf['stripe_size'] = 'infinity' - conf.set_flash_num_blocks_by_bytes(128*MB) + conf["flash_config"]["n_channels_per_dev"] = 4 + conf["stripe_size"] = "infinity" + conf.set_flash_num_blocks_by_bytes(128 * MB) conf.GC_low_threshold_ratio = 0 objs = create_obj_set(conf) - env = objs['env'] + env = objs["env"] - dftl = FtlTest(objs['conf'], objs['rec'], - objs['flash_controller'], objs['env']) + dftl = FtlTest(objs["conf"], objs["rec"], objs["flash_controller"], objs["env"]) env.process(self.proc_test_write(objs, dftl)) env.run() def proc_test_write(self, objs, dftl): - conf = objs['conf'] - env = objs['env'] - rec = objs['rec'] + conf = objs["conf"] + env = objs["env"] + rec = objs["rec"] rec.enable() - time_read_page = objs['flash_controller'].channels[0].read_time - time_program_page = objs['flash_controller'].channels[0].program_time - time_erase_block = objs['flash_controller'].channels[0].erase_time + time_read_page = objs["flash_controller"].channels[0].read_time + time_program_page = objs["flash_controller"].channels[0].program_time + time_erase_block = objs["flash_controller"].channels[0].erase_time block_pool = dftl.block_pool oob = dftl.oob mappings = dftl.get_mappings() cleaner = dftl.get_cleaner() - victims = wiscsim.dftldes.VictimBlocks(objs['conf'], block_pool, oob) + victims = wiscsim.dftldes.VictimBlocks(objs["conf"], block_pool, oob) n_data_used_blocks = len(block_pool.data_usedblocks) @@ -2371,11 +2545,10 @@ def proc_test_write(self, objs, dftl): s = env.now yield env.process(cleaner.clean()) if cleaner.n_cleaners == 1: - self.assertEqual(env.now, s + time_erase_block * 3) #<-------- + self.assertEqual(env.now, s + time_erase_block * 3) # <-------- elif cleaner.n_cleaners >= 4: self.assertEqual(env.now, s + time_erase_block) - # check validation self.assertEqual(oob.states.block_valid_ratio(victim_block), 0) ppn_s, ppn_e = conf.block_to_page_range(victim_block) @@ -2393,34 +2566,37 @@ def proc_test_write(self, objs, dftl): self.assertNotEqual(block, victim_block) -class Experiment(object): +class Experiment: def __init__(self, para): self.para = para + def setup_config(self): self.conf = wiscsim.dftldes.Config() - self.conf['SSDFramework']['ncq_depth'] = 4 + self.conf["SSDFramework"]["ncq_depth"] = 4 - self.conf['flash_config']['n_pages_per_block'] = 2 - self.conf['flash_config']['n_blocks_per_plane'] = 2 - self.conf['flash_config']['n_planes_per_chip'] = 1 - self.conf['flash_config']['n_chips_per_package'] = 1 - self.conf['flash_config']['n_packages_per_channel'] = 1 - self.conf['flash_config']['n_channels_per_dev'] = 1 - self.conf['stripe_size'] = 'infinity' + self.conf["flash_config"]["n_pages_per_block"] = 2 + self.conf["flash_config"]["n_blocks_per_plane"] = 2 + self.conf["flash_config"]["n_planes_per_chip"] = 1 + self.conf["flash_config"]["n_chips_per_package"] = 1 + self.conf["flash_config"]["n_packages_per_channel"] = 1 + self.conf["flash_config"]["n_channels_per_dev"] = 1 + self.conf["stripe_size"] = "infinity" self.conf.GC_high_threshold_ratio = 0.001 self.conf.GC_low_threshold_ratio = 0 - self.conf['do_not_check_gc_setting'] = True - + self.conf["do_not_check_gc_setting"] = True def setup_environment(self): - utils.set_exp_metadata(self.conf, save_data = True, - expname = self.para.expname, - subexpname = utils.chain_items_as_filename(self.para)) + utils.set_exp_metadata( + self.conf, + save_data=True, + expname=self.para.expname, + subexpname=utils.chain_items_as_filename(self.para), + ) - self.conf['enable_blktrace'] = True - self.conf['enable_simulation'] = True + self.conf["enable_blktrace"] = True + self.conf["enable_simulation"] = True def setup_workload(self): w = OP_WRITE @@ -2432,21 +2608,23 @@ def setup_workload(self): self.conf["workload_src"] = LBAGENERATOR self.conf["lba_workload_class"] = "ExtentTestWorkloadFLEX2" self.conf["lba_workload_configs"]["ExtentTestWorkloadFLEX2"] = { - "events": [ - (w, 0, n_pages_per_block), - (w, 0, n_pages_per_block), - (w, 0, n_pages_per_block), - (w, 0, n_pages_per_block) - ]} + "events": [ + (w, 0, n_pages_per_block), + (w, 0, n_pages_per_block), + (w, 0, n_pages_per_block), + (w, 0, n_pages_per_block), + ] + } self.conf["age_workload_class"] = "NoOp" def setup_ftl(self): - self.conf['ftl_type'] = 'dftldes' - self.conf['simulator_class'] = 'SimulatorDESNew' + self.conf["ftl_type"] = "dftldes" + self.conf["simulator_class"] = "SimulatorDESNew" logicsize_mb = 2 - self.conf.mapping_cache_bytes = self.conf.n_mapping_entries_per_page \ - * self.conf['cache_entry_bytes'] # 8 bytes (64bits) needed in mem + self.conf.mapping_cache_bytes = ( + self.conf.n_mapping_entries_per_page * self.conf["cache_entry_bytes"] + ) # 8 bytes (64bits) needed in mem self.conf.set_flash_num_blocks_by_bytes(int(logicsize_mb * 2**20 * 1.28)) def my_run(self): @@ -2460,13 +2638,13 @@ def main(self): self.setup_ftl() self.my_run() + class TestSimpleGC(unittest.TestCase): def test(self): - Parameters = collections.namedtuple("Parameters", - "expname") - expname = 'test' + Parameters = collections.namedtuple("Parameters", "expname") + expname = "test" - exp = Experiment( Parameters(expname = expname) ) + exp = Experiment(Parameters(expname=expname)) exp.main() @@ -2487,42 +2665,44 @@ def test_split_2(self): def test_mapping(self): conf = create_config() - conf['segment_bytes'] = conf.total_flash_bytes() - conf['stripe_size'] = 1 + conf["segment_bytes"] = conf.total_flash_bytes() + conf["stripe_size"] = 1 objs = create_obj_set(conf) - dftl = wiscsim.dftldes.Ftl(objs['conf'], objs['rec'], - objs['flash_controller'], objs['env']) + dftl = wiscsim.dftldes.Ftl( + objs["conf"], objs["rec"], objs["flash_controller"], objs["env"] + ) # all lpn in the same segment mapping = dftl.get_ppns_to_write(Extent(0, 5)) channels = set() - for lpn, ppn in mapping.items(): - channel_id = ppn / conf.n_pages_per_channel + for lpn, ppn in list(mapping.items()): + channel_id = old_div(ppn, conf.n_pages_per_channel) channels.add(channel_id) self.assertEqual(len(channels), 4) def test_mapping_with_segment(self): conf = create_config() - conf['segment_bytes'] = 4 * conf.page_size - conf['stripe_size'] = 1 + conf["segment_bytes"] = 4 * conf.page_size + conf["stripe_size"] = 1 objs = create_obj_set(conf) - dftl = wiscsim.dftldes.Ftl(objs['conf'], objs['rec'], - objs['flash_controller'], objs['env']) + dftl = wiscsim.dftldes.Ftl( + objs["conf"], objs["rec"], objs["flash_controller"], objs["env"] + ) # seg 0 mapping = dftl.get_ppns_to_write(Extent(1, 1)) - ppn0 = mapping.values()[0] - channel_id0 = ppn0 / conf.n_pages_per_channel + ppn0 = list(mapping.values())[0] + channel_id0 = old_div(ppn0, conf.n_pages_per_channel) # seg 1 mapping = dftl.get_ppns_to_write(Extent(7, 1)) mapping = dftl.get_ppns_to_write(Extent(7, 1)) mapping = dftl.get_ppns_to_write(Extent(7, 1)) mapping = dftl.get_ppns_to_write(Extent(7, 1)) - ppn1 = mapping.values()[0] - channel_id1 = ppn1 / conf.n_pages_per_channel + ppn1 = list(mapping.values())[0] + channel_id1 = old_div(ppn1, conf.n_pages_per_channel) self.assertEqual(channel_id0, channel_id1) # it uses another block self.assertEqual(abs(ppn1 - ppn0), conf.n_pages_per_block) @@ -2531,28 +2711,29 @@ def test_mapping_with_segment(self): class TestFTLSegmentedWrite(unittest.TestCase): def test_valid_ratio(self): conf = create_config() - conf['segment_bytes'] = 2 * conf.page_size - conf['flash_config']['n_channels_per_dev'] = 4 - conf['stripe_size'] = 'infinity' - conf.set_flash_num_blocks_by_bytes(128*MB) + conf["segment_bytes"] = 2 * conf.page_size + conf["flash_config"]["n_channels_per_dev"] = 4 + conf["stripe_size"] = "infinity" + conf.set_flash_num_blocks_by_bytes(128 * MB) objs = create_obj_set(conf) - env = objs['env'] + env = objs["env"] - dftl = wiscsim.dftldes.Ftl(objs['conf'], objs['rec'], - objs['flash_controller'], objs['env']) + dftl = wiscsim.dftldes.Ftl( + objs["conf"], objs["rec"], objs["flash_controller"], objs["env"] + ) env.process(self.proc_test_write(objs, dftl)) env.run() def proc_test_write(self, objs, dftl): - env = objs['env'] - rec = objs['rec'] + env = objs["env"] + rec = objs["rec"] rec.enable() block_pool = dftl.block_pool oob = dftl.oob - conf = objs['conf'] + conf = objs["conf"] # should use 5 more blocks n_used = block_pool.total_used_blocks() yield env.process(dftl.write_ext(Extent(0, 9))) @@ -2564,10 +2745,6 @@ def proc_test_write(self, objs, dftl): def main(): unittest.main() -if __name__ == '__main__': - main() - - - - +if __name__ == "__main__": + main() diff --git a/wiscsee/tests/test_dftlext.py b/wiscsee/tests/test_dftlext.py index d84f24f..ffe1817 100644 --- a/wiscsee/tests/test_dftlext.py +++ b/wiscsee/tests/test_dftlext.py @@ -8,7 +8,7 @@ from commons import * -class Experiment(object): +class Experiment: def __init__(self): # Get default setting self.conf = config.Config() @@ -63,11 +63,11 @@ def __init__(self): self.conf = wiscsim.dftlext.Config() def setup_environment(self): - metadata_dic = choose_exp_metadata(self.conf, interactive = False) + metadata_dic = choose_exp_metadata(self.conf, interactive=False) self.conf.update(metadata_dic) - self.conf['enable_blktrace'] = True - self.conf['enable_simulation'] = True + self.conf["enable_blktrace"] = True + self.conf["enable_simulation"] = True def setup_workload(self): self.conf["workload_src"] = config.LBAGENERATOR @@ -75,12 +75,14 @@ def setup_workload(self): self.conf["age_workload_class"] = "NoOp" def setup_ftl(self): - self.conf['ftl_type'] = 'dftlext' - self.conf['simulator_class'] = 'SimulatorNonDESe2e' + self.conf["ftl_type"] = "dftlext" + self.conf["simulator_class"] = "SimulatorNonDESe2e" logicsize_mb = 16 entries_need = int(logicsize_mb * 2**20 * 0.03 / self.conf.page_size) - self.conf.mapping_cache_bytes = int(entries_need * 8) # 8 bytes (64bits) needed in mem + self.conf.mapping_cache_bytes = int( + entries_need * 8 + ) # 8 bytes (64bits) needed in mem self.conf.set_flash_num_blocks_by_bytes(int(logicsize_mb * 2**20 * 1.28)) def run(self): @@ -92,113 +94,127 @@ class DftlextExp2(Experiment): """ This one is for testing the new extent interface """ + def __init__(self): # Get default setting self.conf = wiscsim.dftlext.Config() def setup_environment(self): - metadata_dic = choose_exp_metadata(self.conf, interactive = False) + metadata_dic = choose_exp_metadata(self.conf, interactive=False) self.conf.update(metadata_dic) - self.conf['enable_blktrace'] = True - self.conf['enable_simulation'] = True + self.conf["enable_blktrace"] = True + self.conf["enable_simulation"] = True def setup_workload(self): self.conf["workload_src"] = config.LBAGENERATOR self.conf["lba_workload_class"] = "ExtentTestWorkload" - self.conf["lba_workload_configs"]["ExtentTestWorkload"] = { - "op_count": 100} + self.conf["lba_workload_configs"]["ExtentTestWorkload"] = {"op_count": 100} self.conf["age_workload_class"] = "NoOp" def setup_ftl(self): - self.conf['ftl_type'] = 'dftlext' - self.conf['simulator_class'] = 'SimulatorNonDESe2e' + self.conf["ftl_type"] = "dftlext" + self.conf["simulator_class"] = "SimulatorNonDESe2e" logicsize_mb = 16 entries_need = int(logicsize_mb * 2**20 * 0.03 / self.conf.page_size) - self.conf.mapping_cache_bytes = int(entries_need * 8) # 8 bytes (64bits) needed in mem + self.conf.mapping_cache_bytes = int( + entries_need * 8 + ) # 8 bytes (64bits) needed in mem self.conf.set_flash_num_blocks_by_bytes(int(logicsize_mb * 2**20 * 1.28)) def run(self): runtime_update(self.conf) run_workflow(self.conf) + class DftlextExpE2e(Experiment): """ This one is for testing the new extent interface with e2e data test """ + def __init__(self): # Get default setting self.conf = wiscsim.dftlext.Config() def setup_environment(self): - metadata_dic = choose_exp_metadata(self.conf, interactive = False) + metadata_dic = choose_exp_metadata(self.conf, interactive=False) self.conf.update(metadata_dic) - self.conf['enable_blktrace'] = True - self.conf['enable_simulation'] = True + self.conf["enable_blktrace"] = True + self.conf["enable_simulation"] = True def setup_workload(self): self.conf["workload_src"] = config.LBAGENERATOR self.conf["lba_workload_class"] = "ExtentTestWorkload" - self.conf["lba_workload_configs"]["ExtentTestWorkload"] = { - "op_count": 1000} + self.conf["lba_workload_configs"]["ExtentTestWorkload"] = {"op_count": 1000} self.conf["age_workload_class"] = "NoOp" def setup_ftl(self): - self.conf['ftl_type'] = 'dftlext' - self.conf['simulator_class'] = 'SimulatorNonDESe2e' + self.conf["ftl_type"] = "dftlext" + self.conf["simulator_class"] = "SimulatorNonDESe2e" logicsize_mb = 16 - entries_need = int(logicsize_mb * 2**20 * 0.03 / self.conf['flash_config']['page_size']) - self.conf.mapping_cache_bytes = int(entries_need * 8) # 8 bytes (64bits) needed in mem + entries_need = int( + logicsize_mb * 2**20 * 0.03 / self.conf["flash_config"]["page_size"] + ) + self.conf.mapping_cache_bytes = int( + entries_need * 8 + ) # 8 bytes (64bits) needed in mem self.conf.set_flash_num_blocks_by_bytes(int(logicsize_mb * 2**20 * 1.28)) def run(self): runtime_update(self.conf) run_workflow(self.conf) + class DftlextTest(unittest.TestCase): def test_Dftl(self): exp = DftlextExp() exp.main() + class DftlextTest2(unittest.TestCase): def test_extent(self): exp = DftlextExp2() exp.main() + class DftlextTest3(unittest.TestCase): def test_extent_e2e(self): exp = DftlextExpE2e() exp.main() + class TestDftextGC(unittest.TestCase): def setup_config(self): self.conf = wiscsim.dftlext.Config() self.conf.n_channels_per_dev = 4 def setup_environment(self): - metadata_dic = choose_exp_metadata(self.conf, interactive = False) + metadata_dic = choose_exp_metadata(self.conf, interactive=False) self.conf.update(metadata_dic) - self.conf['enable_blktrace'] = True - self.conf['enable_simulation'] = True + self.conf["enable_blktrace"] = True + self.conf["enable_simulation"] = True def setup_workload(self): self.conf["workload_src"] = config.LBAGENERATOR self.conf["lba_workload_class"] = "ExtentTestWorkload" - self.conf["lba_workload_configs"]["ExtentTestWorkload"] = { - "op_count": 1000} + self.conf["lba_workload_configs"]["ExtentTestWorkload"] = {"op_count": 1000} self.conf["age_workload_class"] = "NoOp" def setup_ftl(self): - self.conf['ftl_type'] = 'dftlext' - self.conf['simulator_class'] = 'SimulatorNonDESe2e' + self.conf["ftl_type"] = "dftlext" + self.conf["simulator_class"] = "SimulatorNonDESe2e" logicsize_mb = 16 - entries_need = int(logicsize_mb * 2**20 * 0.03 / self.conf['flash_config']['page_size']) - self.conf.mapping_cache_bytes = int(entries_need * 8) # 8 bytes (64bits) needed in mem + entries_need = int( + logicsize_mb * 2**20 * 0.03 / self.conf["flash_config"]["page_size"] + ) + self.conf.mapping_cache_bytes = int( + entries_need * 8 + ) # 8 bytes (64bits) needed in mem self.conf.set_flash_num_blocks_by_bytes(int(logicsize_mb * 2**20 * 1.28)) def my_run(self): @@ -217,29 +233,32 @@ class TestDftextGCSingleChannel(unittest.TestCase): def setup_config(self): self.conf = wiscsim.dftlext.Config() - print '1', self.conf.n_blocks_per_dev + print(("1", self.conf.n_blocks_per_dev)) def setup_environment(self): - metadata_dic = choose_exp_metadata(self.conf, interactive = False) + metadata_dic = choose_exp_metadata(self.conf, interactive=False) self.conf.update(metadata_dic) - self.conf['enable_blktrace'] = True - self.conf['enable_simulation'] = True + self.conf["enable_blktrace"] = True + self.conf["enable_simulation"] = True def setup_workload(self): self.conf["workload_src"] = config.LBAGENERATOR self.conf["lba_workload_class"] = "ExtentTestWorkload" - self.conf["lba_workload_configs"]["ExtentTestWorkload"] = { - "op_count": 1000} + self.conf["lba_workload_configs"]["ExtentTestWorkload"] = {"op_count": 1000} self.conf["age_workload_class"] = "NoOp" def setup_ftl(self): - self.conf['ftl_type'] = 'dftlext' - self.conf['simulator_class'] = 'SimulatorNonDESe2e' + self.conf["ftl_type"] = "dftlext" + self.conf["simulator_class"] = "SimulatorNonDESe2e" logicsize_mb = 1 - entries_need = int(logicsize_mb * 2**20 * 0.03 / self.conf['flash_config']['page_size']) - self.conf.mapping_cache_bytes = int(entries_need * 8) # 8 bytes (64bits) needed in mem + entries_need = int( + logicsize_mb * 2**20 * 0.03 / self.conf["flash_config"]["page_size"] + ) + self.conf.mapping_cache_bytes = int( + entries_need * 8 + ) # 8 bytes (64bits) needed in mem self.conf.set_flash_num_blocks_by_bytes(int(logicsize_mb * 2**20 * 1.28)) def my_run(self): @@ -271,12 +290,15 @@ def my_run(self): tl = wiscsim.dftlext.Timeline(self.conf) tl.turn_on() tl.add_logical_op(0, 100, wiscsim.dftlext.LOGICAL_READ) - tl.incr_time_stamp('flash.read', 3) - self.assertEqual(tl.table[-1]['end_timestamp'], - tl.conf['flash_config']['page_read_time'] * 3) + tl.incr_time_stamp("flash.read", 3) + self.assertEqual( + tl.table[-1]["end_timestamp"], tl.conf["flash_config"]["page_read_time"] * 3 + ) tl.add_logical_op(200, 100, wiscsim.dftlext.LOGICAL_READ) - self.assertEqual(tl.table[-1]['start_timestamp'], - tl.conf['flash_config']['page_read_time'] * 3) + self.assertEqual( + tl.table[-1]["start_timestamp"], + tl.conf["flash_config"]["page_read_time"] * 3, + ) def test_main(self): self.setup_config() @@ -290,15 +312,15 @@ class TestDftlextParallelFlash(unittest.TestCase): def setup_config(self): self.conf = wiscsim.dftlext.Config() # 2 pages per block, 2 blocks per channel, 2 channels in total - self.conf['flash_config']['n_pages_per_block'] = 2 - self.conf['flash_config']['n_blocks_per_plane'] = 2 - self.conf['flash_config']['n_planes_per_chip'] = 1 - self.conf['flash_config']['n_chips_per_package'] = 1 - self.conf['flash_config']['n_packages_per_channel'] = 1 - self.conf['flash_config']['n_channels_per_dev'] = 2 + self.conf["flash_config"]["n_pages_per_block"] = 2 + self.conf["flash_config"]["n_blocks_per_plane"] = 2 + self.conf["flash_config"]["n_planes_per_chip"] = 1 + self.conf["flash_config"]["n_chips_per_package"] = 1 + self.conf["flash_config"]["n_packages_per_channel"] = 1 + self.conf["flash_config"]["n_channels_per_dev"] = 2 def setup_environment(self): - metadata_dic = choose_exp_metadata(self.conf, interactive = False) + metadata_dic = choose_exp_metadata(self.conf, interactive=False) self.conf.update(metadata_dic) def setup_workload(self): @@ -309,28 +331,30 @@ def setup_ftl(self): def my_run(self): runtime_update(self.conf) - rec = wiscsim.recorder.Recorder(output_target = self.conf['output_target'], - output_directory = self.conf['result_dir'], - verbose_level = self.conf['verbose_level'], - print_when_finished = self.conf['print_when_finished'] - ) + rec = wiscsim.recorder.Recorder( + output_target=self.conf["output_target"], + output_directory=self.conf["result_dir"], + verbose_level=self.conf["verbose_level"], + print_when_finished=self.conf["print_when_finished"], + ) rec.disable() - fc = wiscsim.dftlext.ParallelFlash(self.conf, rec, - wiscsim.dftlext.GlobalHelper(self.conf)) + fc = wiscsim.dftlext.ParallelFlash( + self.conf, rec, wiscsim.dftlext.GlobalHelper(self.conf) + ) - self.assertEqual(fc.get_max_channel_page_count(ppns = [0]), 1) - self.assertEqual(fc.get_max_channel_page_count(ppns = [0, 1, 2, 3]), 4) - self.assertEqual(fc.get_max_channel_page_count(ppns = [0, 1, 2, 3, 4]), 4) + self.assertEqual(fc.get_max_channel_page_count(ppns=[0]), 1) + self.assertEqual(fc.get_max_channel_page_count(ppns=[0, 1, 2, 3]), 4) + self.assertEqual(fc.get_max_channel_page_count(ppns=[0, 1, 2, 3, 4]), 4) ppns = [0, 1, 99] data = [100 * ppn for ppn in ppns] - fc.write_pages(ppns, data, tag = None) - data_read = fc.read_pages(ppns, tag = None) + fc.write_pages(ppns, data, tag=None) + data_read = fc.read_pages(ppns, tag=None) self.assertListEqual(data, data_read) - fc.erase_blocks([0], tag = None) - data_read = fc.read_pages(ppns, tag = None) + fc.erase_blocks([0], tag=None) + data_read = fc.read_pages(ppns, tag=None) self.assertListEqual(data_read, [None, None, 9900]) def test_main(self): @@ -345,61 +369,65 @@ class TestTimelineAndFlash(unittest.TestCase): def setup_config(self): self.conf = wiscsim.dftlext.Config() # 2 pages per block, 2 blocks per channel, 2 channels in total - self.conf['sector_size'] = self.conf['flash_config']['page_size'] - self.conf['flash_config']['n_pages_per_block'] = 2 - self.conf['flash_config']['n_blocks_per_plane'] = 2 - self.conf['flash_config']['n_planes_per_chip'] = 1 - self.conf['flash_config']['n_chips_per_package'] = 1 - self.conf['flash_config']['n_packages_per_channel'] = 1 - self.conf['flash_config']['n_channels_per_dev'] = 2 - - self.conf['flash_config']['page_read_time'] = 1 - self.conf['flash_config']['page_prog_time'] = 1 - self.conf['flash_config']['block_erase_time'] = 1 + self.conf["sector_size"] = self.conf["flash_config"]["page_size"] + self.conf["flash_config"]["n_pages_per_block"] = 2 + self.conf["flash_config"]["n_blocks_per_plane"] = 2 + self.conf["flash_config"]["n_planes_per_chip"] = 1 + self.conf["flash_config"]["n_chips_per_package"] = 1 + self.conf["flash_config"]["n_packages_per_channel"] = 1 + self.conf["flash_config"]["n_channels_per_dev"] = 2 + + self.conf["flash_config"]["page_read_time"] = 1 + self.conf["flash_config"]["page_prog_time"] = 1 + self.conf["flash_config"]["block_erase_time"] = 1 def setup_environment(self): - metadata_dic = choose_exp_metadata(self.conf, interactive = False) + metadata_dic = choose_exp_metadata(self.conf, interactive=False) self.conf.update(metadata_dic) def setup_workload(self): pass def setup_ftl(self): - self.conf['ftl_type'] = 'dftlext' - self.conf['simulator_class'] = 'SimulatorNonDESe2e' + self.conf["ftl_type"] = "dftlext" + self.conf["simulator_class"] = "SimulatorNonDESe2e" logicsize_mb = 16 - entries_need = int(logicsize_mb * 2**20 * 0.03 / \ - self.conf['flash_config']['page_size']) - self.conf.mapping_cache_bytes = int(entries_need * 8) # 8 bytes (64bits) needed in mem + entries_need = int( + logicsize_mb * 2**20 * 0.03 / self.conf["flash_config"]["page_size"] + ) + self.conf.mapping_cache_bytes = int( + entries_need * 8 + ) # 8 bytes (64bits) needed in mem self.conf.set_flash_num_blocks_by_bytes(int(logicsize_mb * 2**20 * 1.28)) runtime_update(self.conf) self.rec = wiscsim.recorder.Recorder( - output_target = self.conf['output_target'], - output_directory = self.conf['result_dir'], - verbose_level = self.conf['verbose_level'], - print_when_finished = self.conf['print_when_finished'] - ) - - self.ftl = wiscsim.dftlext.Dftl(self.conf, self.rec, - wiscsim.flash.Flash(recorder = self.rec, confobj = self.conf)) + output_target=self.conf["output_target"], + output_directory=self.conf["result_dir"], + verbose_level=self.conf["verbose_level"], + print_when_finished=self.conf["print_when_finished"], + ) + + self.ftl = wiscsim.dftlext.Dftl( + self.conf, + self.rec, + wiscsim.flash.Flash(recorder=self.rec, confobj=self.conf), + ) def my_run(self): self.ftl.global_helper.timeline.turn_on() n = 1 sectors = list(range(n)) data = [wiscsim.simulator.random_data(sec) for sec in sectors] - self.ftl.sec_write(0, n, data = data) + self.ftl.sec_write(0, n, data=data) # a write involve a tranlation page read and data page write - self.assertEqual(self.ftl.global_helper.timeline.table[-1]\ - ['end_timestamp'], 2) + self.assertEqual(self.ftl.global_helper.timeline.table[-1]["end_timestamp"], 2) self.ftl.sec_read(0, n) # a read involve a tranlation page read and data page write - self.assertEqual(self.ftl.global_helper.timeline.table[-1]\ - ['end_timestamp'], 3) + self.assertEqual(self.ftl.global_helper.timeline.table[-1]["end_timestamp"], 3) def test_main(self): self.setup_config() @@ -412,10 +440,20 @@ def test_main(self): class TestEventIter(unittest.TestCase): def test_main2(self): conf = config.ConfigNewFlash() - conf["event_file_column_names"] = ['pid', 'operation', 'offset', 'size', - 'timestamp', 'pre_wait_time', 'sync'] - events = list(wiscsim.hostevent.EventIterator(conf, - ["1123 write 0 4096 0 0 S", "13 write 40960 4096 1 1 S"])) + conf["event_file_column_names"] = [ + "pid", + "operation", + "offset", + "size", + "timestamp", + "pre_wait_time", + "sync", + ] + events = list( + wiscsim.hostevent.EventIterator( + conf, ["1123 write 0 4096 0 0 S", "13 write 40960 4096 1 1 S"] + ) + ) e = events[0] self.assertEqual(e.pid, 1123) self.assertEqual(e.operation, OP_WRITE) @@ -426,6 +464,6 @@ def test_main2(self): def main(): unittest.main() -if __name__ == '__main__': - main() +if __name__ == "__main__": + main() diff --git a/wiscsee/tests/test_nkftl.py b/wiscsee/tests/test_nkftl.py index 604bcac..8acdd50 100644 --- a/wiscsee/tests/test_nkftl.py +++ b/wiscsee/tests/test_nkftl.py @@ -1,3 +1,7 @@ +from future import standard_library + +standard_library.install_aliases() +from past.utils import old_div import unittest import random import simpy @@ -14,28 +18,29 @@ from config import LBAGENERATOR from wiscsim.ftlsim_commons import * -TDATA = 'TDATA' -TLOG = 'TLOG' +TDATA = "TDATA" +TLOG = "TLOG" + def create_config(): conf = wiscsim.nkftl2.Config() - conf['flash_config']['n_pages_per_block'] = 8 - conf['flash_config']['n_blocks_per_plane'] = 2 - conf['flash_config']['n_planes_per_chip'] = 1 - conf['flash_config']['n_chips_per_package'] = 1 - conf['flash_config']['n_packages_per_channel'] = 1 - conf['flash_config']['n_channels_per_dev'] = 4 + conf["flash_config"]["n_pages_per_block"] = 8 + conf["flash_config"]["n_blocks_per_plane"] = 2 + conf["flash_config"]["n_planes_per_chip"] = 1 + conf["flash_config"]["n_chips_per_package"] = 1 + conf["flash_config"]["n_packages_per_channel"] = 1 + conf["flash_config"]["n_channels_per_dev"] = 4 - conf['nkftl']['max_blocks_in_log_group'] = 2 - conf['nkftl']['n_blocks_in_data_group'] = 4 + conf["nkftl"]["max_blocks_in_log_group"] = 2 + conf["nkftl"]["n_blocks_in_data_group"] = 4 - conf['nkftl']['GC_threshold_ratio'] = 0.8 - conf['nkftl']['GC_low_threshold_ratio'] = 0.3 + conf["nkftl"]["GC_threshold_ratio"] = 0.8 + conf["nkftl"]["GC_low_threshold_ratio"] = 0.3 - utils.set_exp_metadata(conf, save_data = False, - expname = 'test_expname', - subexpname = 'test_subexpname') + utils.set_exp_metadata( + conf, save_data=False, expname="test_expname", subexpname="test_subexpname" + ) logicsize_mb = 64 conf.set_flash_num_blocks_by_bytes(int(logicsize_mb * 2**20 * 1.28)) @@ -44,22 +49,23 @@ def create_config(): return conf + def create_config_1_channel(): conf = wiscsim.nkftl2.Config() - conf['flash_config']['n_pages_per_block'] = 32 - conf['flash_config']['n_blocks_per_plane'] = 64 - conf['flash_config']['n_planes_per_chip'] = 1 - conf['flash_config']['n_chips_per_package'] = 1 - conf['flash_config']['n_packages_per_channel'] = 1 - conf['flash_config']['n_channels_per_dev'] = 1 + conf["flash_config"]["n_pages_per_block"] = 32 + conf["flash_config"]["n_blocks_per_plane"] = 64 + conf["flash_config"]["n_planes_per_chip"] = 1 + conf["flash_config"]["n_chips_per_package"] = 1 + conf["flash_config"]["n_packages_per_channel"] = 1 + conf["flash_config"]["n_channels_per_dev"] = 1 - conf['nkftl']['max_blocks_in_log_group'] = 16 - conf['nkftl']['n_blocks_in_data_group'] = 4 + conf["nkftl"]["max_blocks_in_log_group"] = 16 + conf["nkftl"]["n_blocks_in_data_group"] = 4 - utils.set_exp_metadata(conf, save_data = False, - expname = 'test_expname', - subexpname = 'test_subexpname') + utils.set_exp_metadata( + conf, save_data=False, expname="test_expname", subexpname="test_subexpname" + ) utils.runtime_update(conf) @@ -67,53 +73,60 @@ def create_config_1_channel(): def create_recorder(conf): - rec = wiscsim.recorder.Recorder(output_target = conf['output_target'], - output_directory = conf['result_dir'], - verbose_level = conf['verbose_level'], - print_when_finished = conf['print_when_finished'] - ) + rec = wiscsim.recorder.Recorder( + output_target=conf["output_target"], + output_directory=conf["result_dir"], + verbose_level=conf["verbose_level"], + print_when_finished=conf["print_when_finished"], + ) rec.disable() return rec def create_nkblockpool(conf): block_pool = NKBlockPool( - n_channels=conf.n_channels_per_dev, - n_blocks_per_channel=conf.n_blocks_per_channel, - n_pages_per_block=conf.n_pages_per_block, - tags=[TDATA, TLOG]) + n_channels=conf.n_channels_per_dev, + n_blocks_per_channel=conf.n_blocks_per_channel, + n_pages_per_block=conf.n_pages_per_block, + tags=[TDATA, TLOG], + ) return block_pool + def create_env(): env = simpy.Environment() return env + def create_flash_controller(env, conf, rec): - flash_controller = wiscsim.controller.Controller3( - env, conf, rec) + flash_controller = wiscsim.controller.Controller3(env, conf, rec) return flash_controller + def create_nkftl(): conf = create_config() rec = create_recorder(conf) env = create_env() des_flash = create_flash_controller(env, conf, rec) - ftl = Ftl(conf, rec, - wiscsim.flash.Flash(recorder=rec, confobj=conf), env, - des_flash) + ftl = Ftl( + conf, rec, wiscsim.flash.Flash(recorder=rec, confobj=conf), env, des_flash + ) return ftl, conf, rec, env + def create_global_helper(conf): return GlobalHelper(conf) + def create_translator(conf, rec, globalhelper, log_mapping, data_block_mapping): return Translator(conf, rec, globalhelper, log_mapping, data_block_mapping) + def create_gc(): conf = create_config() - conf['nkftl']['max_blocks_in_log_group'] = 4 - conf['nkftl']['n_blocks_in_data_group'] = 4 + conf["nkftl"]["max_blocks_in_log_group"] = 4 + conf["nkftl"]["n_blocks_in_data_group"] = 4 block_pool = create_nkblockpool(conf) rec = create_recorder(conf) oob = OutOfBandAreas(conf) @@ -126,23 +139,47 @@ def create_gc(): des_flash = create_flash_controller(simpy_env, conf, rec) logical_block_locks = LockPool(simpy_env) - gc = GarbageCollector(conf, block_pool, flashobj, oob, rec, - translator, helper, logmaptable, datablocktable, simpy_env, - des_flash, logical_block_locks) - - GCPack = namedtuple('GCPack', 'gc, conf, block_pool, rec, oob, helper,' \ - 'logmaptable, datablocktable, translator, flashobj, simpy_env, des_flash') - - return GCPack(gc=gc, conf=conf, block_pool=block_pool, rec=rec, oob=oob, helper=helper, - logmaptable=logmaptable, datablocktable=datablocktable, - translator=translator, flashobj=flashobj, simpy_env=simpy_env, - des_flash=des_flash) + gc = GarbageCollector( + conf, + block_pool, + flashobj, + oob, + rec, + translator, + helper, + logmaptable, + datablocktable, + simpy_env, + des_flash, + logical_block_locks, + ) + + GCPack = namedtuple( + "GCPack", + "gc, conf, block_pool, rec, oob, helper," + "logmaptable, datablocktable, translator, flashobj, simpy_env, des_flash", + ) + + return GCPack( + gc=gc, + conf=conf, + block_pool=block_pool, + rec=rec, + oob=oob, + helper=helper, + logmaptable=logmaptable, + datablocktable=datablocktable, + translator=translator, + flashobj=flashobj, + simpy_env=simpy_env, + des_flash=des_flash, + ) def random_data_of_extent(extent): data = [] for lpn in extent.lpn_iter(): - d = str(lpn) + '.' + str(random.randint(0, 100)) + d = str(lpn) + "." + str(random.randint(0, 100)) data.append(d) return data @@ -150,24 +187,24 @@ def random_data_of_extent(extent): class AssertFinishTestCase(unittest.TestCase): def setUp(self): self.finished = False - print 'setting up' + print("setting up") def set_finished(self): - print 'setting finsihed' + print("setting finsihed") self.finished = True def tearDown(self): self.assertTrue(self.finished) - print 'asserted finished' + print("asserted finished") -class RWMixin(object): +class RWMixin: def write_proc(self, env, ftl, extent, data=None): yield env.process(ftl.write_ext(extent, data)) def read_proc(self, env, ftl, extent): data = yield env.process(ftl.read_ext(extent)) - env.exit(data) + return data def data_of_extent(self, extent): d = [] @@ -191,12 +228,11 @@ def test_write_and_read(self): env.run() def proc_test_write_and_read(self, env, ftl, conf): - yield env.process(ftl.lba_write(8, data='3')) - ret = yield env.process( ftl.lba_read(8) ) - self.assertEqual(ret, '3') + yield env.process(ftl.lba_write(8, data="3")) + ret = yield env.process(ftl.lba_read(8)) + self.assertEqual(ret, "3") self.set_finished() - def write_and_check(self, ftl, lpns, env): data_mirror = {} for lpn in lpns: @@ -204,7 +240,7 @@ def write_and_check(self, ftl, lpns, env): data_mirror[lpn] = data yield env.process(ftl.lba_write(lpn, data)) - for lpn, data in data_mirror.items(): + for lpn, data in list(data_mirror.items()): ret = yield env.process(ftl.lba_read(lpn)) self.assertEqual(ret, data) self.set_finished() @@ -213,7 +249,7 @@ def test_data_integrity(self): ftl, conf, rec, env = create_nkftl() total_pages = conf.total_num_pages() - lpns = random.sample(range(total_pages), 1000) + lpns = random.sample(list(range(total_pages)), 1000) env.process(self.write_and_check(ftl, lpns, env)) env.run() @@ -221,7 +257,9 @@ def test_data_integrity(self): def test_GC_simple(self): ftl, conf, rec, env = create_nkftl() - lpns = [0] * 4 * conf.n_pages_per_block * conf['nkftl']['max_blocks_in_log_group'] + lpns = ( + [0] * 4 * conf.n_pages_per_block * conf["nkftl"]["max_blocks_in_log_group"] + ) env.process(self.write_and_check(ftl, lpns, env)) env.run() @@ -229,7 +267,12 @@ def test_GC_simple(self): def test_GC_harder(self): ftl, conf, rec, env = create_nkftl() - lpns = [0, 3, 1] * 4 * conf.n_pages_per_block * conf['nkftl']['max_blocks_in_log_group'] + lpns = ( + [0, 3, 1] + * 4 + * conf.n_pages_per_block + * conf["nkftl"]["max_blocks_in_log_group"] + ) env.process(self.write_and_check(ftl, lpns, env)) env.run() @@ -237,7 +280,12 @@ def test_GC_harder(self): def test_GC_harder2(self): ftl, conf, rec, env = create_nkftl() - lpns = [0, 128, 3, 129, 1] * 4 * conf.n_pages_per_block * conf['nkftl']['max_blocks_in_log_group'] + lpns = ( + [0, 128, 3, 129, 1] + * 4 + * conf.n_pages_per_block + * conf["nkftl"]["max_blocks_in_log_group"] + ) env.process(self.write_and_check(ftl, lpns, env)) env.run() @@ -246,7 +294,7 @@ def test_GC_harder2(self): def test_GC_harder_super(self): ftl, conf, rec, env = create_nkftl() - print 'total pages', conf.total_num_pages() + print(("total pages", conf.total_num_pages())) lpns = [0, 128, 3, 129, 1] * 4 * conf.total_num_pages() env.process(self.write_and_check(ftl, lpns, env)) @@ -257,10 +305,11 @@ class TestBlockPool(unittest.TestCase): def test_init(self): conf = create_config() block_pool = NKBlockPool( - n_channels=conf.n_channels_per_dev, - n_blocks_per_channel=conf.n_blocks_per_channel, - n_pages_per_block=conf.n_pages_per_block, - tags=[TDATA, TLOG]) + n_channels=conf.n_channels_per_dev, + n_blocks_per_channel=conf.n_blocks_per_channel, + n_pages_per_block=conf.n_pages_per_block, + tags=[TDATA, TLOG], + ) self.assertEqual(block_pool.used_ratio(), 0) self.assertEqual(block_pool.total_used_blocks(), 0) @@ -268,10 +317,11 @@ def test_init(self): def test_log_blocks(self): conf = create_config() block_pool = NKBlockPool( - n_channels=conf.n_channels_per_dev, - n_blocks_per_channel=conf.n_blocks_per_channel, - n_pages_per_block=conf.n_pages_per_block, - tags=[TDATA, TLOG]) + n_channels=conf.n_channels_per_dev, + n_blocks_per_channel=conf.n_blocks_per_channel, + n_pages_per_block=conf.n_pages_per_block, + tags=[TDATA, TLOG], + ) blocknum = block_pool.pop_a_free_block_to_log_blocks() self.assertIn(blocknum, block_pool.log_usedblocks) @@ -288,10 +338,11 @@ def test_log_blocks(self): def test_data_blocks(self): conf = create_config() block_pool = NKBlockPool( - n_channels=conf.n_channels_per_dev, - n_blocks_per_channel=conf.n_blocks_per_channel, - n_pages_per_block=conf.n_pages_per_block, - tags=[TDATA, TLOG]) + n_channels=conf.n_channels_per_dev, + n_blocks_per_channel=conf.n_blocks_per_channel, + n_pages_per_block=conf.n_pages_per_block, + tags=[TDATA, TLOG], + ) blocknum = block_pool.pop_a_free_block_to_data_blocks() self.assertIn(blocknum, block_pool.data_usedblocks) @@ -304,10 +355,11 @@ def test_data_blocks(self): def test_data_blocks_overflow(self): conf = create_config() block_pool = NKBlockPool( - n_channels=conf.n_channels_per_dev, - n_blocks_per_channel=conf.n_blocks_per_channel, - n_pages_per_block=conf.n_pages_per_block, - tags=[TDATA, TLOG]) + n_channels=conf.n_channels_per_dev, + n_blocks_per_channel=conf.n_blocks_per_channel, + n_pages_per_block=conf.n_pages_per_block, + tags=[TDATA, TLOG], + ) n = conf.n_blocks_per_dev for i in range(n): @@ -320,10 +372,11 @@ def test_data_blocks_overflow(self): def test_free_used_log(self): conf = create_config() block_pool = NKBlockPool( - n_channels=conf.n_channels_per_dev, - n_blocks_per_channel=conf.n_blocks_per_channel, - n_pages_per_block=conf.n_pages_per_block, - tags=[TDATA, TLOG]) + n_channels=conf.n_channels_per_dev, + n_blocks_per_channel=conf.n_blocks_per_channel, + n_pages_per_block=conf.n_pages_per_block, + tags=[TDATA, TLOG], + ) blocknum = block_pool.pop_a_free_block_to_log_blocks() block_pool.free_used_log_block(blocknum) @@ -333,32 +386,49 @@ def test_free_used_log(self): def test_freeblocks(self): conf = create_config() block_pool = NKBlockPool( - n_channels=conf.n_channels_per_dev, - n_blocks_per_channel=conf.n_blocks_per_channel, - n_pages_per_block=conf.n_pages_per_block, - tags=[TDATA, TLOG]) + n_channels=conf.n_channels_per_dev, + n_blocks_per_channel=conf.n_blocks_per_channel, + n_pages_per_block=conf.n_pages_per_block, + tags=[TDATA, TLOG], + ) self.assertEqual(len(block_pool.freeblocks), conf.n_blocks_per_dev) class TestBlockInfo(unittest.TestCase): def test_init(self): - blkinfo = BlockInfo(block_type=TYPE_DATA_BLOCK, - block_num=23, last_used_time=88, valid_ratio=0.8, - data_group_no=8) + blkinfo = BlockInfo( + block_type=TYPE_DATA_BLOCK, + block_num=23, + last_used_time=88, + valid_ratio=0.8, + data_group_no=8, + ) def test_comp(self): - blkinfo1 = BlockInfo(block_type=TYPE_DATA_BLOCK, - block_num=23, last_used_time=88, valid_ratio=0.8, - data_group_no=8) + blkinfo1 = BlockInfo( + block_type=TYPE_DATA_BLOCK, + block_num=23, + last_used_time=88, + valid_ratio=0.8, + data_group_no=8, + ) - blkinfo2 = BlockInfo(block_type=TYPE_DATA_BLOCK, - block_num=23, last_used_time=89, valid_ratio=0.8, - data_group_no=8) + blkinfo2 = BlockInfo( + block_type=TYPE_DATA_BLOCK, + block_num=23, + last_used_time=89, + valid_ratio=0.8, + data_group_no=8, + ) - blkinfo3 = BlockInfo(block_type=TYPE_DATA_BLOCK, - block_num=23, last_used_time=87, valid_ratio=0.8, - data_group_no=8) + blkinfo3 = BlockInfo( + block_type=TYPE_DATA_BLOCK, + block_num=23, + last_used_time=87, + valid_ratio=0.8, + data_group_no=8, + ) self.assertTrue(blkinfo1 < blkinfo2) self.assertTrue(blkinfo3 < blkinfo1) @@ -367,17 +437,21 @@ def test_priority_queue(self): priority_q = Queue.PriorityQueue() for i in range(10): - blkinfo = BlockInfo(block_type=TYPE_DATA_BLOCK, - block_num=23, last_used_time=i, valid_ratio=0.8, - data_group_no=8) + blkinfo = BlockInfo( + block_type=TYPE_DATA_BLOCK, + block_num=23, + last_used_time=i, + valid_ratio=0.8, + data_group_no=8, + ) priority_q.put(blkinfo) used_times = [] while not priority_q.empty(): - time = priority_q.get().last_used_time - used_times.append( time ) + time = priority_q.get().last_used_time + used_times.append(time) - self.assertListEqual(used_times, range(10)) + self.assertListEqual(used_times, list(range(10))) class TestOutOfBandAreas(unittest.TestCase): @@ -419,7 +493,7 @@ def test_erase_block(self): n_pages_per_block = conf.n_pages_per_block - ppns = range(1*n_pages_per_block, 1*n_pages_per_block+3) + ppns = list(range(1 * n_pages_per_block, 1 * n_pages_per_block + 3)) for lpn, ppn in zip([3, 88, 23], ppns): oob.remap(lpn=lpn, old_ppn=None, new_ppn=ppn) @@ -436,11 +510,11 @@ def test_lpns_of_block(self): n_pages_per_block = conf.n_pages_per_block lpns = [3, 88, 23] - ppns = range(1*n_pages_per_block, 1*n_pages_per_block+3) + ppns = list(range(1 * n_pages_per_block, 1 * n_pages_per_block + 3)) for lpn, ppn in zip(lpns, ppns): oob.remap(lpn=lpn, old_ppn=None, new_ppn=ppn) - lpns_with_na = lpns + ['NA'] * (n_pages_per_block - 3) + lpns_with_na = lpns + ["NA"] * (n_pages_per_block - 3) self.assertListEqual(sorted(oob.lpns_of_block(1)), sorted(lpns_with_na)) @@ -458,10 +532,11 @@ def test_add_mapping(self): rec = create_recorder(conf) helper = create_global_helper(conf) block_pool = NKBlockPool( - n_channels=conf.n_channels_per_dev, - n_blocks_per_channel=conf.n_blocks_per_channel, - n_pages_per_block=conf.n_pages_per_block, - tags=[TDATA, TLOG]) + n_channels=conf.n_channels_per_dev, + n_blocks_per_channel=conf.n_blocks_per_channel, + n_pages_per_block=conf.n_pages_per_block, + tags=[TDATA, TLOG], + ) logmaptable = LogMappingTable(conf, block_pool, rec, helper) @@ -492,10 +567,11 @@ def test_next_ppns(self): rec = create_recorder(conf) helper = create_global_helper(conf) block_pool = NKBlockPool( - n_channels=conf.n_channels_per_dev, - n_blocks_per_channel=conf.n_blocks_per_channel, - n_pages_per_block=conf.n_pages_per_block, - tags=[TDATA, TLOG]) + n_channels=conf.n_channels_per_dev, + n_blocks_per_channel=conf.n_blocks_per_channel, + n_pages_per_block=conf.n_pages_per_block, + tags=[TDATA, TLOG], + ) logmaptable = LogMappingTable(conf, block_pool, rec, helper) @@ -530,7 +606,7 @@ def test_adding(self): found, ppn = datablocktable.lpn_to_ppn(n_pages_per_block + 1) self.assertEqual(found, True) - self.assertEqual(ppn, 8*n_pages_per_block + 1) + self.assertEqual(ppn, 8 * n_pages_per_block + 1) def test_removing(self): conf = create_config() @@ -565,10 +641,10 @@ def test_high_threshold(self): rec = create_recorder(conf) n = len(block_pool.freeblocks) - high_blocks = conf['nkftl']['GC_threshold_ratio'] * n - low_blocks = conf['nkftl']['GC_low_threshold_ratio'] * n + high_blocks = conf["nkftl"]["GC_threshold_ratio"] * n + low_blocks = conf["nkftl"]["GC_low_threshold_ratio"] * n diff = high_blocks - low_blocks - print 'high..', high_blocks + print(("high..", high_blocks)) gcdecider = GcDecider(conf, block_pool, rec) @@ -585,8 +661,8 @@ def test_high_threshold(self): block_pool.free_used_log_block(blocks[i]) self.assertEqual(gcdecider.should_stop(), False) - block_pool.free_used_log_block(blocks[i+1]) - block_pool.free_used_log_block(blocks[i+2]) + block_pool.free_used_log_block(blocks[i + 1]) + block_pool.free_used_log_block(blocks[i + 2]) self.assertEqual(gcdecider.should_stop(), True) @@ -600,11 +676,13 @@ def test_init(self): logmaptable = LogMappingTable(conf, block_pool, rec, helper) datablocktable = DataBlockMappingTable(conf, rec, helper) - vblocks = VictimDataBlocks(conf, block_pool, oob, rec, logmaptable, - datablocktable) + vblocks = VictimDataBlocks( + conf, block_pool, oob, rec, logmaptable, datablocktable + ) - vblocks = VictimLogBlocks(conf, block_pool, oob, rec, logmaptable, - datablocktable) + vblocks = VictimLogBlocks( + conf, block_pool, oob, rec, logmaptable, datablocktable + ) def test_empty_victims_log(self): conf = create_config() @@ -615,8 +693,9 @@ def test_empty_victims_log(self): logmaptable = LogMappingTable(conf, block_pool, rec, helper) datablocktable = DataBlockMappingTable(conf, rec, helper) - vblocks = VictimLogBlocks(conf, block_pool, oob, rec, logmaptable, - datablocktable) + vblocks = VictimLogBlocks( + conf, block_pool, oob, rec, logmaptable, datablocktable + ) cnt = 0 for blkinfo in vblocks: @@ -633,8 +712,9 @@ def test_empty_victims_data(self): logmaptable = LogMappingTable(conf, block_pool, rec, helper) datablocktable = DataBlockMappingTable(conf, rec, helper) - vblocks = VictimDataBlocks(conf, block_pool, oob, rec, logmaptable, - datablocktable) + vblocks = VictimDataBlocks( + conf, block_pool, oob, rec, logmaptable, datablocktable + ) cnt = 0 for blkinfo in vblocks: @@ -645,10 +725,11 @@ def test_empty_victims_data(self): def test_one_victim_blocks(self): conf = create_config() block_pool = NKBlockPool( - n_channels=conf.n_channels_per_dev, - n_blocks_per_channel=conf.n_blocks_per_channel, - n_pages_per_block=conf.n_pages_per_block, - tags=[TDATA, TLOG]) + n_channels=conf.n_channels_per_dev, + n_blocks_per_channel=conf.n_blocks_per_channel, + n_pages_per_block=conf.n_pages_per_block, + tags=[TDATA, TLOG], + ) rec = create_recorder(conf) oob = OutOfBandAreas(conf) helper = create_global_helper(conf) @@ -657,12 +738,14 @@ def test_one_victim_blocks(self): # use one block # +1 is because current log block may not be a victim - self.use_a_log_block(conf, oob, block_pool, logmaptable, - cnt=conf.n_pages_per_block+1, dgn=1) + self.use_a_log_block( + conf, oob, block_pool, logmaptable, cnt=conf.n_pages_per_block + 1, dgn=1 + ) # check the block - vblocks = VictimLogBlocks(conf, block_pool, oob, rec, logmaptable, - datablocktable) + vblocks = VictimLogBlocks( + conf, block_pool, oob, rec, logmaptable, datablocktable + ) self.assertEqual(len(vblocks), 2) cnt = 0 @@ -676,8 +759,9 @@ def use_a_log_block(self, conf, oob, block_pool, logmapping, cnt, dgn): remaining = cnt while remaining > 0: - ppns = logmapping.next_ppns_to_program(dgn=dgn, - n=remaining, strip_unit_size='infinity') + ppns = logmapping.next_ppns_to_program( + dgn=dgn, n=remaining, strip_unit_size="infinity" + ) remaining = remaining - len(ppns) # invalidate them (not the same as in production) @@ -686,29 +770,37 @@ def use_a_log_block(self, conf, oob, block_pool, logmapping, cnt, dgn): def test_log_used(self): conf = create_config() - conf['nkftl']['max_blocks_in_log_group'] = 4 + conf["nkftl"]["max_blocks_in_log_group"] = 4 block_pool = NKBlockPool( - n_channels=conf.n_channels_per_dev, - n_blocks_per_channel=conf.n_blocks_per_channel, - n_pages_per_block=conf.n_pages_per_block, - tags=[TDATA, TLOG]) + n_channels=conf.n_channels_per_dev, + n_blocks_per_channel=conf.n_blocks_per_channel, + n_pages_per_block=conf.n_pages_per_block, + tags=[TDATA, TLOG], + ) rec = create_recorder(conf) oob = OutOfBandAreas(conf) helper = create_global_helper(conf) logmaptable = LogMappingTable(conf, block_pool, rec, helper) datablocktable = DataBlockMappingTable(conf, rec, helper) - self.use_a_log_block(conf, oob, block_pool, logmaptable, - cnt=2*conf.n_pages_per_block+1, dgn=1) + self.use_a_log_block( + conf, + oob, + block_pool, + logmaptable, + cnt=2 * conf.n_pages_per_block + 1, + dgn=1, + ) - vblocks = VictimLogBlocks(conf, block_pool, oob, rec, logmaptable, - datablocktable) + vblocks = VictimLogBlocks( + conf, block_pool, oob, rec, logmaptable, datablocktable + ) self.assertEqual(len(vblocks), 3) def test_data_used(self): conf = create_config() - conf['nkftl']['max_blocks_in_log_group'] = 4 + conf["nkftl"]["max_blocks_in_log_group"] = 4 block_pool = create_nkblockpool(conf) rec = create_recorder(conf) oob = OutOfBandAreas(conf) @@ -719,8 +811,9 @@ def test_data_used(self): self.use_a_data_block(conf, block_pool, oob) self.use_a_data_block(conf, block_pool, oob) - vblocks = VictimDataBlocks(conf, block_pool, oob, rec, logmaptable, - datablocktable) + vblocks = VictimDataBlocks( + conf, block_pool, oob, rec, logmaptable, datablocktable + ) self.assertEqual(len(vblocks), 2) @@ -736,10 +829,11 @@ class TestWearLevelingVictimBlocks(AssertFinishTestCase): def test_data_block(self): conf = create_config() block_pool = NKBlockPool( - n_channels=conf.n_channels_per_dev, - n_blocks_per_channel=conf.n_blocks_per_channel, - n_pages_per_block=conf.n_pages_per_block, - tags=[TDATA, TLOG]) + n_channels=conf.n_channels_per_dev, + n_blocks_per_channel=conf.n_blocks_per_channel, + n_pages_per_block=conf.n_pages_per_block, + tags=[TDATA, TLOG], + ) rec = create_recorder(conf) oob = OutOfBandAreas(conf) helper = create_global_helper(conf) @@ -748,8 +842,9 @@ def test_data_block(self): datablock = self.use_a_data_block(conf, block_pool, oob, datablocktable) - vblocks = WearLevelingVictimBlocks(conf, block_pool, oob, 2, - logmaptable, datablocktable) + vblocks = WearLevelingVictimBlocks( + conf, block_pool, oob, 2, logmaptable, datablocktable + ) blocknums = [blk for _, _, blk in vblocks.iterator_verbose()] self.assertListEqual(blocknums, [datablock]) self.assertEqual(len(list(vblocks.iterator_verbose())), 1) @@ -759,21 +854,24 @@ def test_data_block(self): def test_log_block(self): conf = create_config() block_pool = NKBlockPool( - n_channels=conf.n_channels_per_dev, - n_blocks_per_channel=conf.n_blocks_per_channel, - n_pages_per_block=conf.n_pages_per_block, - tags=[TDATA, TLOG]) + n_channels=conf.n_channels_per_dev, + n_blocks_per_channel=conf.n_blocks_per_channel, + n_pages_per_block=conf.n_pages_per_block, + tags=[TDATA, TLOG], + ) rec = create_recorder(conf) oob = OutOfBandAreas(conf) helper = create_global_helper(conf) logmaptable = LogMappingTable(conf, block_pool, rec, helper) datablocktable = DataBlockMappingTable(conf, rec, helper) - self.use_a_log_block(conf, oob, block_pool, logmaptable, - cnt=conf.n_pages_per_block, dgn=1) + self.use_a_log_block( + conf, oob, block_pool, logmaptable, cnt=conf.n_pages_per_block, dgn=1 + ) - vblocks = WearLevelingVictimBlocks(conf, block_pool, oob, 2, - logmaptable, datablocktable) + vblocks = WearLevelingVictimBlocks( + conf, block_pool, oob, 2, logmaptable, datablocktable + ) blocknums = [blk for _, _, blk in vblocks.iterator_verbose()] self.assertEqual(len(list(vblocks.iterator_verbose())), 1) @@ -795,8 +893,9 @@ def use_a_log_block(self, conf, oob, block_pool, logmapping, cnt, dgn): remaining = cnt while remaining > 0: - ppns = logmapping.next_ppns_to_program(dgn=dgn, - n=remaining, strip_unit_size='infinity') + ppns = logmapping.next_ppns_to_program( + dgn=dgn, n=remaining, strip_unit_size="infinity" + ) remaining = remaining - len(ppns) # invalidate them (not the same as in production) @@ -804,32 +903,61 @@ def use_a_log_block(self, conf, oob, block_pool, logmapping, cnt, dgn): states.invalidate_page(ppn) - - class TestCleaningDataBlocks(AssertFinishTestCase): def test_init_gc(self): - gc, conf, block_pool, rec, oob, helper, \ - logmaptable, datablocktable, translator, \ - flashobj, simpy_env, des_flash = create_gc() + ( + gc, + conf, + block_pool, + rec, + oob, + helper, + logmaptable, + datablocktable, + translator, + flashobj, + simpy_env, + des_flash, + ) = create_gc() self.set_finished() def test_clean_data_blocks(self): pk = create_gc() - gc, conf, block_pool, rec, oob, helper, \ - logmaptable, datablocktable, translator, \ - flashobj, simpy_env, des_flash = pk - - simpy_env.process( - self.proc_test_clean_data_blocks(pk)) + ( + gc, + conf, + block_pool, + rec, + oob, + helper, + logmaptable, + datablocktable, + translator, + flashobj, + simpy_env, + des_flash, + ) = pk + + simpy_env.process(self.proc_test_clean_data_blocks(pk)) simpy_env.run() - def proc_test_clean_data_blocks(self, pk): - gc, conf, block_pool, rec, oob, helper, \ - logmaptable, datablocktable, translator, \ - flashobj, simpy_env, des_flash = pk + ( + gc, + conf, + block_pool, + rec, + oob, + helper, + logmaptable, + datablocktable, + translator, + flashobj, + simpy_env, + des_flash, + ) = pk lbn = 8 blocknum = self.use_a_data_block(conf, block_pool, oob, datablocktable, lbn) @@ -881,27 +1009,59 @@ def use_a_data_block_no_mapping(self, conf, block_pool, oob, datablocktable): class TestMovingDataBlocks(AssertFinishTestCase): def test_init_gc(self): - gc, conf, block_pool, rec, oob, helper, \ - logmaptable, datablocktable, translator, \ - flashobj, simpy_env, des_flash = create_gc() + ( + gc, + conf, + block_pool, + rec, + oob, + helper, + logmaptable, + datablocktable, + translator, + flashobj, + simpy_env, + des_flash, + ) = create_gc() self.set_finished() def test_move_data_blocks(self): pk = create_gc() - gc, conf, block_pool, rec, oob, helper, \ - logmaptable, datablocktable, translator, \ - flashobj, simpy_env, des_flash = pk - - simpy_env.process( - self.proc_test_move_data_blocks(pk)) + ( + gc, + conf, + block_pool, + rec, + oob, + helper, + logmaptable, + datablocktable, + translator, + flashobj, + simpy_env, + des_flash, + ) = pk + + simpy_env.process(self.proc_test_move_data_blocks(pk)) simpy_env.run() def proc_test_move_data_blocks(self, pk): - gc, conf, block_pool, rec, oob, helper, \ - logmaptable, datablocktable, translator, \ - flashobj, simpy_env, des_flash = pk + ( + gc, + conf, + block_pool, + rec, + oob, + helper, + logmaptable, + datablocktable, + translator, + flashobj, + simpy_env, + des_flash, + ) = pk lbn = 8 pbn = self.use_a_data_block(conf, block_pool, oob, datablocktable, lbn) @@ -962,29 +1122,61 @@ def use_a_data_block(self, conf, block_pool, oob, datablocktable, lbn): class TestMovingLogBlocks(AssertFinishTestCase): def test_init_gc(self): - gc, conf, block_pool, rec, oob, helper, \ - logmaptable, datablocktable, translator, \ - flashobj, simpy_env, des_flash = create_gc() + ( + gc, + conf, + block_pool, + rec, + oob, + helper, + logmaptable, + datablocktable, + translator, + flashobj, + simpy_env, + des_flash, + ) = create_gc() self.set_finished() def test_move_data_blocks(self): pk = create_gc() - gc, conf, block_pool, rec, oob, helper, \ - logmaptable, datablocktable, translator, \ - flashobj, simpy_env, des_flash = pk - - simpy_env.process( - self.proc_test_move_data_blocks(pk)) + ( + gc, + conf, + block_pool, + rec, + oob, + helper, + logmaptable, + datablocktable, + translator, + flashobj, + simpy_env, + des_flash, + ) = pk + + simpy_env.process(self.proc_test_move_data_blocks(pk)) simpy_env.run() def proc_test_move_data_blocks(self, pk): - gc, conf, block_pool, rec, oob, helper, \ - logmaptable, datablocktable, translator, \ - flashobj, simpy_env, des_flash = pk - - lpns = range(1, 1 + conf.n_pages_per_block) + ( + gc, + conf, + block_pool, + rec, + oob, + helper, + logmaptable, + datablocktable, + translator, + flashobj, + simpy_env, + des_flash, + ) = pk + + lpns = list(range(1, 1 + conf.n_pages_per_block)) pbn = self.use_a_log_block(conf, oob, block_pool, logmaptable, lpns) # we have programmed a whole block @@ -1043,8 +1235,9 @@ def use_a_log_block(self, conf, oob, block_pool, logmapping, lpns): cnt = len(lpns) # next_ppns_to_program() will allocate block - ppns = logmapping.next_ppns_to_program(dgn=dgn, - n=cnt, strip_unit_size='infinity') + ppns = logmapping.next_ppns_to_program( + dgn=dgn, n=cnt, strip_unit_size="infinity" + ) assert len(ppns) == cnt pbn, _ = conf.page_to_block_off(ppns[0]) @@ -1058,26 +1251,27 @@ def use_a_log_block(self, conf, oob, block_pool, logmapping, lpns): return pbn -class UseLogBlocksMixin(object): - def use_log_blocks(self, conf, oob, block_pool, - logmapping, cnt, lpn_start, translator): +class UseLogBlocksMixin: + def use_log_blocks( + self, conf, oob, block_pool, logmapping, cnt, lpn_start, translator + ): dgn = conf.nkftl_data_group_number_of_lpn(lpn_start) used_blocks, ppns = self.get_ppns_from_data_group( - conf, oob, block_pool, logmapping, cnt=cnt, dgn=dgn) + conf, oob, block_pool, logmapping, cnt=cnt, dgn=dgn + ) self.assertEqual(len(ppns), cnt) - lpns = range(lpn_start, lpn_start + cnt) - self.set_mappings(oob, block_pool, logmapping, lpns, ppns, - translator) + lpns = list(range(lpn_start, lpn_start + cnt)) + self.set_mappings(oob, block_pool, logmapping, lpns, ppns, translator) return used_blocks - def get_ppns_from_data_group(self, - conf, oob, block_pool, logmapping, cnt, dgn): + def get_ppns_from_data_group(self, conf, oob, block_pool, logmapping, cnt, dgn): used_blocks = [] - ppns = logmapping.next_ppns_to_program(dgn=dgn, n=cnt, - strip_unit_size='infinity') + ppns = logmapping.next_ppns_to_program( + dgn=dgn, n=cnt, strip_unit_size="infinity" + ) for ppn in ppns: block, _ = conf.page_to_block_off(ppn) if block not in used_blocks: @@ -1086,10 +1280,9 @@ def get_ppns_from_data_group(self, return used_blocks, ppns def page_ext(self, start, cnt): - return range(start, start + cnt) + return list(range(start, start + cnt)) - def set_mappings(self, oob, block_pool, logmapping, lpns, ppns, - translator): + def set_mappings(self, oob, block_pool, logmapping, lpns, ppns, translator): states = oob.states for lpn, ppn in zip(lpns, ppns): # oob states @@ -1109,31 +1302,62 @@ class TestSwitchMerge(unittest.TestCase, UseLogBlocksMixin): def test_is_switch_mergable(self): pk = create_gc() - gc, conf, block_pool, rec, oob, helper, \ - logmaptable, datablocktable, translator, \ - flashobj, simpy_env, des_flash = pk - - used_blocks = self.use_log_blocks(conf, oob, block_pool, - logmaptable, cnt=conf.n_pages_per_block, - lpn_start=conf.n_pages_per_block, - translator=translator) + ( + gc, + conf, + block_pool, + rec, + oob, + helper, + logmaptable, + datablocktable, + translator, + flashobj, + simpy_env, + des_flash, + ) = pk + + used_blocks = self.use_log_blocks( + conf, + oob, + block_pool, + logmaptable, + cnt=conf.n_pages_per_block, + lpn_start=conf.n_pages_per_block, + translator=translator, + ) mergable, lbn = gc.is_switch_mergable(log_pbn=used_blocks[0]) self.assertEqual(mergable, True) self.assertEqual(lbn, 1) - def test_is_not_switch_mergable_half_used(self): pk = create_gc() - gc, conf, block_pool, rec, oob, helper, \ - logmaptable, datablocktable, translator, \ - flashobj, simpy_env, des_flash = pk - - used_blocks = self.use_log_blocks(conf, oob, block_pool, - logmaptable, cnt=int(conf.n_pages_per_block/2), - lpn_start=conf.n_pages_per_block, - translator=translator) + ( + gc, + conf, + block_pool, + rec, + oob, + helper, + logmaptable, + datablocktable, + translator, + flashobj, + simpy_env, + des_flash, + ) = pk + + used_blocks = self.use_log_blocks( + conf, + oob, + block_pool, + logmaptable, + cnt=int(old_div(conf.n_pages_per_block, 2)), + lpn_start=conf.n_pages_per_block, + translator=translator, + ) mergable, lbn = gc.is_switch_mergable(log_pbn=used_blocks[0]) self.assertEqual(mergable, False) @@ -1142,40 +1366,83 @@ def test_is_not_switch_mergable_half_used(self): def test_is_not_switch_mergable(self): pk = create_gc() - gc, conf, block_pool, rec, oob, helper, \ - logmaptable, datablocktable, translator, \ - flashobj, simpy_env, des_flash = pk - - used_blocks = self.use_log_blocks(conf, oob, block_pool, - logmaptable, cnt=conf.n_pages_per_block+1, - lpn_start=conf.n_pages_per_block+1, - translator=translator) + ( + gc, + conf, + block_pool, + rec, + oob, + helper, + logmaptable, + datablocktable, + translator, + flashobj, + simpy_env, + des_flash, + ) = pk + + used_blocks = self.use_log_blocks( + conf, + oob, + block_pool, + logmaptable, + cnt=conf.n_pages_per_block + 1, + lpn_start=conf.n_pages_per_block + 1, + translator=translator, + ) mergable, lbn = gc.is_switch_mergable(log_pbn=used_blocks[0]) self.assertEqual(mergable, False) self.assertEqual(lbn, None) + class TestSwitchMerge(AssertFinishTestCase, UseLogBlocksMixin): def test_switch_merge(self): pk = create_gc() - gc, conf, block_pool, rec, oob, helper, \ - logmaptable, datablocktable, translator, \ - flashobj, simpy_env, des_flash = pk - - simpy_env.process( - self.proc_test_switch_merge(pk)) + ( + gc, + conf, + block_pool, + rec, + oob, + helper, + logmaptable, + datablocktable, + translator, + flashobj, + simpy_env, + des_flash, + ) = pk + + simpy_env.process(self.proc_test_switch_merge(pk)) simpy_env.run() def proc_test_switch_merge(self, pk): - gc, conf, block_pool, rec, oob, helper, \ - logmaptable, datablocktable, translator, \ - flashobj, simpy_env, des_flash = pk - - used_blocks = self.use_log_blocks(conf, oob, block_pool, - logmaptable, cnt=conf.n_pages_per_block+1, - lpn_start=conf.n_pages_per_block, - translator=translator) + ( + gc, + conf, + block_pool, + rec, + oob, + helper, + logmaptable, + datablocktable, + translator, + flashobj, + simpy_env, + des_flash, + ) = pk + + used_blocks = self.use_log_blocks( + conf, + oob, + block_pool, + logmaptable, + cnt=conf.n_pages_per_block + 1, + lpn_start=conf.n_pages_per_block, + translator=translator, + ) lbn = 1 pbn = used_blocks[0] @@ -1230,35 +1497,68 @@ def proc_test_switch_merge(self, pk): self.set_finished() + class TestPartialMergeMergable(unittest.TestCase, UseLogBlocksMixin): def test_is_partial_mergable(self): pk = create_gc() - gc, conf, block_pool, rec, oob, helper, \ - logmaptable, datablocktable, translator, \ - flashobj, simpy_env, des_flash = pk - - used_blocks = self.use_log_blocks(conf, oob, block_pool, - logmaptable, cnt=int(conf.n_pages_per_block/2), - lpn_start=conf.n_pages_per_block, - translator=translator) + ( + gc, + conf, + block_pool, + rec, + oob, + helper, + logmaptable, + datablocktable, + translator, + flashobj, + simpy_env, + des_flash, + ) = pk + + used_blocks = self.use_log_blocks( + conf, + oob, + block_pool, + logmaptable, + cnt=int(old_div(conf.n_pages_per_block, 2)), + lpn_start=conf.n_pages_per_block, + translator=translator, + ) mergable, lbn, off = gc.is_partial_mergable(log_pbn=used_blocks[0]) self.assertEqual(mergable, True) self.assertEqual(lbn, 1) - self.assertEqual(off, int(conf.n_pages_per_block/2)) + self.assertEqual(off, int(old_div(conf.n_pages_per_block, 2))) def test_is_not_partial_mergable_not_aligned(self): pk = create_gc() - gc, conf, block_pool, rec, oob, helper, \ - logmaptable, datablocktable, translator, \ - flashobj, simpy_env, des_flash = pk - - used_blocks = self.use_log_blocks(conf, oob, block_pool, - logmaptable, cnt=int(conf.n_pages_per_block/2), - lpn_start=conf.n_pages_per_block+1, - translator=translator) + ( + gc, + conf, + block_pool, + rec, + oob, + helper, + logmaptable, + datablocktable, + translator, + flashobj, + simpy_env, + des_flash, + ) = pk + + used_blocks = self.use_log_blocks( + conf, + oob, + block_pool, + logmaptable, + cnt=int(old_div(conf.n_pages_per_block, 2)), + lpn_start=conf.n_pages_per_block + 1, + translator=translator, + ) mergable, lbn, off = gc.is_partial_mergable(log_pbn=used_blocks[0]) self.assertEqual(mergable, False) @@ -1267,14 +1567,30 @@ def test_is_not_partial_mergable_not_aligned(self): def test_is_not_partial_mergable_because_its_switch_mergable(self): pk = create_gc() - gc, conf, block_pool, rec, oob, helper, \ - logmaptable, datablocktable, translator, \ - flashobj, simpy_env, des_flash = pk - - used_blocks = self.use_log_blocks(conf, oob, block_pool, - logmaptable, cnt=conf.n_pages_per_block, - lpn_start=conf.n_pages_per_block, - translator=translator) + ( + gc, + conf, + block_pool, + rec, + oob, + helper, + logmaptable, + datablocktable, + translator, + flashobj, + simpy_env, + des_flash, + ) = pk + + used_blocks = self.use_log_blocks( + conf, + oob, + block_pool, + logmaptable, + cnt=conf.n_pages_per_block, + lpn_start=conf.n_pages_per_block, + translator=translator, + ) mergable, lbn, off = gc.is_partial_mergable(log_pbn=used_blocks[0]) self.assertEqual(mergable, False) @@ -1285,27 +1601,54 @@ class TestPartialMerge_Merge(AssertFinishTestCase, UseLogBlocksMixin): def test_partial_merge(self): pk = create_gc() - gc, conf, block_pool, rec, oob, helper, \ - logmaptable, datablocktable, translator, \ - flashobj, simpy_env, des_flash = pk + ( + gc, + conf, + block_pool, + rec, + oob, + helper, + logmaptable, + datablocktable, + translator, + flashobj, + simpy_env, + des_flash, + ) = pk simpy_env.process(self.proc_test_partial_merge(pk)) simpy_env.run() def proc_test_partial_merge(self, pk): - gc, conf, block_pool, rec, oob, helper, \ - logmaptable, datablocktable, translator, \ - flashobj, simpy_env, des_flash = pk - - used_blocks = self.use_log_blocks(conf, oob, block_pool, - logmaptable, cnt=int(conf.n_pages_per_block/2), - lpn_start=conf.n_pages_per_block, - translator=translator) + ( + gc, + conf, + block_pool, + rec, + oob, + helper, + logmaptable, + datablocktable, + translator, + flashobj, + simpy_env, + des_flash, + ) = pk + + used_blocks = self.use_log_blocks( + conf, + oob, + block_pool, + logmaptable, + cnt=int(old_div(conf.n_pages_per_block, 2)), + lpn_start=conf.n_pages_per_block, + translator=translator, + ) mergable, lbn, off = gc.is_partial_mergable(log_pbn=used_blocks[0]) self.assertEqual(mergable, True) self.assertEqual(lbn, 1) - self.assertEqual(off, int(conf.n_pages_per_block/2)) + self.assertEqual(off, int(old_div(conf.n_pages_per_block, 2))) pbn = used_blocks[0] # data block mapping @@ -1339,8 +1682,8 @@ def proc_test_partial_merge(self, pk): self.assertIn(pbn, block_pool.log_usedblocks) yield simpy_env.process( - gc.partial_merge(log_pbn=used_blocks[0], lbn=lbn, - first_free_offset=off)) + gc.partial_merge(log_pbn=used_blocks[0], lbn=lbn, first_free_offset=off) + ) # data block mapping found, retrieved_pbn = datablocktable.lbn_to_pbn(lbn) @@ -1377,35 +1720,60 @@ def proc_test_partial_merge(self, pk): self.set_finished() + class TestPartialMergeWithMoving(AssertFinishTestCase, UseLogBlocksMixin): def test(self): pk = create_gc() - gc, conf, block_pool, rec, oob, helper, \ - logmaptable, datablocktable, translator, \ - flashobj, simpy_env, des_flash = pk + ( + gc, + conf, + block_pool, + rec, + oob, + helper, + logmaptable, + datablocktable, + translator, + flashobj, + simpy_env, + des_flash, + ) = pk simpy_env.process(self.proc(pk)) simpy_env.run() - def proc(self, pk): - gc, conf, block_pool, rec, oob, helper, \ - logmaptable, datablocktable, translator, \ - flashobj, simpy_env, des_flash = pk - - half_block_pages = int(conf.n_pages_per_block/2) + ( + gc, + conf, + block_pool, + rec, + oob, + helper, + logmaptable, + datablocktable, + translator, + flashobj, + simpy_env, + des_flash, + ) = pk + + half_block_pages = int(old_div(conf.n_pages_per_block, 2)) ########################## # unaligned, second half, first half used_blocks, ppns = self.get_ppns_from_data_group( - conf, oob, block_pool, logmaptable, cnt=half_block_pages * 3, - dgn=0) - lpns = self.page_ext(conf.n_pages_per_block + 1, half_block_pages) +\ - self.page_ext(2 * conf.n_pages_per_block + half_block_pages, half_block_pages) +\ - self.page_ext(2 * conf.n_pages_per_block, half_block_pages) - self.set_mappings(oob, block_pool, logmaptable, lpns, ppns, - translator) + conf, oob, block_pool, logmaptable, cnt=half_block_pages * 3, dgn=0 + ) + lpns = ( + self.page_ext(conf.n_pages_per_block + 1, half_block_pages) + + self.page_ext( + 2 * conf.n_pages_per_block + half_block_pages, half_block_pages + ) + + self.page_ext(2 * conf.n_pages_per_block, half_block_pages) + ) + self.set_mappings(oob, block_pool, logmaptable, lpns, ppns, translator) self.assertEqual(len(used_blocks), 2) mergable, lbn, off = gc.is_partial_mergable(log_pbn=used_blocks[1]) @@ -1450,7 +1818,8 @@ def proc(self, pk): self.assertIn(pbn, block_pool.log_usedblocks) yield simpy_env.process( - gc.partial_merge(log_pbn=pbn, lbn=lbn, first_free_offset=off)) + gc.partial_merge(log_pbn=pbn, lbn=lbn, first_free_offset=off) + ) # data block mapping found, retrieved_pbn = datablocktable.lbn_to_pbn(lbn) @@ -1479,31 +1848,54 @@ def proc(self, pk): self.set_finished() + class TestFullMerge_Unaligned(AssertFinishTestCase, UseLogBlocksMixin): def test(self): pk = create_gc() - gc, conf, block_pool, rec, oob, helper, \ - logmaptable, datablocktable, translator, \ - flashobj, simpy_env, des_flash = pk + ( + gc, + conf, + block_pool, + rec, + oob, + helper, + logmaptable, + datablocktable, + translator, + flashobj, + simpy_env, + des_flash, + ) = pk simpy_env.process(self.proc(pk)) simpy_env.run() def proc(self, pk): - gc, conf, block_pool, rec, oob, helper, \ - logmaptable, datablocktable, translator, \ - flashobj, simpy_env, des_flash = pk - - half_block_pages = int(conf.n_pages_per_block/2) + ( + gc, + conf, + block_pool, + rec, + oob, + helper, + logmaptable, + datablocktable, + translator, + flashobj, + simpy_env, + des_flash, + ) = pk + + half_block_pages = int(old_div(conf.n_pages_per_block, 2)) used_blocks, ppns = self.get_ppns_from_data_group( - conf, oob, block_pool, logmaptable, cnt=half_block_pages * 2, - dgn=0) - lpns = self.page_ext(conf.n_pages_per_block + half_block_pages, half_block_pages) +\ - self.page_ext(conf.n_pages_per_block, half_block_pages) - self.set_mappings(oob, block_pool, logmaptable, lpns, ppns, - translator) + conf, oob, block_pool, logmaptable, cnt=half_block_pages * 2, dgn=0 + ) + lpns = self.page_ext( + conf.n_pages_per_block + half_block_pages, half_block_pages + ) + self.page_ext(conf.n_pages_per_block, half_block_pages) + self.set_mappings(oob, block_pool, logmaptable, lpns, ppns, translator) self.assertEqual(len(used_blocks), 1) pbn = used_blocks[0] @@ -1517,8 +1909,9 @@ def proc(self, pk): lpn = conf.block_off_to_page(lbn, i) found, ppn = logmaptable.lpn_to_ppn(lpn) self.assertEqual(found, True) - correct_ppn = conf.block_off_to_page(pbn, - (i + half_block_pages) % conf.n_pages_per_block) + correct_ppn = conf.block_off_to_page( + pbn, (i + half_block_pages) % conf.n_pages_per_block + ) self.assertEqual(ppn, correct_ppn) # oob states for i in range(conf.n_pages_per_block): @@ -1528,8 +1921,9 @@ def proc(self, pk): for i in range(conf.n_pages_per_block): ppn = conf.block_off_to_page(pbn, i) lpn = oob.translate_ppn_to_lpn(ppn) - correct_lpn = conf.block_off_to_page(lbn, - (i + half_block_pages) % conf.n_pages_per_block) + correct_lpn = conf.block_off_to_page( + lbn, (i + half_block_pages) % conf.n_pages_per_block + ) self.assertEqual(lpn, correct_lpn) # block pool self.assertIn(pbn, block_pool.log_usedblocks) @@ -1579,31 +1973,58 @@ def test(self): """ pk = create_gc() - gc, conf, block_pool, rec, oob, helper, \ - logmaptable, datablocktable, translator, \ - flashobj, simpy_env, des_flash = pk + ( + gc, + conf, + block_pool, + rec, + oob, + helper, + logmaptable, + datablocktable, + translator, + flashobj, + simpy_env, + des_flash, + ) = pk simpy_env.process(self.proc(pk)) simpy_env.run() def proc(self, pk): - gc, conf, block_pool, rec, oob, helper, \ - logmaptable, datablocktable, translator, \ - flashobj, simpy_env, des_flash = pk - - half_block_pages = int(conf.n_pages_per_block/2) + ( + gc, + conf, + block_pool, + rec, + oob, + helper, + logmaptable, + datablocktable, + translator, + flashobj, + simpy_env, + des_flash, + ) = pk + + half_block_pages = int(old_div(conf.n_pages_per_block, 2)) used_blocks, ppns = self.get_ppns_from_data_group( - conf, oob, block_pool, logmaptable, cnt=half_block_pages * 4, - dgn=0) + conf, oob, block_pool, logmaptable, cnt=half_block_pages * 4, dgn=0 + ) lbn1 = 1 lbn2 = 3 - lpns = self.page_ext(lbn1 * conf.n_pages_per_block + half_block_pages, half_block_pages) +\ - self.page_ext(lbn2 * conf.n_pages_per_block + half_block_pages, half_block_pages) +\ - self.page_ext(lbn1 * conf.n_pages_per_block, half_block_pages) +\ - self.page_ext(lbn2 * conf.n_pages_per_block, half_block_pages) - self.set_mappings(oob, block_pool, logmaptable, lpns, ppns, - translator) + lpns = ( + self.page_ext( + lbn1 * conf.n_pages_per_block + half_block_pages, half_block_pages + ) + + self.page_ext( + lbn2 * conf.n_pages_per_block + half_block_pages, half_block_pages + ) + + self.page_ext(lbn1 * conf.n_pages_per_block, half_block_pages) + + self.page_ext(lbn2 * conf.n_pages_per_block, half_block_pages) + ) + self.set_mappings(oob, block_pool, logmaptable, lpns, ppns, translator) self.assertEqual(len(used_blocks), 2) ######## start checking ######## @@ -1655,26 +2076,26 @@ def proc(self, pk): for i in range(half_block_pages): ppn = conf.block_off_to_page(pbn1, i) lpn = oob.translate_ppn_to_lpn(ppn) - correct_lpn = conf.block_off_to_page(lbn1, - (i + half_block_pages) % conf.n_pages_per_block) + correct_lpn = conf.block_off_to_page( + lbn1, (i + half_block_pages) % conf.n_pages_per_block + ) self.assertEqual(lpn, correct_lpn) for i in range(half_block_pages, conf.n_pages_per_block): ppn = conf.block_off_to_page(pbn1, i) lpn = oob.translate_ppn_to_lpn(ppn) - correct_lpn = conf.block_off_to_page(lbn2, - i % conf.n_pages_per_block) + correct_lpn = conf.block_off_to_page(lbn2, i % conf.n_pages_per_block) self.assertEqual(lpn, correct_lpn) for i in range(half_block_pages): ppn = conf.block_off_to_page(pbn2, i) lpn = oob.translate_ppn_to_lpn(ppn) - correct_lpn = conf.block_off_to_page(lbn1, - i % conf.n_pages_per_block) + correct_lpn = conf.block_off_to_page(lbn1, i % conf.n_pages_per_block) self.assertEqual(lpn, correct_lpn) for i in range(half_block_pages, conf.n_pages_per_block): ppn = conf.block_off_to_page(pbn2, i) lpn = oob.translate_ppn_to_lpn(ppn) - correct_lpn = conf.block_off_to_page(lbn2, - (i-half_block_pages) % conf.n_pages_per_block) + correct_lpn = conf.block_off_to_page( + lbn2, (i - half_block_pages) % conf.n_pages_per_block + ) self.assertEqual(lpn, correct_lpn) # block pool @@ -1738,7 +2159,6 @@ def proc(self, pk): correct_lpn = conf.block_off_to_page(lbn2, i) self.assertEqual(lpn, correct_lpn) - # block pool self.assertIn(pbn1, block_pool.freeblocks) self.assertIn(pbn2, block_pool.freeblocks) @@ -1803,7 +2223,6 @@ def proc(self, pk): correct_lpn = conf.block_off_to_page(lbn2, i) self.assertEqual(lpn, correct_lpn) - # block pool self.assertIn(pbn1, block_pool.freeblocks) self.assertIn(pbn2, block_pool.freeblocks) @@ -1817,34 +2236,62 @@ class TestFullMerge_with_data_blocks(AssertFinishTestCase, UseLogBlocksMixin): def test(self): pk = create_gc() - gc, conf, block_pool, rec, oob, helper, \ - logmaptable, datablocktable, translator, \ - flashobj, simpy_env, des_flash = pk + ( + gc, + conf, + block_pool, + rec, + oob, + helper, + logmaptable, + datablocktable, + translator, + flashobj, + simpy_env, + des_flash, + ) = pk simpy_env.process(self.proc(pk)) simpy_env.run() def proc(self, pk): - gc, conf, block_pool, rec, oob, helper, \ - logmaptable, datablocktable, translator, \ - flashobj, simpy_env, des_flash = pk - - half_block_pages = int(conf.n_pages_per_block/2) - - lbn1=3 + ( + gc, + conf, + block_pool, + rec, + oob, + helper, + logmaptable, + datablocktable, + translator, + flashobj, + simpy_env, + des_flash, + ) = pk + + half_block_pages = int(old_div(conf.n_pages_per_block, 2)) + + lbn1 = 3 # put first half of lba1 in data block - usedblocks1 = self.use_data_blocks(conf, block_pool, oob, datablocktable, - lpn_start=lbn1 * conf.n_pages_per_block, - cnt=half_block_pages, translator=translator) + usedblocks1 = self.use_data_blocks( + conf, + block_pool, + oob, + datablocktable, + lpn_start=lbn1 * conf.n_pages_per_block, + cnt=half_block_pages, + translator=translator, + ) self.assertEqual(len(usedblocks1), 1) - used_blocks, ppns = self.get_ppns_from_data_group( - conf, oob, block_pool, logmaptable, cnt=half_block_pages * 2, - dgn=0) - lpns = self.page_ext(lbn1 * conf.n_pages_per_block + half_block_pages, half_block_pages) - self.set_mappings(oob, block_pool, logmaptable, lpns, ppns, - translator) + conf, oob, block_pool, logmaptable, cnt=half_block_pages * 2, dgn=0 + ) + lpns = self.page_ext( + lbn1 * conf.n_pages_per_block + half_block_pages, half_block_pages + ) + self.set_mappings(oob, block_pool, logmaptable, lpns, ppns, translator) self.assertEqual(len(used_blocks), 1) data_pbn = usedblocks1[0] @@ -1870,8 +2317,8 @@ def proc(self, pk): self.assertTrue(oob.states.is_page_valid(ppn)) for i in range(half_block_pages, conf.n_pages_per_block): ppn = conf.block_off_to_page(data_pbn, i) - print ppn - print 'page state ..........', oob.states.page_state_human(ppn) + print(ppn) + print(("page state ..........", oob.states.page_state_human(ppn))) self.assertTrue(oob.states.is_page_erased(ppn)) for i in range(half_block_pages): @@ -1885,8 +2332,7 @@ def proc(self, pk): for i in range(half_block_pages): ppn = conf.block_off_to_page(data_pbn, i) lpn = oob.translate_ppn_to_lpn(ppn) - correct_lpn = conf.block_off_to_page(lbn1, - i % conf.n_pages_per_block) + correct_lpn = conf.block_off_to_page(lbn1, i % conf.n_pages_per_block) self.assertEqual(lpn, correct_lpn) for i in range(half_block_pages, conf.n_pages_per_block): ppn = conf.block_off_to_page(data_pbn, i) @@ -1896,8 +2342,9 @@ def proc(self, pk): for i in range(half_block_pages): ppn = conf.block_off_to_page(log_pbn, i) lpn = oob.translate_ppn_to_lpn(ppn) - correct_lpn = conf.block_off_to_page(lbn1, - (i+half_block_pages) % conf.n_pages_per_block) + correct_lpn = conf.block_off_to_page( + lbn1, (i + half_block_pages) % conf.n_pages_per_block + ) self.assertEqual(lpn, correct_lpn) for i in range(half_block_pages, conf.n_pages_per_block): ppn = conf.block_off_to_page(log_pbn, i) @@ -1945,8 +2392,7 @@ def proc(self, pk): ppn = conf.block_off_to_page(retrieved_pbn1, i) lpn = oob.translate_ppn_to_lpn(ppn) - correct_lpn = conf.block_off_to_page(lbn1, - i % conf.n_pages_per_block) + correct_lpn = conf.block_off_to_page(lbn1, i % conf.n_pages_per_block) self.assertEqual(lpn, correct_lpn) # block pool @@ -1956,9 +2402,9 @@ def proc(self, pk): self.set_finished() - def use_data_blocks(self, conf, block_pool, oob, datablocktable, lpn_start, - cnt, translator): - + def use_data_blocks( + self, conf, block_pool, oob, datablocktable, lpn_start, cnt, translator + ): lpn = lpn_start used_blocks = [] while cnt > 0: @@ -1987,27 +2433,48 @@ class TestFullMergeOnePage(AssertFinishTestCase, UseLogBlocksMixin): def test(self): pk = create_gc() - gc, conf, block_pool, rec, oob, helper, \ - logmaptable, datablocktable, translator, \ - flashobj, simpy_env, des_flash = pk + ( + gc, + conf, + block_pool, + rec, + oob, + helper, + logmaptable, + datablocktable, + translator, + flashobj, + simpy_env, + des_flash, + ) = pk simpy_env.process(self.proc(pk)) simpy_env.run() def proc(self, pk): - gc, conf, block_pool, rec, oob, helper, \ - logmaptable, datablocktable, translator, \ - flashobj, simpy_env, des_flash = pk - - half_block_pages = int(conf.n_pages_per_block/2) + ( + gc, + conf, + block_pool, + rec, + oob, + helper, + logmaptable, + datablocktable, + translator, + flashobj, + simpy_env, + des_flash, + ) = pk + + half_block_pages = int(old_div(conf.n_pages_per_block, 2)) used_blocks, ppns = self.get_ppns_from_data_group( - conf, oob, block_pool, logmaptable, cnt=1, - dgn=0) + conf, oob, block_pool, logmaptable, cnt=1, dgn=0 + ) lbn = 1 lpns = self.page_ext(lbn * conf.n_pages_per_block + half_block_pages, 1) - self.set_mappings(oob, block_pool, logmaptable, lpns, ppns, - translator) + self.set_mappings(oob, block_pool, logmaptable, lpns, ppns, translator) self.assertEqual(len(used_blocks), 1) pbn = used_blocks[0] @@ -2087,35 +2554,60 @@ def test(self): """ pk = create_gc() - gc, conf, block_pool, rec, oob, helper, \ - logmaptable, datablocktable, translator, \ - flashobj, simpy_env, des_flash = pk + ( + gc, + conf, + block_pool, + rec, + oob, + helper, + logmaptable, + datablocktable, + translator, + flashobj, + simpy_env, + des_flash, + ) = pk simpy_env.process(self.proc(pk)) simpy_env.run() def proc(self, pk): - gc, conf, block_pool, rec, oob, helper, \ - logmaptable, datablocktable, translator, \ - flashobj, simpy_env, des_flash = pk - - half_block_pages = int(conf.n_pages_per_block/2) + ( + gc, + conf, + block_pool, + rec, + oob, + helper, + logmaptable, + datablocktable, + translator, + flashobj, + simpy_env, + des_flash, + ) = pk + + half_block_pages = int(old_div(conf.n_pages_per_block, 2)) used_blocks, ppns = self.get_ppns_from_data_group( - conf, oob, block_pool, logmaptable, cnt=half_block_pages * 4, - dgn=0) + conf, oob, block_pool, logmaptable, cnt=half_block_pages * 4, dgn=0 + ) lbn1 = 1 lbn2 = 3 - lpns = self.page_ext(lbn1 * conf.n_pages_per_block + half_block_pages, half_block_pages) +\ - self.page_ext(lbn2 * conf.n_pages_per_block + half_block_pages, half_block_pages) +\ - self.page_ext(lbn1 * conf.n_pages_per_block, half_block_pages) +\ - self.page_ext(lbn2 * conf.n_pages_per_block, half_block_pages) - self.set_mappings(oob, block_pool, logmaptable, lpns, ppns, - translator) + lpns = ( + self.page_ext( + lbn1 * conf.n_pages_per_block + half_block_pages, half_block_pages + ) + + self.page_ext( + lbn2 * conf.n_pages_per_block + half_block_pages, half_block_pages + ) + + self.page_ext(lbn1 * conf.n_pages_per_block, half_block_pages) + + self.page_ext(lbn2 * conf.n_pages_per_block, half_block_pages) + ) + self.set_mappings(oob, block_pool, logmaptable, lpns, ppns, translator) self.assertEqual(len(used_blocks), 2) - - ######## start checking ######## pbn1 = used_blocks[0] pbn2 = used_blocks[1] @@ -2165,26 +2657,26 @@ def proc(self, pk): for i in range(half_block_pages): ppn = conf.block_off_to_page(pbn1, i) lpn = oob.translate_ppn_to_lpn(ppn) - correct_lpn = conf.block_off_to_page(lbn1, - (i + half_block_pages) % conf.n_pages_per_block) + correct_lpn = conf.block_off_to_page( + lbn1, (i + half_block_pages) % conf.n_pages_per_block + ) self.assertEqual(lpn, correct_lpn) for i in range(half_block_pages, conf.n_pages_per_block): ppn = conf.block_off_to_page(pbn1, i) lpn = oob.translate_ppn_to_lpn(ppn) - correct_lpn = conf.block_off_to_page(lbn2, - i % conf.n_pages_per_block) + correct_lpn = conf.block_off_to_page(lbn2, i % conf.n_pages_per_block) self.assertEqual(lpn, correct_lpn) for i in range(half_block_pages): ppn = conf.block_off_to_page(pbn2, i) lpn = oob.translate_ppn_to_lpn(ppn) - correct_lpn = conf.block_off_to_page(lbn1, - i % conf.n_pages_per_block) + correct_lpn = conf.block_off_to_page(lbn1, i % conf.n_pages_per_block) self.assertEqual(lpn, correct_lpn) for i in range(half_block_pages, conf.n_pages_per_block): ppn = conf.block_off_to_page(pbn2, i) lpn = oob.translate_ppn_to_lpn(ppn) - correct_lpn = conf.block_off_to_page(lbn2, - (i-half_block_pages) % conf.n_pages_per_block) + correct_lpn = conf.block_off_to_page( + lbn2, (i - half_block_pages) % conf.n_pages_per_block + ) self.assertEqual(lpn, correct_lpn) # block pool @@ -2192,8 +2684,11 @@ def proc(self, pk): self.assertIn(pbn2, block_pool.log_usedblocks) ########### clean ############## - yield simpy_env.process(gc.clean_data_group( - data_group_no=conf.nkftl_data_group_number_of_logical_block(lbn1))) + yield simpy_env.process( + gc.clean_data_group( + data_group_no=conf.nkftl_data_group_number_of_logical_block(lbn1) + ) + ) ########### check ##################### # data block mapping @@ -2257,9 +2752,9 @@ def proc(self, pk): self.set_finished() - def use_data_blocks(self, conf, block_pool, oob, datablocktable, lpn_start, - cnt, translator): - + def use_data_blocks( + self, conf, block_pool, oob, datablocktable, lpn_start, cnt, translator + ): lpn = lpn_start used_blocks = [] while cnt > 0: @@ -2287,19 +2782,19 @@ def use_data_blocks(self, conf, block_pool, oob, datablocktable, lpn_start, def create_config_2(): conf = wiscsim.nkftl2.Config() - conf['flash_config']['n_pages_per_block'] = 32 - conf['flash_config']['n_blocks_per_plane'] = 64 - conf['flash_config']['n_planes_per_chip'] = 1 - conf['flash_config']['n_chips_per_package'] = 1 - conf['flash_config']['n_packages_per_channel'] = 1 - conf['flash_config']['n_channels_per_dev'] = 8 + conf["flash_config"]["n_pages_per_block"] = 32 + conf["flash_config"]["n_blocks_per_plane"] = 64 + conf["flash_config"]["n_planes_per_chip"] = 1 + conf["flash_config"]["n_chips_per_package"] = 1 + conf["flash_config"]["n_packages_per_channel"] = 1 + conf["flash_config"]["n_channels_per_dev"] = 8 - conf['nkftl']['max_blocks_in_log_group'] = 16 - conf['nkftl']['n_blocks_in_data_group'] = 4 + conf["nkftl"]["max_blocks_in_log_group"] = 16 + conf["nkftl"]["n_blocks_in_data_group"] = 4 - utils.set_exp_metadata(conf, save_data = False, - expname = 'test_expname', - subexpname = 'test_subexpname') + utils.set_exp_metadata( + conf, save_data=False, expname="test_expname", subexpname="test_subexpname" + ) logicsize_mb = 64 conf.set_flash_num_blocks_by_bytes(int(logicsize_mb * 2**20 * 1.28)) @@ -2312,37 +2807,41 @@ def create_config_2(): def create_loggroup2(): conf = create_config_2() blockpool = NKBlockPool( - n_channels=conf.n_channels_per_dev, - n_blocks_per_channel=conf.n_blocks_per_channel, - n_pages_per_block=conf.n_pages_per_block, - tags=[TDATA, TLOG]) + n_channels=conf.n_channels_per_dev, + n_blocks_per_channel=conf.n_blocks_per_channel, + n_pages_per_block=conf.n_pages_per_block, + tags=[TDATA, TLOG], + ) loggroup = LogGroup2( - conf = conf, - block_pool=blockpool, - max_n_log_blocks=conf['nkftl']['max_blocks_in_log_group']) + conf=conf, + block_pool=blockpool, + max_n_log_blocks=conf["nkftl"]["max_blocks_in_log_group"], + ) - loggroup._cur_channel = 0 # override random cur channel + loggroup._cur_channel = 0 # override random cur channel return blockpool, loggroup + def create_loggroup2_one_channel(): conf = create_config_1_channel() blockpool = NKBlockPool( - n_channels=conf.n_channels_per_dev, - n_blocks_per_channel=conf.n_blocks_per_channel, - n_pages_per_block=conf.n_pages_per_block, - tags=[TDATA, TLOG]) + n_channels=conf.n_channels_per_dev, + n_blocks_per_channel=conf.n_blocks_per_channel, + n_pages_per_block=conf.n_pages_per_block, + tags=[TDATA, TLOG], + ) loggroup = LogGroup2( - conf = conf, - block_pool=blockpool, - max_n_log_blocks=conf['nkftl']['max_blocks_in_log_group']) + conf=conf, + block_pool=blockpool, + max_n_log_blocks=conf["nkftl"]["max_blocks_in_log_group"], + ) - loggroup._cur_channel = 0 # override random cur channel + loggroup._cur_channel = 0 # override random cur channel return blockpool, loggroup - class TestLogGroup2(unittest.TestCase): def test_next_ppns_in_channel(self): blockpool, loggroup = create_loggroup2() @@ -2364,14 +2863,16 @@ def test_next_ppns_in_channel_failure(self): def test_incr_channel(self): conf = create_config_2() blockpool = NKBlockPool( - n_channels=conf.n_channels_per_dev, - n_blocks_per_channel=conf.n_blocks_per_channel, - n_pages_per_block=conf.n_pages_per_block, - tags=[TDATA, TLOG]) + n_channels=conf.n_channels_per_dev, + n_blocks_per_channel=conf.n_blocks_per_channel, + n_pages_per_block=conf.n_pages_per_block, + tags=[TDATA, TLOG], + ) loggroup = LogGroup2( - conf = conf, - block_pool=blockpool, - max_n_log_blocks=conf['nkftl']['max_blocks_in_log_group']) + conf=conf, + block_pool=blockpool, + max_n_log_blocks=conf["nkftl"]["max_blocks_in_log_group"], + ) loggroup._cur_channel = 0 self.assertEqual(loggroup._cur_channel, 0) @@ -2401,12 +2902,11 @@ def test_allocate_blocks_in_channel(self): self.assertEqual(loggroup.n_log_blocks(), 2) self.assertEqual(loggroup.n_channel_free_pages(channel_id=2), 64) - for i in range(16-2): + for i in range(16 - 2): loggroup._allocate_block_in_channel(channel_id=0) self.assertEqual(loggroup.n_log_blocks(), 16) - self.assertEqual(loggroup.n_channel_free_pages(channel_id=0), - 32 * (16 - 2)) + self.assertEqual(loggroup.n_channel_free_pages(channel_id=0), 32 * (16 - 2)) allocated = loggroup._allocate_block_in_channel(channel_id=0) self.assertEqual(allocated, False) @@ -2425,44 +2925,46 @@ def test_next_ppns_simple(self): def test_next_ppns_two_blocks_per_channel(self): blockpool, loggroup = create_loggroup2() - ppns = loggroup.next_ppns(n=33*8, strip_unit_size=32) + ppns = loggroup.next_ppns(n=33 * 8, strip_unit_size=32) - self.assertEqual(len(ppns), 33*8) + self.assertEqual(len(ppns), 33 * 8) self.assertEqual(loggroup.n_log_blocks(), 9) - self.assertEqual(loggroup.n_channel_free_pages(0), 32-8) + self.assertEqual(loggroup.n_channel_free_pages(0), 32 - 8) for i in range(1, 8): self.assertEqual(loggroup.n_channel_free_pages(i), 0) def test_next_ppns_overflow_it(self): blockpool, loggroup = create_loggroup2() - ppns = loggroup.next_ppns(n=32*2*8 + 1, strip_unit_size=32) + ppns = loggroup.next_ppns(n=32 * 2 * 8 + 1, strip_unit_size=32) - self.assertEqual(len(ppns), 32*2*8) - self.assertEqual(loggroup.n_log_blocks(), 8*2) + self.assertEqual(len(ppns), 32 * 2 * 8) + self.assertEqual(loggroup.n_log_blocks(), 8 * 2) for i in range(8): self.assertEqual(loggroup.n_channel_free_pages(i), 0) def test_next_ppns_infinit_strip(self): blockpool, loggroup = create_loggroup2() - ppns = loggroup.next_ppns(n=33, strip_unit_size='infinity') + ppns = loggroup.next_ppns(n=33, strip_unit_size="infinity") self.assertEqual(len(ppns), 33) self.assertEqual(loggroup.n_log_blocks(), 2) def test_next_ppns_small_strip(self): conf = create_config_2() - conf['nkftl']['max_blocks_in_log_group'] = 2 + conf["nkftl"]["max_blocks_in_log_group"] = 2 blockpool = NKBlockPool( - n_channels=conf.n_channels_per_dev, - n_blocks_per_channel=conf.n_blocks_per_channel, - n_pages_per_block=conf.n_pages_per_block, - tags=[TDATA, TLOG]) + n_channels=conf.n_channels_per_dev, + n_blocks_per_channel=conf.n_blocks_per_channel, + n_pages_per_block=conf.n_pages_per_block, + tags=[TDATA, TLOG], + ) loggroup = LogGroup2( - conf = conf, - block_pool=blockpool, - max_n_log_blocks=conf['nkftl']['max_blocks_in_log_group']) + conf=conf, + block_pool=blockpool, + max_n_log_blocks=conf["nkftl"]["max_blocks_in_log_group"], + ) ppns = loggroup.next_ppns(n=32, strip_unit_size=16) self.assertEqual(len(ppns), 32) @@ -2477,7 +2979,7 @@ def test_next_ppns_small_strip(self): def test_next_ppns_overflow_channel(self): blockpool, loggroup = create_loggroup2() - ppns = loggroup.next_ppns(n=65, strip_unit_size='infinity') + ppns = loggroup.next_ppns(n=65, strip_unit_size="infinity") self.assertEqual(len(set(ppns)), 65) self.assertEqual(loggroup.n_log_blocks(), 3) @@ -2572,7 +3074,6 @@ def proc_test_write(self, env, ftl, conf): self.set_finished() - def test_read(self): ftl, conf, rec, env = create_nkftl() @@ -2671,7 +3172,7 @@ def test_write_large_extent(self): def proc_test_write_large_extent(self, env, ftl, conf): for i in range(8): - ext = Extent(lpn_start=1, lpn_count=conf.n_pages_per_block*63) + ext = Extent(lpn_start=1, lpn_count=conf.n_pages_per_block * 63) yield env.process(ftl.write_ext(ext)) ppns = [] @@ -2698,13 +3199,11 @@ def test_write(self): def read_proc(self, env, ftl, extent): data = yield env.process(ftl.read_ext(extent)) self.set_finished() - env.exit(data) - + return data def reader_main(self, env, ftl): ext = Extent(0, 3) - yield env.process( - self.write_proc(env, ftl, ext, self.data_of_extent(ext))) + yield env.process(self.write_proc(env, ftl, ext, self.data_of_extent(ext))) ret_data = yield env.process(self.read_proc(env, ftl, ext)) self.assertListEqual(ret_data, [str(x) for x in ext.lpn_iter()]) self.set_finished() @@ -2716,7 +3215,9 @@ def test_read(self): env.run() -class TestLogicalBlockSerialization_DifferentLogicalBlock(AssertFinishTestCase, RWMixin): +class TestLogicalBlockSerialization_DifferentLogicalBlock( + AssertFinishTestCase, RWMixin +): def test_write(self): ftl, conf, rec, env = create_nkftl() @@ -2726,18 +3227,17 @@ def test_write(self): def main_proc(self, env, ftl, conf): # write different logical_blocks at the same time p1 = env.process(ftl.write_ext(Extent(0, 1))) - p2 = env.process(ftl.write_ext( - Extent(4 * conf.n_pages_per_block, 1))) + p2 = env.process(ftl.write_ext(Extent(4 * conf.n_pages_per_block, 1))) yield simpy.AllOf(env, [p1, p2]) self.assertTrue( - env.now == conf.page_prog_time() or - env.now == 2 * conf.page_prog_time(), - ) + env.now == conf.page_prog_time() or env.now == 2 * conf.page_prog_time(), + ) self.set_finished() + class TestLogicalBlockSerialization_SameLogicalPage(AssertFinishTestCase, RWMixin): def test_write(self): ftl, conf, rec, env = create_nkftl() @@ -2775,6 +3275,7 @@ def main_proc(self, env, ftl, conf): self.set_finished() + class TestLogicalBlockSerialization_WriteAndRead2Procs(AssertFinishTestCase, RWMixin): def test_write(self): ftl, conf, rec, env = create_nkftl() @@ -2785,22 +3286,18 @@ def test_write(self): def main_proc(self, env, ftl, conf): # write different logical_blocks at the same time extent1 = Extent(0, 1) - p1 = env.process(ftl.write_ext(extent1, - self.data_of_extent(extent1))) + p1 = env.process(ftl.write_ext(extent1, self.data_of_extent(extent1))) extent2 = Extent(1, 1) - p2 = env.process(ftl.write_ext(extent2, - self.data_of_extent(extent2))) + p2 = env.process(ftl.write_ext(extent2, self.data_of_extent(extent2))) yield simpy.AllOf(env, [p1, p2]) ret_data = yield env.process(ftl.read_ext(extent1)) - self.assertListEqual(ret_data, [str(x) - for x in self.data_of_extent(extent1)]) + self.assertListEqual(ret_data, [str(x) for x in self.data_of_extent(extent1)]) ret_data = yield env.process(ftl.read_ext(extent2)) - self.assertListEqual(ret_data, [str(x) - for x in self.data_of_extent(extent2)]) + self.assertListEqual(ret_data, [str(x) for x in self.data_of_extent(extent2)]) self.set_finished() @@ -2815,18 +3312,16 @@ def test_write(self): def main_proc(self, env, ftl, conf): # write different logical_blocks at the same time extent = Extent(0, 1) - yield env.process(ftl.write_ext(extent, - self.data_of_extent(extent))) + yield env.process(ftl.write_ext(extent, self.data_of_extent(extent))) ret_data = yield env.process(ftl.read_ext(extent)) - self.assertListEqual(ret_data, [str(x) - for x in self.data_of_extent(extent)]) + self.assertListEqual(ret_data, [str(x) for x in self.data_of_extent(extent)]) - self.assertEqual(env.now, conf.page_prog_time() + - conf.page_read_time()) + self.assertEqual(env.now, conf.page_prog_time() + conf.page_read_time()) self.set_finished() + class TestLogicalBlockSerialization_Discard(AssertFinishTestCase, RWMixin): def test(self): ftl, conf, rec, env = create_nkftl() @@ -2837,12 +3332,10 @@ def test(self): def main_proc(self, env, ftl, conf): # write different logical_blocks at the same time extent = Extent(0, 1) - yield env.process(ftl.write_ext(extent, - self.data_of_extent(extent))) + yield env.process(ftl.write_ext(extent, self.data_of_extent(extent))) ret_data = yield env.process(ftl.read_ext(extent)) - self.assertListEqual(ret_data, [str(x) - for x in self.data_of_extent(extent)]) + self.assertListEqual(ret_data, [str(x) for x in self.data_of_extent(extent)]) yield env.process(ftl.discard_ext(extent)) @@ -2852,12 +3345,12 @@ def main_proc(self, env, ftl, conf): self.assertEqual(found, False) ret_data = yield env.process(ftl.read_ext(extent)) - self.assertListEqual(ret_data, [None - for x in self.data_of_extent(extent)]) + self.assertListEqual(ret_data, [None for x in self.data_of_extent(extent)]) self.set_finished() -class WriteNCheckMixin(object): + +class WriteNCheckMixin: def write_and_check(self, ftl, extents, env): data_mirror = {} for ext in extents: @@ -2865,17 +3358,17 @@ def write_and_check(self, ftl, extents, env): self.update_data_mirror(data_mirror, ext, data) yield env.process(ftl.write_ext(ext, data)) - for lpn, data in data_mirror.items(): + for lpn, data in list(data_mirror.items()): ret = yield env.process(ftl.lba_read(lpn)) self.assertEqual(ret, data) def update_data_mirror(self, data_mirror, extent, data): - data_mirror.update(dict(zip(extent.lpn_iter(), data))) + data_mirror.update(dict(list(zip(extent.lpn_iter(), data)))) def random_data(self, extent): data = [] for lpn in extent.lpn_iter(): - d = str(lpn) + '.' + str(random.randint(0, 100)) + d = str(lpn) + "." + str(random.randint(0, 100)) data.append(d) return data @@ -2892,14 +3385,14 @@ def main_proc(self, env, ftl, conf): n = conf.n_pages_per_data_group() extents = [] for i in range(100): - start = random.randint(0, n-1) + start = random.randint(0, n - 1) cnt = random.randint(1, n - start) ext = Extent(start, cnt) - extents.append( ext ) + extents.append(ext) yield env.process(self.write_and_check(ftl, extents, env)) self.set_finished() - print 'end......' + print("end......") @unittest.skipUnless(TESTALL == True, "Takes too long") @@ -2915,10 +3408,10 @@ def main_proc(self, env, ftl, conf): n = int(conf.total_num_pages() * 0.6) extents = [] for i in range(10000): - start = random.randint(0, n-1) - cnt = max(1, int(random.randint(1, n - start) / 100)) + start = random.randint(0, n - 1) + cnt = max(1, int(old_div(random.randint(1, n - start), 100))) ext = Extent(start, cnt) - extents.append( ext ) + extents.append(ext) yield env.process(self.write_and_check(ftl, extents, env)) self.set_finished() @@ -2931,31 +3424,58 @@ def test(self): """ pk = create_gc() - gc, conf, block_pool, rec, oob, helper, \ - logmaptable, datablocktable, translator, \ - flashobj, simpy_env, des_flash = pk + ( + gc, + conf, + block_pool, + rec, + oob, + helper, + logmaptable, + datablocktable, + translator, + flashobj, + simpy_env, + des_flash, + ) = pk simpy_env.process(self.proc(pk)) simpy_env.run() def proc(self, pk): - gc, conf, block_pool, rec, oob, helper, \ - logmaptable, datablocktable, translator, \ - flashobj, simpy_env, des_flash = pk - - half_block_pages = int(conf.n_pages_per_block/2) + ( + gc, + conf, + block_pool, + rec, + oob, + helper, + logmaptable, + datablocktable, + translator, + flashobj, + simpy_env, + des_flash, + ) = pk + + half_block_pages = int(old_div(conf.n_pages_per_block, 2)) used_blocks, ppns = self.get_ppns_from_data_group( - conf, oob, block_pool, logmaptable, cnt=half_block_pages * 4, - dgn=0) + conf, oob, block_pool, logmaptable, cnt=half_block_pages * 4, dgn=0 + ) lbn1 = 1 lbn2 = 3 - lpns = self.page_ext(lbn1 * conf.n_pages_per_block + half_block_pages, half_block_pages) +\ - self.page_ext(lbn2 * conf.n_pages_per_block + half_block_pages, half_block_pages) +\ - self.page_ext(lbn1 * conf.n_pages_per_block, half_block_pages) +\ - self.page_ext(lbn2 * conf.n_pages_per_block, half_block_pages) - self.set_mappings(oob, block_pool, logmaptable, lpns, ppns, - translator) + lpns = ( + self.page_ext( + lbn1 * conf.n_pages_per_block + half_block_pages, half_block_pages + ) + + self.page_ext( + lbn2 * conf.n_pages_per_block + half_block_pages, half_block_pages + ) + + self.page_ext(lbn1 * conf.n_pages_per_block, half_block_pages) + + self.page_ext(lbn2 * conf.n_pages_per_block, half_block_pages) + ) + self.set_mappings(oob, block_pool, logmaptable, lpns, ppns, translator) self.assertEqual(len(used_blocks), 2) ######## start checking ######## @@ -3007,26 +3527,26 @@ def proc(self, pk): for i in range(half_block_pages): ppn = conf.block_off_to_page(pbn1, i) lpn = oob.translate_ppn_to_lpn(ppn) - correct_lpn = conf.block_off_to_page(lbn1, - (i + half_block_pages) % conf.n_pages_per_block) + correct_lpn = conf.block_off_to_page( + lbn1, (i + half_block_pages) % conf.n_pages_per_block + ) self.assertEqual(lpn, correct_lpn) for i in range(half_block_pages, conf.n_pages_per_block): ppn = conf.block_off_to_page(pbn1, i) lpn = oob.translate_ppn_to_lpn(ppn) - correct_lpn = conf.block_off_to_page(lbn2, - i % conf.n_pages_per_block) + correct_lpn = conf.block_off_to_page(lbn2, i % conf.n_pages_per_block) self.assertEqual(lpn, correct_lpn) for i in range(half_block_pages): ppn = conf.block_off_to_page(pbn2, i) lpn = oob.translate_ppn_to_lpn(ppn) - correct_lpn = conf.block_off_to_page(lbn1, - i % conf.n_pages_per_block) + correct_lpn = conf.block_off_to_page(lbn1, i % conf.n_pages_per_block) self.assertEqual(lpn, correct_lpn) for i in range(half_block_pages, conf.n_pages_per_block): ppn = conf.block_off_to_page(pbn2, i) lpn = oob.translate_ppn_to_lpn(ppn) - correct_lpn = conf.block_off_to_page(lbn2, - (i-half_block_pages) % conf.n_pages_per_block) + correct_lpn = conf.block_off_to_page( + lbn2, (i - half_block_pages) % conf.n_pages_per_block + ) self.assertEqual(lpn, correct_lpn) # block pool @@ -3092,7 +3612,6 @@ def proc(self, pk): correct_lpn = conf.block_off_to_page(lbn2, i) self.assertEqual(lpn, correct_lpn) - # block pool self.assertIn(pbn1, block_pool.freeblocks) self.assertIn(pbn2, block_pool.freeblocks) @@ -3109,31 +3628,58 @@ def test(self): """ pk = create_gc() - gc, conf, block_pool, rec, oob, helper, \ - logmaptable, datablocktable, translator, \ - flashobj, simpy_env, des_flash = pk + ( + gc, + conf, + block_pool, + rec, + oob, + helper, + logmaptable, + datablocktable, + translator, + flashobj, + simpy_env, + des_flash, + ) = pk simpy_env.process(self.proc(pk)) simpy_env.run() def proc(self, pk): - gc, conf, block_pool, rec, oob, helper, \ - logmaptable, datablocktable, translator, \ - flashobj, simpy_env, des_flash = pk - - half_block_pages = int(conf.n_pages_per_block/2) + ( + gc, + conf, + block_pool, + rec, + oob, + helper, + logmaptable, + datablocktable, + translator, + flashobj, + simpy_env, + des_flash, + ) = pk + + half_block_pages = int(old_div(conf.n_pages_per_block, 2)) used_blocks, ppns = self.get_ppns_from_data_group( - conf, oob, block_pool, logmaptable, cnt=half_block_pages * 4, - dgn=0) + conf, oob, block_pool, logmaptable, cnt=half_block_pages * 4, dgn=0 + ) lbn1 = 1 lbn2 = 3 - lpns = self.page_ext(lbn1 * conf.n_pages_per_block + half_block_pages, half_block_pages) +\ - self.page_ext(lbn2 * conf.n_pages_per_block + half_block_pages, half_block_pages) +\ - self.page_ext(lbn1 * conf.n_pages_per_block, half_block_pages) +\ - self.page_ext(lbn2 * conf.n_pages_per_block, half_block_pages) - self.set_mappings(oob, block_pool, logmaptable, lpns, ppns, - translator) + lpns = ( + self.page_ext( + lbn1 * conf.n_pages_per_block + half_block_pages, half_block_pages + ) + + self.page_ext( + lbn2 * conf.n_pages_per_block + half_block_pages, half_block_pages + ) + + self.page_ext(lbn1 * conf.n_pages_per_block, half_block_pages) + + self.page_ext(lbn2 * conf.n_pages_per_block, half_block_pages) + ) + self.set_mappings(oob, block_pool, logmaptable, lpns, ppns, translator) self.assertEqual(len(used_blocks), 2) ######## start checking ######## @@ -3185,26 +3731,26 @@ def proc(self, pk): for i in range(half_block_pages): ppn = conf.block_off_to_page(pbn1, i) lpn = oob.translate_ppn_to_lpn(ppn) - correct_lpn = conf.block_off_to_page(lbn1, - (i + half_block_pages) % conf.n_pages_per_block) + correct_lpn = conf.block_off_to_page( + lbn1, (i + half_block_pages) % conf.n_pages_per_block + ) self.assertEqual(lpn, correct_lpn) for i in range(half_block_pages, conf.n_pages_per_block): ppn = conf.block_off_to_page(pbn1, i) lpn = oob.translate_ppn_to_lpn(ppn) - correct_lpn = conf.block_off_to_page(lbn2, - i % conf.n_pages_per_block) + correct_lpn = conf.block_off_to_page(lbn2, i % conf.n_pages_per_block) self.assertEqual(lpn, correct_lpn) for i in range(half_block_pages): ppn = conf.block_off_to_page(pbn2, i) lpn = oob.translate_ppn_to_lpn(ppn) - correct_lpn = conf.block_off_to_page(lbn1, - i % conf.n_pages_per_block) + correct_lpn = conf.block_off_to_page(lbn1, i % conf.n_pages_per_block) self.assertEqual(lpn, correct_lpn) for i in range(half_block_pages, conf.n_pages_per_block): ppn = conf.block_off_to_page(pbn2, i) lpn = oob.translate_ppn_to_lpn(ppn) - correct_lpn = conf.block_off_to_page(lbn2, - (i-half_block_pages) % conf.n_pages_per_block) + correct_lpn = conf.block_off_to_page( + lbn2, (i - half_block_pages) % conf.n_pages_per_block + ) self.assertEqual(lpn, correct_lpn) # block pool @@ -3213,9 +3759,11 @@ def proc(self, pk): ########### full merge 1 ############## p1 = simpy_env.process( - gc.clean_log_block(log_pbn=pbn1, data_group_no=0, tag="")) + gc.clean_log_block(log_pbn=pbn1, data_group_no=0, tag="") + ) p2 = simpy_env.process( - gc.clean_log_block(log_pbn=pbn2, data_group_no=0, tag="")) + gc.clean_log_block(log_pbn=pbn2, data_group_no=0, tag="") + ) yield simpy.AllOf(simpy_env, [p1, p2]) ########### check ##################### @@ -3272,7 +3820,6 @@ def proc(self, pk): correct_lpn = conf.block_off_to_page(lbn2, i) self.assertEqual(lpn, correct_lpn) - # block pool self.assertIn(pbn1, block_pool.freeblocks) self.assertIn(pbn2, block_pool.freeblocks) @@ -3288,6 +3835,7 @@ class TestConcurrency_WriteNGC(AssertFinishTestCase, WriteNCheckMixin): Trigger GC and Write the logical space, the write and gc may mess with each other. """ + def test_write(self): ftl, conf, rec, env = create_nkftl() @@ -3298,19 +3846,19 @@ def main_proc(self, env, ftl, conf): n = conf.n_pages_per_data_group() extents = [] for i in range(10): - start = 1 * n + random.randint(0, 2*n-1) + start = 1 * n + random.randint(0, 2 * n - 1) cnt = random.randint(1, 3 * n - start) ext = Extent(start, cnt) - extents.append( ext ) + extents.append(ext) yield env.process(self.write_and_check(ftl, extents, env)) # gc and write extents = [] for i in range(10): - start = 1 * n + random.randint(0, 2*n-1) + start = 1 * n + random.randint(0, 2 * n - 1) cnt = random.randint(1, 3 * n - start) ext = Extent(start, cnt) - extents.append( ext ) + extents.append(ext) p_gc = env.process(ftl.garbage_collector.clean()) p_write = env.process(self.write_and_check(ftl, extents, env)) @@ -3332,7 +3880,7 @@ def main_proc(self, env, ftl, conf): for i in range(100): # print i if i % 1000 == 0: - print i + print(i) ext = self.random_extent(conf) op = self.random_op() yield env.process(self.operate(env, ftl, conf, op, ext)) @@ -3345,24 +3893,24 @@ def main_proc(self, env, ftl, conf): self.set_finished() def random_extent(self, conf): - n = int(conf.total_num_pages() * 0.8) # don't use the full logical space - start = random.randint(0, n-1) - cnt = max(1, int(random.randint(1, n - start) / 100)) + n = int(conf.total_num_pages() * 0.8) # don't use the full logical space + start = random.randint(0, n - 1) + cnt = max(1, int(old_div(random.randint(1, n - start), 100))) ext = Extent(start, cnt) return ext def random_op(self): - return random.choice(['read', 'write', 'discard']) + return random.choice(["read", "write", "discard"]) def operate(self, env, ftl, conf, op, extent): # print str(extent) - if op == 'write': + if op == "write": extent_data = random_data_of_extent(extent) self.write_mirror(extent, extent_data) yield env.process(ftl.write_ext(extent, extent_data)) - elif op == 'read': + elif op == "read": yield env.process(ftl.read_ext(extent)) - elif op == 'discard': + elif op == "discard": self.discard_mirror(extent) yield env.process(ftl.discard_ext(extent)) @@ -3378,9 +3926,8 @@ def discard_mirror(self, extent): pass def check_mirror(self, env, ftl, conf): - for lpn, data in self.data_mirror.items(): - data_read = yield env.process( - ftl.read_ext(Extent(lpn, 1))) + for lpn, data in list(self.data_mirror.items()): + data_read = yield env.process(ftl.read_ext(Extent(lpn, 1))) self.assertEqual(data, data_read[0]) @@ -3412,25 +3959,26 @@ def op_proc(self, env, ftl, conf): yield env.process(ftl.clean(forced=False)) # yield env.process(self.check_mirror(env, ftl, conf)) + def random_extent(self, conf): - n = int(conf.total_num_pages() * 0.8) # don't use the full logical space - start = random.randint(0, n-1) - cnt = max(1, int(random.randint(1, n - start) / 100)) + n = int(conf.total_num_pages() * 0.8) # don't use the full logical space + start = random.randint(0, n - 1) + cnt = max(1, int(old_div(random.randint(1, n - start), 100))) ext = Extent(start, cnt) return ext def random_op(self): - return random.choice(['read', 'write', 'discard']) + return random.choice(["read", "write", "discard"]) def operate(self, env, ftl, conf, op, extent): # print str(extent) - if op == 'write': + if op == "write": extent_data = random_data_of_extent(extent) yield env.process(ftl.write_ext(extent, extent_data)) self.write_mirror(extent, extent_data) - elif op == 'read': + elif op == "read": yield env.process(ftl.read_ext(extent)) - elif op == 'discard': + elif op == "discard": self.discard_mirror(extent) yield env.process(ftl.discard_ext(extent)) @@ -3446,11 +3994,11 @@ def discard_mirror(self, extent): pass def check_mirror(self, env, ftl, conf): - for lpn, data in self.data_mirror.items(): - data_read = yield env.process( - ftl.read_ext(Extent(lpn, 1))) + for lpn, data in list(self.data_mirror.items()): + data_read = yield env.process(ftl.read_ext(Extent(lpn, 1))) self.assertEqual(data, data_read[0]) + # @unittest.skip("Take a little too long") class TestConcurrency_DataIntegrity(AssertFinishTestCase): def test_write(self): @@ -3491,16 +4039,16 @@ def op_proc(self, env, ftl, conf): yield env.process(ftl.clean(forced=False)) def random_op(self): - return random.choice(['read', 'write', 'discard']) + return random.choice(["read", "write", "discard"]) def operate(self, env, ftl, conf, op, extent): - if op == 'write': + if op == "write": extent_data = random_data_of_extent(extent) yield env.process(ftl.write_ext(extent, extent_data)) self.write_mirror(extent, extent_data) - elif op == 'read': + elif op == "read": yield env.process(ftl.read_ext(extent)) - elif op == 'discard': + elif op == "discard": self.discard_mirror(extent) yield env.process(ftl.discard_ext(extent)) @@ -3516,14 +4064,14 @@ def discard_mirror(self, extent): pass def check_mirror(self, env, ftl, conf): - for lpn, data in self.data_mirror.items(): - data_read = yield env.process( - ftl.read_ext(Extent(lpn, 1))) + for lpn, data in list(self.data_mirror.items()): + data_read = yield env.process(ftl.read_ext(Extent(lpn, 1))) self.assertEqual(data, data_read[0]) def init_non_overlap_exts(self, conf): self.nonoverlap_exts = self.non_overlap_extents( - int(conf.total_num_pages() * 0.6), conf.n_pages_per_block) + int(conf.total_num_pages() * 0.6), conf.n_pages_per_block + ) random.shuffle(self.nonoverlap_exts) self.ext_i = 0 @@ -3550,6 +4098,7 @@ def non_overlap_extents(self, n, n_pages_per_block): return extents + class TestBlockIter(unittest.TestCase): def test_1(self): ftl, conf, rec, env = create_nkftl() @@ -3565,19 +4114,20 @@ def test_1(self): ret = list(ftl._block_iter_of_extent(Extent(0, n_pages_per_block + 1))) self.assertListEqual(ret, [0, 1]) - ret = list(ftl._block_iter_of_extent( - Extent(n_pages_per_block + 1, 2 * n_pages_per_block))) + ret = list( + ftl._block_iter_of_extent( + Extent(n_pages_per_block + 1, 2 * n_pages_per_block) + ) + ) self.assertListEqual(ret, [1, 2, 3]) - # Add test without lpn overlap + def main(): unittest.main() -if __name__ == '__main__': - main() - - +if __name__ == "__main__": + main() diff --git a/wiscsee/tests/test_tagblockpool.py b/wiscsee/tests/test_tagblockpool.py index 9c23dbe..d0e18c6 100644 --- a/wiscsee/tests/test_tagblockpool.py +++ b/wiscsee/tests/test_tagblockpool.py @@ -2,8 +2,8 @@ from wiscsim.tagblockpool import * -TDATA = 'TDATA' -TTRANS = 'TTRANS' +TDATA = "TDATA" +TTRANS = "TTRANS" NPAGESPERBLOCK = 64 @@ -58,7 +58,7 @@ def test_initial_count(self): pool = TagBlockPool(100, [TDATA, TTRANS]) for i in range(100): - cnt = pool.get_erasure_count(blocknum = i) + cnt = pool.get_erasure_count(blocknum=i) self.assertEqual(cnt, 0) def test_count_of_blocks(self): @@ -66,16 +66,16 @@ def test_count_of_blocks(self): block = pool.pick_and_move(TFREE, TDATA) - cnt = pool.get_erasure_count(blocknum = block) + cnt = pool.get_erasure_count(blocknum=block) self.assertEqual(cnt, 0) pool.change_tag(block, TDATA, TFREE) - cnt = pool.get_erasure_count(blocknum = block) + cnt = pool.get_erasure_count(blocknum=block) self.assertEqual(cnt, 1) pool.change_tag(block, TFREE, TDATA) pool.change_tag(block, TDATA, TFREE) - cnt = pool.get_erasure_count(blocknum = block) + cnt = pool.get_erasure_count(blocknum=block) self.assertEqual(cnt, 2) def test_least_erased_block(self): @@ -106,13 +106,14 @@ def test_most_erased_block(self): pool.change_tag(blocknum, TFREE, TDATA) pool.change_tag(blocknum, TDATA, TFREE) - most = pool.get_least_or_most_erased_block(TFREE, choice='most') - self.assertIn(most, range(9)) + most = pool.get_least_or_most_erased_block(TFREE, choice="most") + self.assertIn(most, list(range(9))) - most = pool.get_least_or_most_erased_blocks(TFREE, - choice=MOST_ERASED, nblocks=2) - self.assertIn(most[0], range(9)) - self.assertIn(most[1], range(9)) + most = pool.get_least_or_most_erased_blocks( + TFREE, choice=MOST_ERASED, nblocks=2 + ) + self.assertIn(most[0], list(range(9))) + self.assertIn(most[1], list(range(9))) least = pool.get_least_or_most_erased_block(TDATA) self.assertEqual(least, None) @@ -155,7 +156,7 @@ def test_getting_count_distribution(self): self.assertEqual(dist[0], 1) self.assertEqual(dist[2], 2) self.assertEqual(dist[3], 2) - print dist + print(dist) class TestBlockPoolWithCurBlocks(unittest.TestCase): @@ -248,8 +249,9 @@ def test_next_ppns_from_cur_block_all(self): allppns = [] while remaining > 0: - ppnlist = pool.next_ppns_from_cur_block(n=remaining, tag=TDATA, - block_index=0) + ppnlist = pool.next_ppns_from_cur_block( + n=remaining, tag=TDATA, block_index=0 + ) allppns.extend(ppnlist) if len(ppnlist) == 0: @@ -269,8 +271,6 @@ def test_next_ppns_context(self): pool = BlockPoolWithCurBlocks(100, [TDATA], 8) - - class TestCurrentBlock(unittest.TestCase): def test_init(self): cur_block = CurrentBlock(64, 1) @@ -306,8 +306,6 @@ def test_num_free(self): def main(): unittest.main() -if __name__ == '__main__': - main() - - +if __name__ == "__main__": + main() diff --git a/wiscsee/tests/test_workflow.py b/wiscsee/tests/test_workflow.py index 3188b0f..83b480d 100644 --- a/wiscsee/tests/test_workflow.py +++ b/wiscsee/tests/test_workflow.py @@ -14,34 +14,34 @@ def create_config(): - #conf = wiscsim.dftldes.Config() + # conf = wiscsim.dftldes.Config() conf = wiscsim.nkftl2.Config() - conf['SSDFramework']['ncq_depth'] = 1 + conf["SSDFramework"]["ncq_depth"] = 1 - conf['flash_config']['n_pages_per_block'] = 256 - conf['flash_config']['n_blocks_per_plane'] = 2048 - conf['flash_config']['n_planes_per_chip'] = 4 - conf['flash_config']['n_chips_per_package'] = 4 - conf['flash_config']['n_packages_per_channel'] = 1 - conf['flash_config']['n_channels_per_dev'] = 8 + conf["flash_config"]["n_pages_per_block"] = 256 + conf["flash_config"]["n_blocks_per_plane"] = 2048 + conf["flash_config"]["n_planes_per_chip"] = 4 + conf["flash_config"]["n_chips_per_package"] = 4 + conf["flash_config"]["n_packages_per_channel"] = 1 + conf["flash_config"]["n_channels_per_dev"] = 8 # set ftl - conf['do_not_check_gc_setting'] = True + conf["do_not_check_gc_setting"] = True conf.GC_high_threshold_ratio = 0.96 conf.GC_low_threshold_ratio = 0 - conf['enable_simulation'] = True + conf["enable_simulation"] = True - utils.set_exp_metadata(conf, save_data = False, - expname = 'test_expname', - subexpname = 'test_subexpname') + utils.set_exp_metadata( + conf, save_data=False, expname="test_expname", subexpname="test_subexpname" + ) - conf['ftl_type'] = 'nkftl2' - conf['simulator_class'] = 'SimulatorDESNew' + conf["ftl_type"] = "nkftl2" + conf["simulator_class"] = "SimulatorDESNew" logicsize_mb = 16 - #conf.n_cache_entries = conf.n_mapping_entries_per_page * 16 - #conf.set_flash_num_blocks_by_bytes(int(logicsize_mb * 2**20 * 1.28)) + # conf.n_cache_entries = conf.n_mapping_entries_per_page * 16 + # conf.set_flash_num_blocks_by_bytes(int(logicsize_mb * 2**20 * 1.28)) utils.runtime_update(conf) @@ -50,15 +50,16 @@ def create_config(): def on_fs_config(conf): # environment - conf['device_path'] = "/dev/loop0" - conf['dev_size_mb'] = 16 - conf['filesystem'] = "ext4" - conf["n_online_cpus"] = 'all' + conf["device_path"] = "/dev/loop0" + conf["dev_size_mb"] = 16 + conf["filesystem"] = "ext4" + conf["n_online_cpus"] = "all" - conf['linux_ncq_depth'] = 31 + conf["linux_ncq_depth"] = 31 # workload - conf['workload_class'] = 'SimpleRandReadWrite' + conf["workload_class"] = "SimpleRandReadWrite" + class TestWorkflow(unittest.TestCase): def test_init(self): @@ -67,8 +68,8 @@ def test_init(self): def test_save_conf(self): conf = create_config() - conf['result_dir'] = '/tmp/' - jsonpath = os.path.join(conf['result_dir'], 'config.json') + conf["result_dir"] = "/tmp/" + jsonpath = os.path.join(conf["result_dir"], "config.json") if os.path.exists(jsonpath): os.remove(jsonpath) @@ -78,12 +79,11 @@ def test_save_conf(self): self.assertTrue(os.path.exists(jsonpath)) - def test_onfs_workload(self): conf = create_config() on_fs_config(conf) - datapath = os.path.join(conf["fs_mount_point"], 'datafile') + datapath = os.path.join(conf["fs_mount_point"], "datafile") if os.path.exists(datapath): os.remove(datapath) @@ -97,11 +97,12 @@ def test_simulation(self): ctrl_event = ControlEvent(OP_ENABLE_RECORDER) events = [] - events += [Event(512, 0, OP_WRITE, i*4096, 4096) for i in reversed(range(257))] - + events += [ + Event(512, 0, OP_WRITE, i * 4096, 4096) for i in reversed(list(range(257))) + ] wf = Workflow(conf) - sim = wf.run_simulator([ctrl_event]+events) + sim = wf.run_simulator([ctrl_event] + events) # dftl print # print([row for row in sim.ssd.ftl._mappings._lpn_table.rows() if row.dirty]) # nkftl print @@ -109,17 +110,23 @@ def test_simulation(self): # for dgn, log_group in sim.ssd.ftl.log_mapping_table.log_group_info.items() \ # for lpn, ppn in log_group._page_map]) # print(conf) - print([log_group._page_map \ - for dgn, log_group in sim.ssd.ftl.log_mapping_table.log_group_info.items() ]) + print( + [ + log_group._page_map + for dgn, log_group in list( + sim.ssd.ftl.log_mapping_table.log_group_info.items() + ) + ] + ) print(sim.ssd.ftl.data_block_mapping_table.logical_to_physical_block) def test_on_fs_run_and_sim(self): conf = create_config() on_fs_config(conf) - conf['enable_blktrace'] = True + conf["enable_blktrace"] = True - datapath = os.path.join(conf["fs_mount_point"], 'datafile') + datapath = os.path.join(conf["fs_mount_point"], "datafile") if os.path.exists(datapath): os.remove(datapath) @@ -129,7 +136,5 @@ def test_on_fs_run_and_sim(self): self.assertTrue(os.path.exists(datapath)) - -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() - diff --git a/wiscsee/utilities/utils.py b/wiscsee/utilities/utils.py index b742bbe..482f01a 100644 --- a/wiscsee/utilities/utils.py +++ b/wiscsee/utilities/utils.py @@ -14,16 +14,17 @@ from pyreuse.helpers import table_to_str + def shcmd(cmd, ignore_error=False): - print 'Doing:', cmd + print(("Doing:", cmd)) ret = subprocess.call(cmd, shell=True) - print 'Returned', ret, cmd + print(("Returned", ret, cmd)) if ignore_error == False and ret != 0: - raise RuntimeError("Failed to execute {}. Return code:{}".format( - cmd, ret)) + raise RuntimeError(f"Failed to execute {cmd}. Return code:{ret}") return ret -def run_and_get_output(cmd, shell = False): + +def run_and_get_output(cmd, shell=False): output = [] cmd = shlex.split(cmd) p = subprocess.Popen(cmd, shell=shell, stdout=subprocess.PIPE) @@ -31,8 +32,10 @@ def run_and_get_output(cmd, shell = False): return p.stdout.readlines() + class cd: """Context manager for changing the current working directory""" + def __init__(self, newPath): self.newPath = newPath @@ -43,6 +46,7 @@ def __enter__(self): def __exit__(self, etype, value, traceback): os.chdir(self.savedPath) + ######################################################## # table = [ # {'col1':data, 'col2':data, ..}, @@ -50,31 +54,36 @@ def __exit__(self, etype, value, traceback): # ... # ] def table_to_file(table, filepath, adddic=None, width=32): - 'save table to a file with additional columns' - with open(filepath, 'w') as f: + "save table to a file with additional columns" + with open(filepath, "w") as f: if len(table) == 0: return - f.write( table_to_str(table, adddic=adddic, width=width) ) + f.write(table_to_str(table, adddic=adddic, width=width)) + def load_json(fpath): - decoded = json.load(open(fpath, 'r')) + decoded = json.load(open(fpath)) return decoded + def dump_json(dic, file_path): with open(file_path, "w") as f: json.dump(dic, f, indent=4) + def prepare_dir_for_path(path): "create parent dirs for path if necessary" dirpath = os.path.dirname(path) if not os.path.exists(dirpath): os.makedirs(dirpath) + def prepare_dir(dirpath): "create parent dirs for path if necessary" if not os.path.exists(dirpath): os.makedirs(dirpath) + def ParameterCombinations(parameter_dict): """ Get all the cominbation of the values from each key @@ -90,28 +99,35 @@ def ParameterCombinations(parameter_dict): ] """ d = parameter_dict - return [dict(zip(d, v)) for v in itertools.product(*d.values())] + return [dict(list(zip(d, v))) for v in itertools.product(*list(d.values()))] + def debug_decor(function): def wrapper(*args, **kwargs): ret = function(*args, **kwargs) - print function.__name__, args, kwargs, 'ret:', ret + print((function.__name__, args, kwargs, "ret:", ret)) return ret + return wrapper + def breakpoint(): - import pdb; pdb.set_trace() + import pdb + + pdb.set_trace() def linux_kernel_version(): - kernel_ver = run_and_get_output('uname -r')[0].strip() + kernel_ver = run_and_get_output("uname -r")[0].strip() return kernel_ver + def set_vm(name, value): filepath = os.path.join("/proc/sys/vm/", name) - f = open(filepath, 'w') + f = open(filepath, "w") f.write(str(value)) + def set_vm_default(): set_vm("dirty_background_bytes", 0) set_vm("dirty_background_ratio", 10) @@ -120,26 +136,27 @@ def set_vm_default(): set_vm("dirty_expire_centisecs", 3000) set_vm("dirty_writeback_centisecs", 500) + def set_linux_ncq_depth(devname, depth): - filepath = "/sys/block/{}/device/queue_depth".format(devname) - shcmd("echo {depth} > {filepath}".format( - depth = depth, filepath = filepath)) + filepath = f"/sys/block/{devname}/device/queue_depth" + shcmd(f"echo {depth} > {filepath}") + def get_linux_ncq_depth(devname): - filepath = "/sys/block/{}/device/queue_depth".format(devname) - with open(filepath, 'r') as f: + filepath = f"/sys/block/{devname}/device/queue_depth" + with open(filepath) as f: line = f.readline() return int(line.strip()) + def set_linux_io_scheduler(devname, scheduler_name): - filepath = "/sys/block/{}/queue/scheduler".format(devname) - shcmd("echo {scheduler_name} > {filepath}".format( - scheduler_name = scheduler_name, - filepath = filepath)) + filepath = f"/sys/block/{devname}/queue/scheduler" + shcmd(f"echo {scheduler_name} > {filepath}") + def get_linux_io_scheduler(devname): - filepath = "/sys/block/{}/queue/scheduler".format(devname) - with open(filepath, 'r') as f: + filepath = f"/sys/block/{devname}/queue/scheduler" + with open(filepath) as f: line = f.readline() # return re.search(r'\[(\w+)\]', line).group(1) if not m is None: @@ -147,23 +164,26 @@ def get_linux_io_scheduler(devname): else: return line.strip() + def runtime_update(conf): """ This function has to be called before running each treatment. """ - conf['time'] = time.strftime("%m-%d-%H-%M-%S", time.localtime()) - conf['hash'] = hash(str(conf)) - if conf.has_key('filesystem') and conf['filesystem'] != None: - fs = str(conf['filesystem']) + conf["time"] = time.strftime("%m-%d-%H-%M-%S", time.localtime()) + conf["hash"] = hash(str(conf)) + if "filesystem" in conf and conf["filesystem"] != None: + fs = str(conf["filesystem"]) else: - fs = 'fsnotset' - conf['result_dir'] = "{targetdir}/{expname}/{subexpname}-{unique}".format( - targetdir = conf['targetdir'], expname = conf['expname'], - subexpname = conf['subexpname'], - unique = '-'.join((fs, conf['time'], str(conf['hash'])))) + fs = "fsnotset" + conf["result_dir"] = "{targetdir}/{expname}/{subexpname}-{unique}".format( + targetdir=conf["targetdir"], + expname=conf["expname"], + subexpname=conf["subexpname"], + unique="-".join((fs, conf["time"], str(conf["hash"]))), + ) -def choose_exp_metadata(default_conf, interactive = True): +def choose_exp_metadata(default_conf, interactive=True): """ This function will return a dictionary containing a few things relating to result dir. You can update the experiment configuration by @@ -177,125 +197,135 @@ def choose_exp_metadata(default_conf, interactive = True): later for each experiment. """ conf = {} - result_dir = '/tmp/results' + result_dir = "/tmp/results" # result_dir = '/users/jhe/results' # result_dir = '/mnt/ramdisk/results' if interactive == True: - toresult = raw_input('Save this experiments to {}? (y/n)'.format(result_dir)) + toresult = eval(input(f"Save this experiments to {result_dir}? (y/n)")) else: - toresult = 'n' - if toresult.lower() == 'y': + toresult = "n" + if toresult.lower() == "y": targetdir = result_dir - expname = raw_input('Enter expname ({}):'.format(default_conf['expname'])) - if expname.strip() != '': - conf['expname'] = expname + expname = eval(input("Enter expname ({}):".format(default_conf["expname"]))) + if expname.strip() != "": + conf["expname"] = expname else: - conf['expname'] = default_conf['expname'] + conf["expname"] = default_conf["expname"] - subexpname = raw_input('Enter subexpname ({}):'.format( - default_conf['subexpname'])) - if subexpname.strip() != '': - conf['subexpname'] = subexpname + subexpname = eval( + input("Enter subexpname ({}):".format(default_conf["subexpname"])) + ) + if subexpname.strip() != "": + conf["subexpname"] = subexpname else: - conf['subexpname'] = default_conf['subexpname'] + conf["subexpname"] = default_conf["subexpname"] else: - targetdir = '/tmp/resulttmp' - conf['expname'] = default_conf['expname'] - conf['subexpname'] = default_conf['subexpname'] + targetdir = "/tmp/resulttmp" + conf["expname"] = default_conf["expname"] + conf["subexpname"] = default_conf["subexpname"] - conf['targetdir'] = targetdir + conf["targetdir"] = targetdir return conf + def set_exp_metadata(conf, save_data, expname, subexpname): if save_data == True: - targetdir = '/tmp/results' + targetdir = "/tmp/results" else: - targetdir = '/tmp/tmpresults' + targetdir = "/tmp/tmpresults" + + conf["targetdir"] = targetdir - conf['targetdir'] = targetdir + conf["expname"] = expname + conf["subexpname"] = subexpname - conf['expname'] = expname - conf['subexpname'] = subexpname def chain_items_as_str(iterator): - return '.'.join([str(x) for x in iterator]) + return ".".join([str(x) for x in iterator]) + def chain_items_as_filename(iterator): s = chain_items_as_str(iterator) return str_as_filename(s) + def str_as_filename(s): """ valid_chars '-_.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789' """ - valid_chars = "-_.%s%s" % (string.ascii_letters, string.digits) - return ''.join(c for c in s if c in valid_chars) + valid_chars = f"-_.{string.ascii_letters}{string.digits}" + return "".join(c for c in s if c in valid_chars) def assert_multiple(n, divider): "n is multiple of divider" - assert n % divider == 0, "{} is not mutliple of {}".format(n, divider) + assert n % divider == 0, f"{n} is not mutliple of {divider}" def get_expname(): - ret = raw_input("Enter expname (default-expname):") + ret = eval(input("Enter expname (default-expname):")) if ret == "": return "default-expname" else: return ret + def str_as_filename(s): """ valid_chars '-_.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789' """ - valid_chars = "-_.%s%s" % (string.ascii_letters, string.digits) - return ''.join(c for c in s if c in valid_chars) + valid_chars = f"-_.{string.ascii_letters}{string.digits}" + return "".join(c for c in s if c in valid_chars) + def drop_caches(): subprocess.call("sync", shell=True) cmd = "echo 3 > /proc/sys/vm/drop_caches" subprocess.call(cmd, shell=True) + def invoke_f2fs_gc(mountpoint, sync, n=1): - binpath = './bin/forcef2fsgc' + binpath = "./bin/forcef2fsgc" if not os.path.exists(binpath): - raise RuntimeError("{} does not exist. To compile, do 'make f2fsgc'"\ - .format(binpath)) + raise RuntimeError(f"{binpath} does not exist. To compile, do 'make f2fsgc'") cmd = [binpath, mountpoint, str(sync), str(n)] ret = subprocess.call(cmd) return ret + def disable_ext4_journal(conf): - if '^has_journal' in conf['ext4']['make_opts']['-O']: - conf['ext4']['make_opts']['-O'].remove('^has_journal') + if "^has_journal" in conf["ext4"]["make_opts"]["-O"]: + conf["ext4"]["make_opts"]["-O"].remove("^has_journal") - if 'has_journal' in conf['ext4']['make_opts']['-O']: - conf['ext4']['make_opts']['-O'].remove('has_journal') + if "has_journal" in conf["ext4"]["make_opts"]["-O"]: + conf["ext4"]["make_opts"]["-O"].remove("has_journal") - conf['ext4']['make_opts']['-O'].append('^has_journal') + conf["ext4"]["make_opts"]["-O"].append("^has_journal") try: - del conf['mnt_opts']['ext4']['data'] + del conf["mnt_opts"]["ext4"]["data"] except KeyError: pass + def enable_ext4_journal(conf): - if '^has_journal' in conf['ext4']['make_opts']['-O']: - conf['ext4']['make_opts']['-O'].remove('^has_journal') + if "^has_journal" in conf["ext4"]["make_opts"]["-O"]: + conf["ext4"]["make_opts"]["-O"].remove("^has_journal") - if 'has_journal' in conf['ext4']['make_opts']['-O']: - conf['ext4']['make_opts']['-O'].remove('has_journal') + if "has_journal" in conf["ext4"]["make_opts"]["-O"]: + conf["ext4"]["make_opts"]["-O"].remove("has_journal") - conf['ext4']['make_opts']['-O'].append('has_journal') + conf["ext4"]["make_opts"]["-O"].append("has_journal") try: - del conf['mnt_opts']['ext4']['data'] + del conf["mnt_opts"]["ext4"]["data"] except KeyError: pass + def replicate_parameters(parameter_combs, n): result = [] for i in range(n): @@ -304,8 +334,7 @@ def replicate_parameters(parameter_combs, n): return result - -def get_dir_size(start_path = '.'): +def get_dir_size(start_path="."): total_size = 0 for dirpath, dirnames, filenames in os.walk(start_path): for f in filenames: @@ -318,11 +347,12 @@ def group_to_batches(mylist, cnt_per_chunk): n = len(mylist) chunks = [] for i in range(0, n, cnt_per_chunk): - chunk = mylist[i:(i+cnt_per_chunk)] + chunk = mylist[i : (i + cnt_per_chunk)] chunks.append(chunk) return chunks + def top_or_bottom_total(counter, need_nblocks, choice): """ Get the top x percent used blocks's total erase count @@ -337,15 +367,15 @@ def top_or_bottom_total(counter, need_nblocks, choice): list(top_or_bottom_total(c, 3, 'bottom')), [3, 3]) """ - if choice == 'bottom': + if choice == "bottom": sorted_keys = sorted(counter.keys()) - elif choice == 'top': - sorted_keys = sorted(counter.keys(), reverse=True) + elif choice == "top": + sorted_keys = sorted(list(counter.keys()), reverse=True) table = [] for k in sorted_keys: v = counter[k] - table.append( (k, v) ) + table.append((k, v)) total_blocks = 0 total_erase = 0 @@ -360,5 +390,3 @@ def top_or_bottom_total(counter, need_nblocks, choice): break return total_erase, total_blocks - - diff --git a/wiscsee/wiscsim/__init__.py b/wiscsee/wiscsim/__init__.py index 1641f93..4d63632 100644 --- a/wiscsee/wiscsim/__init__.py +++ b/wiscsee/wiscsim/__init__.py @@ -1,4 +1,4 @@ -import simulator +from . import simulator from utilities import utils -import ssdframework -import hostevent +from . import ssdframework +from . import hostevent diff --git a/wiscsee/wiscsim/bitmap.py b/wiscsee/wiscsim/bitmap.py index 0879973..8a929bc 100644 --- a/wiscsee/wiscsim/bitmap.py +++ b/wiscsee/wiscsim/bitmap.py @@ -1,17 +1,21 @@ import bitarray import config -class FlashBitmap2(object): + +class FlashBitmap2: "Using two bit to represent state of a page" - ERASED, VALID, INVALID = (bitarray.bitarray('00'), - bitarray.bitarray('01'), bitarray.bitarray('10')) + + ERASED, VALID, INVALID = ( + bitarray.bitarray("00"), + bitarray.bitarray("01"), + bitarray.bitarray("10"), + ) def __init__(self, conf): if not isinstance(conf, config.Config): - raise TypeError("conf is not conf.Config. it is {}". - format(type(conf).__name__)) + raise TypeError(f"conf is not conf.Config. it is {type(conf).__name__}") - self.conf = conf + self.conf = conf # We use two bits to record state of a page so that # we will be able to record ERASED state @@ -121,12 +125,9 @@ def page_state_human(self, pagenum): elif state == self.ERASED: return "ERASED" else: - raise RuntimeError("page {} state is not recognized: {}".format( - pagenum, state)) + raise RuntimeError(f"page {pagenum} state is not recognized: {state}") def initialize(self): - """ this method should be called in FTL """ + """this method should be called in FTL""" # set the state of all pages to ERASED self.bitmap.setall(0) - - diff --git a/wiscsee/wiscsim/blkpool.py b/wiscsee/wiscsim/blkpool.py index e25d457..270b219 100644 --- a/wiscsee/wiscsim/blkpool.py +++ b/wiscsee/wiscsim/blkpool.py @@ -1,28 +1,30 @@ from collections import deque import sys from wiscsim.devblockpool import * -from ftlsim_commons import random_channel_id +from .ftlsim_commons import random_channel_id + +TDATA = "TDATA" +TTRANS = "TTRANS" -TDATA = 'TDATA' -TTRANS = 'TTRANS' class OutOfSpaceError(RuntimeError): pass -class BlockPool(object): + +class BlockPool: def __init__(self, confobj): self.conf = confobj - self.n_channels = self.conf['flash_config']['n_channels_per_dev'] - self.stripe_size = self.conf['stripe_size'] + self.n_channels = self.conf["flash_config"]["n_channels_per_dev"] + self.stripe_size = self.conf["stripe_size"] self.pool = MultiChannelBlockPool( - n_channels=self.n_channels, - n_blocks_per_channel=self.conf.n_blocks_per_channel, - n_pages_per_block=self.conf.n_pages_per_block, - tags=[TDATA, TTRANS], - leveling_factor = self.conf['wear_leveling_factor'], - leveling_diff = self.conf['wear_leveling_diff'] - ) + n_channels=self.n_channels, + n_blocks_per_channel=self.conf.n_blocks_per_channel, + n_pages_per_block=self.conf.n_pages_per_block, + tags=[TDATA, TTRANS], + leveling_factor=self.conf["wear_leveling_factor"], + leveling_diff=self.conf["wear_leveling_diff"], + ) @property def freeblocks(self): @@ -63,16 +65,18 @@ def count_blocks(self, tag, channels=None): def pop_a_free_block_to_trans(self, choice=LEAST_ERASED): try: - blocknum = self.pool.pick_and_move(src=TFREE, dst=TTRANS, - choice=LEAST_ERASED) + blocknum = self.pool.pick_and_move( + src=TFREE, dst=TTRANS, choice=LEAST_ERASED + ) except TagOutOfSpaceError: raise OutOfSpaceError return blocknum def pop_a_free_block_to_data(self, choice=LEAST_ERASED): try: - blocknum = self.pool.pick_and_move(src=TFREE, dst=TDATA, - choice=LEAST_ERASED) + blocknum = self.pool.pick_and_move( + src=TFREE, dst=TDATA, choice=LEAST_ERASED + ) except TagOutOfSpaceError: raise OutOfSpaceError return blocknum @@ -88,38 +92,40 @@ def move_used_trans_block_to_data(self, blocknum): def next_n_data_pages_to_program_striped(self, n, seg_id=0, choice=LEAST_ERASED): try: - ppns = self.pool.next_ppns(n=n, tag=TDATA, block_index=seg_id, - stripe_size=self.conf['stripe_size']) + ppns = self.pool.next_ppns( + n=n, tag=TDATA, block_index=seg_id, stripe_size=self.conf["stripe_size"] + ) except TagOutOfSpaceError: raise OutOfSpaceError return ppns def next_n_data_pages_to_program_unstriped(self, n, seg_id=0, choice=LEAST_ERASED): try: - ppns = self.pool.next_ppns(n=n, tag=TDATA, block_index=seg_id, - stripe_size=1) + ppns = self.pool.next_ppns( + n=n, tag=TDATA, block_index=seg_id, stripe_size=1 + ) except TagOutOfSpaceError: raise OutOfSpaceError return ppns def next_data_page_to_program(self, seg_id=0): - ppns = self.pool.next_ppns(n=1, tag=TDATA, block_index=seg_id, - stripe_size=1) + ppns = self.pool.next_ppns(n=1, tag=TDATA, block_index=seg_id, stripe_size=1) return ppns[0] def next_translation_page_to_program(self): - ppns = self.pool.next_ppns(n=1, tag=TTRANS, block_index=0, - stripe_size=1) + ppns = self.pool.next_ppns(n=1, tag=TTRANS, block_index=0, stripe_size=1) return ppns[0] def next_gc_data_page_to_program(self, choice=LEAST_ERASED): - ppns = self.pool.next_ppns(n=1, tag=TDATA, block_index=0, - stripe_size=1, choice=choice) + ppns = self.pool.next_ppns( + n=1, tag=TDATA, block_index=0, stripe_size=1, choice=choice + ) return ppns[0] def next_gc_translation_page_to_program(self, choice=LEAST_ERASED): - ppns = self.pool.next_ppns(n=1, tag=TTRANS, block_index=0, - stripe_size=1, choice=choice) + ppns = self.pool.next_ppns( + n=1, tag=TTRANS, block_index=0, stripe_size=1, choice=choice + ) return ppns[0] def current_blocks(self): @@ -139,5 +145,3 @@ def num_freeblocks(self): def remove_full_cur_blocks(self): self.pool.remove_full_cur_blocks() - - diff --git a/wiscsee/wiscsim/controller.py b/wiscsee/wiscsim/controller.py index 4bdf7ad..7d89bae 100644 --- a/wiscsee/wiscsim/controller.py +++ b/wiscsee/wiscsim/controller.py @@ -1,12 +1,15 @@ +from past.utils import old_div import simpy import wiscsim from collections import Counter from commons import * from wiscsim.utils import * import random + random.seed(1000) -class FlashAddress(object): + +class FlashAddress: def __init__(self): self.page_index = 5 self.block_index = 4 @@ -15,14 +18,14 @@ def __init__(self): self.package_index = 1 self.channel_index = 0 - self.names = ['channel', 'package', 'chip', 'plane', 'block', 'page'] + self.names = ["channel", "package", "chip", "plane", "block", "page"] self.location = [0 for _ in self.names] def __str__(self): lines = [] for name, no in zip(self.names, self.location): lines.append(name.ljust(8) + str(no)) - return '\n'.join(lines) + return "\n".join(lines) def __eq__(self, other): for i in range(len(self.names)): @@ -30,10 +33,10 @@ def __eq__(self, other): return False return True - @property def page(self): return self.location[self.page_index] + @page.setter def page(self, value): self.location[self.page_index] = value @@ -41,6 +44,7 @@ def page(self, value): @property def block(self): return self.location[self.block_index] + @block.setter def block(self, value): self.location[self.block_index] = value @@ -48,6 +52,7 @@ def block(self, value): @property def plane(self): return self.location[self.plane_index] + @plane.setter def plane(self, value): self.location[self.plane_index] = value @@ -55,6 +60,7 @@ def plane(self, value): @property def chip(self): return self.location[self.chip_index] + @chip.setter def chip(self, value): self.location[self.chip_index] = value @@ -62,6 +68,7 @@ def chip(self, value): @property def package(self): return self.location[self.package_index] + @package.setter def package(self, value): self.location[self.package_index] = value @@ -69,12 +76,13 @@ def package(self, value): @property def channel(self): return self.location[self.channel_index] + @channel.setter def channel(self, value): self.location[self.channel_index] = value -class FlashRequest(object): +class FlashRequest: # OP_READ, OP_WRITE, OP_ERASE = 'OP_READ', 'OP_WRITE', 'OP_ERASE' def __init__(self): self.addr = None @@ -82,20 +90,20 @@ def __init__(self): def __str__(self): lines = [] - lines.append( "OPERATION " + str(self.operation) ) - lines.append( str(self.addr) ) - return '\n'.join(lines) + lines.append("OPERATION " + str(self.operation)) + lines.append(str(self.addr)) + return "\n".join(lines) def create_flashrequest(addr, op): req = FlashRequest() req.addr = addr - if op == 'read': + if op == "read": req.operation = OP_READ - elif op == 'write': + elif op == "write": req.operation = OP_WRITE - elif op == 'erase': + elif op == "erase": req.operation = OP_ERASE else: raise RuntimeError() @@ -103,38 +111,46 @@ def create_flashrequest(addr, op): return req -class Controller(object): +class Controller: """ This base class implements the core functions of a flash controller. It should not have those side functions like recording """ + def __init__(self, simpy_env, conf): self.env = simpy_env self.conf = conf # TODO: should these be in config? - self.page_size = self.conf['flash_config']['page_size'] - self.n_pages_per_block = self.conf['flash_config']['n_pages_per_block'] - self.n_blocks_per_plane = self.conf['flash_config']['n_blocks_per_plane'] - self.n_planes_per_chip = self.conf['flash_config']['n_planes_per_chip'] - self.n_chips_per_package = self.conf['flash_config']['n_chips_per_package'] - self.n_packages_per_channel = self.conf['flash_config']['n_packages_per_channel'] - self.n_channels_per_dev = self.conf['flash_config']['n_channels_per_dev'] + self.page_size = self.conf["flash_config"]["page_size"] + self.n_pages_per_block = self.conf["flash_config"]["n_pages_per_block"] + self.n_blocks_per_plane = self.conf["flash_config"]["n_blocks_per_plane"] + self.n_planes_per_chip = self.conf["flash_config"]["n_planes_per_chip"] + self.n_chips_per_package = self.conf["flash_config"]["n_chips_per_package"] + self.n_packages_per_channel = self.conf["flash_config"][ + "n_packages_per_channel" + ] + self.n_channels_per_dev = self.conf["flash_config"]["n_channels_per_dev"] self.n_pages_per_plane = self.n_pages_per_block * self.n_blocks_per_plane self.n_pages_per_chip = self.n_pages_per_plane * self.n_planes_per_chip self.n_pages_per_package = self.n_pages_per_chip * self.n_chips_per_package - self.n_pages_per_channel = self.n_pages_per_package * self.n_packages_per_channel + self.n_pages_per_channel = ( + self.n_pages_per_package * self.n_packages_per_channel + ) self.n_pages_per_dev = self.n_pages_per_channel * self.n_channels_per_dev - self.page_hierarchy = [self.n_pages_per_channel, - self.n_pages_per_package, - self.n_pages_per_chip, - self.n_pages_per_plane, - self.n_pages_per_block] + self.page_hierarchy = [ + self.n_pages_per_channel, + self.n_pages_per_package, + self.n_pages_per_chip, + self.n_pages_per_plane, + self.n_pages_per_block, + ] - self.channels = [Channel(self, self.env, conf, i) - for i in range( self.n_channels_per_dev)] + self.channels = [ + Channel(self, self.env, conf, i) for i in range(self.n_channels_per_dev) + ] def get_flash_requests_for_pbns(self, block_start, block_count, op): """ @@ -144,9 +160,9 @@ def get_flash_requests_for_pbns(self, block_start, block_count, op): ret_requests = [] for block in range(block_start, block_start + block_count): machine_block_addr = self.physical_to_machine_block(block) - flash_req = create_flashrequest( machine_block_addr, op = op) + flash_req = create_flashrequest(machine_block_addr, op=op) ret_requests.append(flash_req) - + return ret_requests def physical_to_machine_block(self, block): @@ -157,7 +173,7 @@ def physical_to_machine_block(self, block): page = block * self.n_pages_per_block addr = self.physical_to_machine_page(page) - addr.page = None # so we dont' mistakely use it for other purposes + addr.page = None # so we dont' mistakely use it for other purposes return addr @@ -168,7 +184,7 @@ def get_flash_requests_for_ppns(self, page_start, page_count, op): ret_requests = [] for page in range(page_start, page_start + page_count): machine_page_addr = self.physical_to_machine_page(page) - flash_req = create_flashrequest(machine_page_addr, op = op) + flash_req = create_flashrequest(machine_page_addr, op=op) ret_requests.append(flash_req) return ret_requests @@ -179,7 +195,7 @@ def physical_to_machine_page(self, page_no): # page_hierarchy has [channel, package, ..., block] # location has [channel, package, ..., block, page] for i, count in enumerate(self.page_hierarchy): - addr.location[i] = page_no / count + addr.location[i] = old_div(page_no, count) page_no = page_no % count addr.location[-1] = page_no @@ -189,14 +205,12 @@ def rw_ppn_extent(self, ppn_start, ppn_count, op): """ op is 'read' or 'write' """ - flash_reqs = self.get_flash_requests_for_ppns(ppn_start, ppn_count, - op = op) - yield self.env.process( self.execute_request_list(flash_reqs) ) + flash_reqs = self.get_flash_requests_for_ppns(ppn_start, ppn_count, op=op) + yield self.env.process(self.execute_request_list(flash_reqs)) def erase_pbn_extent(self, pbn_start, pbn_count): - flash_reqs = self.get_flash_requests_for_pbns(pbn_start, pbn_count, - op = 'erase') - yield self.env.process( self.execute_request_list(flash_reqs) ) + flash_reqs = self.get_flash_requests_for_pbns(pbn_start, pbn_count, op="erase") + yield self.env.process(self.execute_request_list(flash_reqs)) def execute_request_list(self, flash_request_list): procs = [] @@ -208,19 +222,15 @@ def execute_request_list(self, flash_request_list): def execute_request(self, flash_request): if flash_request.operation == OP_READ: - yield self.env.process( - self.read_page(flash_request.addr)) + yield self.env.process(self.read_page(flash_request.addr)) elif flash_request.operation == OP_WRITE: - yield self.env.process( - self.write_page(flash_request.addr)) + yield self.env.process(self.write_page(flash_request.addr)) elif flash_request.operation == OP_ERASE: - yield self.env.process( - self.erase_block(flash_request.addr)) + yield self.env.process(self.erase_block(flash_request.addr)) else: - raise RuntimeError("operation {} is not supported".format( - flash_request.operation)) + raise RuntimeError(f"operation {flash_request.operation} is not supported") - def write_page(self, addr, data = None): + def write_page(self, addr, data=None): """ Usage: if you do: @@ -237,28 +247,28 @@ def write_page(self, addr, data = None): the calling process will not wait the second write has to wait for the first """ - yield self.env.process( - self.channels[addr.channel].write_page(None)) + yield self.env.process(self.channels[addr.channel].write_page(None)) def read_page(self, addr): - yield self.env.process( - self.channels[addr.channel].read_page(None)) + yield self.env.process(self.channels[addr.channel].read_page(None)) def erase_block(self, addr): - yield self.env.process( - self.channels[addr.channel].erase_block(None)) + yield self.env.process(self.channels[addr.channel].erase_block(None)) class Controller3(Controller): """ With tag, and recorder """ + def __init__(self, simpy_env, conf, recorderobj): - super(Controller3, self).__init__(simpy_env, conf) + super().__init__(simpy_env, conf) self.recorder = recorderobj - self.channels = [Channel3(self, self.env, conf, self.recorder, i) - for i in range( self.n_channels_per_dev)] + self.channels = [ + Channel3(self, self.env, conf, self.recorder, i) + for i in range(self.n_channels_per_dev) + ] def execute_request_list(self, flash_request_list, tag): procs = [] @@ -268,25 +278,25 @@ def execute_request_list(self, flash_request_list, tag): event = simpy.events.AllOf(self.env, procs) yield event - def write_page(self, addr, tag, data = None): + def write_page(self, addr, tag, data=None): yield self.env.process( - self.channels[addr.channel].write_page(tag = tag, - addr = addr, data = None)) + self.channels[addr.channel].write_page(tag=tag, addr=addr, data=None) + ) def read_page(self, addr, tag): yield self.env.process( - self.channels[addr.channel].read_page(tag = tag, - addr = addr)) + self.channels[addr.channel].read_page(tag=tag, addr=addr) + ) def erase_block(self, addr, tag): yield self.env.process( - self.channels[addr.channel].erase_block(tag = tag, addr = None)) + self.channels[addr.channel].erase_block(tag=tag, addr=None) + ) def rw_ppns(self, ppns, op, tag): procs = [] for ppn in ppns: - p = self.env.process( - self.rw_ppn_extent(ppn, 1, op, tag)) + p = self.env.process(self.rw_ppn_extent(ppn, 1, op, tag)) procs.append(p) yield simpy.events.AllOf(self.env, procs) @@ -295,32 +305,26 @@ def rw_ppn_extent(self, ppn_start, ppn_count, op, tag): """ op is 'read' or 'write' """ - flash_reqs = self.get_flash_requests_for_ppns(ppn_start, ppn_count, - op = op) - yield self.env.process( self.execute_request_list(flash_reqs, tag) ) + flash_reqs = self.get_flash_requests_for_ppns(ppn_start, ppn_count, op=op) + yield self.env.process(self.execute_request_list(flash_reqs, tag)) def erase_pbn_extent(self, pbn_start, pbn_count, tag): - flash_reqs = self.get_flash_requests_for_pbns(pbn_start, pbn_count, - op = 'erase') - yield self.env.process( self.execute_request_list(flash_reqs, tag) ) + flash_reqs = self.get_flash_requests_for_pbns(pbn_start, pbn_count, op="erase") + yield self.env.process(self.execute_request_list(flash_reqs, tag)) def execute_request(self, flash_request, tag): - self.recorder.count_me('flash_ops', flash_request.operation) + self.recorder.count_me("flash_ops", flash_request.operation) if flash_request.operation == OP_READ: - yield self.env.process( - self.read_page(addr = flash_request.addr, tag = tag)) + yield self.env.process(self.read_page(addr=flash_request.addr, tag=tag)) elif flash_request.operation == OP_WRITE: - yield self.env.process( - self.write_page(flash_request.addr, tag = tag)) + yield self.env.process(self.write_page(flash_request.addr, tag=tag)) elif flash_request.operation == OP_ERASE: - yield self.env.process( - self.erase_block(flash_request.addr, tag = tag)) + yield self.env.process(self.erase_block(flash_request.addr, tag=tag)) else: - raise RuntimeError("operation {} is not supported".format( - flash_request.operation)) + raise RuntimeError(f"operation {flash_request.operation} is not supported") -class Channel(object): +class Channel: """ This is a channel with only single package, chip, and plane. This is how a request is processed in it: @@ -333,11 +337,12 @@ class Channel(object): write: 7*t_wc + nbytes*t_wc + t_prog """ - def __init__(self, controller, simpy_env, conf, channel_id = None): + + def __init__(self, controller, simpy_env, conf, channel_id=None): self.controller = controller self.env = simpy_env self.conf = conf - self.resource = simpy.Resource(self.env, capacity = 1) + self.resource = simpy.Resource(self.env, capacity=1) self.channel_id = channel_id t_wc = 1 @@ -345,24 +350,27 @@ def __init__(self, controller, simpy_env, conf, channel_id = None): t_rc = 1 t_prog = 1 t_erase = 1 - page_size = self.conf['flash_config']['page_size'] + page_size = self.conf["flash_config"]["page_size"] # self.read_time = 7 * t_wc + t_r + page_size * t_rc # self.program_time = 7 * t_wc + page_size * t_wc + t_prog # self.erase_time = 5 * t_wc + t_erase - self.read_time = 7 * self.conf['flash_config']['t_WC'] + \ - self.conf['flash_config']['t_R'] + \ - self.conf['flash_config']['page_size'] * \ - self.conf['flash_config']['t_RC'] - self.program_time = 7 * self.conf['flash_config']['t_WC'] + \ - self.conf['flash_config']['page_size'] * \ - self.conf['flash_config']['t_WC'] +\ - self.conf['flash_config']['t_PROG'] - self.erase_time = 5 * self.conf['flash_config']['t_WC'] + \ - self.conf['flash_config']['t_BERS'] - - def write_page(self, addr = None , data = None): + self.read_time = ( + 7 * self.conf["flash_config"]["t_WC"] + + self.conf["flash_config"]["t_R"] + + self.conf["flash_config"]["page_size"] * self.conf["flash_config"]["t_RC"] + ) + self.program_time = ( + 7 * self.conf["flash_config"]["t_WC"] + + self.conf["flash_config"]["page_size"] * self.conf["flash_config"]["t_WC"] + + self.conf["flash_config"]["t_PROG"] + ) + self.erase_time = ( + 5 * self.conf["flash_config"]["t_WC"] + self.conf["flash_config"]["t_BERS"] + ) + + def write_page(self, addr=None, data=None): """ If you want to when this operation is finished, just print env.now. If you want to know how long it takes, use env.now before and after @@ -370,27 +378,31 @@ def write_page(self, addr = None , data = None): """ with self.resource.request() as request: yield request - yield self.env.timeout( int(self.program_time * (random.random()*0.2+0.9)) ) + yield self.env.timeout( + int(self.program_time * (random.random() * 0.2 + 0.9)) + ) - def read_page(self, addr = None): + def read_page(self, addr=None): with self.resource.request() as request: yield request - yield self.env.timeout( int(self.read_time * (random.random()*0.2+0.9))) + yield self.env.timeout(int(self.read_time * (random.random() * 0.2 + 0.9))) - def erase_block(self, addr = None): + def erase_block(self, addr=None): with self.resource.request() as request: yield request - yield self.env.timeout( self.erase_time ) + yield self.env.timeout(self.erase_time) + class Channel2(Channel): """ It has recorder """ - def __init__(self, controller, simpy_env, conf, recorderobj, channel_id = None): - super(Channel2, self).__init__(controller, simpy_env, conf, channel_id) + + def __init__(self, controller, simpy_env, conf, recorderobj, channel_id=None): + super().__init__(controller, simpy_env, conf, channel_id) self.recorder = recorderobj - def write_page(self, addr = None , data = None): + def write_page(self, addr=None, data=None): """ If you want to when this operation is finished, just print env.now. If you want to know how long it takes, use env.now before and after @@ -399,34 +411,32 @@ def write_page(self, addr = None , data = None): with self.resource.request() as request: yield request s = self.env.now - yield self.env.timeout( self.program_time ) + yield self.env.timeout(self.program_time) e = self.env.now - self.recorder.add_to_timer("channel_busy_time", self.channel_id, - e - s) + self.recorder.add_to_timer("channel_busy_time", self.channel_id, e - s) - def read_page(self, addr = None): + def read_page(self, addr=None): with self.resource.request() as request: yield request s = self.env.now - yield self.env.timeout( self.read_time ) + yield self.env.timeout(self.read_time) e = self.env.now - self.recorder.add_to_timer("channel_busy_time", self.channel_id, - e - s) + self.recorder.add_to_timer("channel_busy_time", self.channel_id, e - s) - def erase_block(self, addr = None): + def erase_block(self, addr=None): with self.resource.request() as request: yield request s = self.env.now - yield self.env.timeout( self.erase_time ) + yield self.env.timeout(self.erase_time) e = self.env.now - self.recorder.add_to_timer("channel_busy_time", self.channel_id, - e - s) + self.recorder.add_to_timer("channel_busy_time", self.channel_id, e - s) class Channel3(Channel2): """ Operations can be tagged """ + def counter_set_name(self): return "channel_busy_time" @@ -434,17 +444,21 @@ def _convert_tag(self, tag): if isinstance(tag, dict): return tag elif isinstance(tag, str): - return {'tag':tag} + return {"tag": tag} def _write_channel_timeline(self, channel_id, start_time, end_time, tag): write = self.conf.get("write_channel_timeline", False) if write is True: tag = self._convert_tag(tag) - self.recorder.write_file('channel_timeline.txt', - channel=channel_id, start_time=start_time, end_time=end_time, - **tag) - - def write_page(self, tag, addr = None , data = None): + self.recorder.write_file( + "channel_timeline.txt", + channel=channel_id, + start_time=start_time, + end_time=end_time, + **tag, + ) + + def write_page(self, tag, addr=None, data=None): """ If you want to when this operation is finished, just print env.now. If you want to know how long it takes, use env.now before and after @@ -453,7 +467,7 @@ def write_page(self, tag, addr = None , data = None): with self.resource.request() as request: yield request s = self.env.now - yield self.env.timeout( self.program_time ) + yield self.env.timeout(self.program_time) e = self.env.now # if addr == self.controller.physical_to_machine_page(33): # log_msg("controller", 33, s, e) @@ -463,17 +477,18 @@ def write_page(self, tag, addr = None , data = None): self.recorder.count_me("channel", "write") self.recorder.add_to_timer( self.counter_set_name(), - "channel_{id}-write-{tag}".format(id = self.channel_id, - tag = self.recorder.tag_group(tag)), - e - s) - self._write_channel_timeline(channel_id=self.channel_id, - start_time=s, end_time=e, tag=tag) - - def read_page(self, tag, addr = None): + f"channel_{self.channel_id}-write-{self.recorder.tag_group(tag)}", + e - s, + ) + self._write_channel_timeline( + channel_id=self.channel_id, start_time=s, end_time=e, tag=tag + ) + + def read_page(self, tag, addr=None): with self.resource.request() as request: yield request s = self.env.now - yield self.env.timeout( self.read_time ) + yield self.env.timeout(self.read_time) e = self.env.now # if addr == self.controller.physical_to_machine_page(33): # log_msg("controller", 33, s, e) @@ -483,24 +498,25 @@ def read_page(self, tag, addr = None): # log_msg("controller", 184568676, s, e) self.recorder.add_to_timer( self.counter_set_name(), - "channel_{id}-read-{tag}".format(id = self.channel_id, - tag = self.recorder.tag_group(tag)), - e - s) - - self._write_channel_timeline(channel_id=self.channel_id, - start_time=s, end_time=e, tag=tag) - - def erase_block(self, tag, addr = None): + f"channel_{self.channel_id}-read-{self.recorder.tag_group(tag)}", + e - s, + ) + + self._write_channel_timeline( + channel_id=self.channel_id, start_time=s, end_time=e, tag=tag + ) + + def erase_block(self, tag, addr=None): with self.resource.request() as request: yield request s = self.env.now - yield self.env.timeout( self.erase_time ) + yield self.env.timeout(self.erase_time) e = self.env.now self.recorder.add_to_timer( self.counter_set_name(), - "channel_{id}-erase-{tag}".format(id = self.channel_id, - tag = self.recorder.tag_group(tag)), - e - s) - self._write_channel_timeline(channel_id=self.channel_id, - start_time=s, end_time=e, tag=tag) - + f"channel_{self.channel_id}-erase-{self.recorder.tag_group(tag)}", + e - s, + ) + self._write_channel_timeline( + channel_id=self.channel_id, start_time=s, end_time=e, tag=tag + ) diff --git a/wiscsee/wiscsim/crb_standalone.py b/wiscsee/wiscsim/crb_standalone.py index 41468bb..8e46b30 100644 --- a/wiscsee/wiscsim/crb_standalone.py +++ b/wiscsee/wiscsim/crb_standalone.py @@ -1,28 +1,29 @@ class CRB: HEAD = None + def __init__(self, list): self.items = list def search(self, item): - index = len(self.items)//2 + index = len(self.items) // 2 - tail = float('inf') - while(tail > item): + tail = float("inf") + while tail > item: index -= 1 - while(index >= 0 and self.items[index-1] != None): + while index >= 0 and self.items[index - 1] != None: index -= 1 - tail = self.items[index-2] + tail = self.items[index - 2] - index = len(self.items)//2 + index = len(self.items) // 2 head = -1 - while(head < item): + while head < item: index += 1 - while(index < len(self.items) and self.items[index+1] != None): + while index < len(self.items) and self.items[index + 1] != None: index += 1 - head = self.items[index+2] + head = self.items[index + 2] - print(tail, head) + print((tail, head)) -crb = CRB([0,1,2,10,None,4,5,8,13,None,6,7,17,18,None]) -crb.search(7) \ No newline at end of file +crb = CRB([0, 1, 2, 10, None, 4, 5, 8, 13, None, 6, 7, 17, 18, None]) +crb.search(7) diff --git a/wiscsee/wiscsim/datacache.py b/wiscsee/wiscsim/datacache.py index a73444a..a02645e 100755 --- a/wiscsee/wiscsim/datacache.py +++ b/wiscsee/wiscsim/datacache.py @@ -1,25 +1,27 @@ #!/usr/bin/env python2 +from past.utils import old_div from collections import OrderedDict, defaultdict import numpy as np from wiscsim.utils import * + class LFUCache(OrderedDict): def __init__(self): self.freq = defaultdict(list) - super(LFUCache, self).__init__() + super().__init__() def append(self, key): self[key] = "" - + def evict(self, n): for _ in range(n): - for l in self.freq.values(): + for l in list(self.freq.values()): if len(l) == 0: continue del self[l[0]] del l[0] break - #self.popitem(last=True) + # self.popitem(last=True) def _write(self, lpn): self[lpn] = 0 @@ -39,7 +41,7 @@ def _read(self, lpn): class LIFOCache(OrderedDict): def __init__(self): - super(LIFOCache, self).__init__() + super().__init__() def append(self, key): self[key] = "" @@ -50,6 +52,7 @@ def evict(self, n): def _write(self, lpn): self[lpn] = False + def _read(self, lpn): pass # self.move_to_head(lpn, self[lpn]) @@ -61,18 +64,18 @@ def move_to_head(self, key, value, dict_setitem=dict.__setitem__): class FIFOCache(OrderedDict): def __init__(self): - super(FIFOCache, self).__init__() + super().__init__() def append(self, key): self[key] = "" - def evict(self, n): for _ in range(n): self.popitem(last=False) def _write(self, lpn): self[lpn] = False + def _read(self, lpn): pass # self.move_to_head(lpn, self[lpn]) @@ -81,19 +84,21 @@ def move_to_head(self, key, value, dict_setitem=dict.__setitem__): del self[key] self[key] = value + class MRUCache(OrderedDict): def __init__(self): - super(MRUCache, self).__init__() + super().__init__() def append(self, key): self[key] = "" - + def evict(self, n): for _ in range(n): self.popitem(last=True) def _write(self, lpn): self[lpn] = False + def _read(self, lpn): self.move_to_head(lpn, self[lpn]) @@ -101,9 +106,10 @@ def move_to_head(self, key, value, dict_setitem=dict.__setitem__): del self[key] self[key] = value + class LRUCache(OrderedDict): def __init__(self): - super(LRUCache, self).__init__() + super().__init__() def append(self, key): self[key] = "" @@ -129,16 +135,16 @@ def evict_lpn(self, lpn): def _write(self, lpn, state=""): self[lpn] = state - + def _read(self, lpn): self.move_to_head(lpn, self[lpn]) return self[lpn] def peak(self): - return next(self.iteritems()) + return next(iter(list(self.items()))) def peak_n(self, n): - return dict([next(self.iteritems()) for i in range(n)]) + return dict([next(iter(list(self.items()))) for i in range(n)]) def move_to_head(self, key, value, dict_setitem=dict.__setitem__): del self[key] @@ -165,26 +171,28 @@ def move_to_head(self, key, value, dict_setitem=dict.__setitem__): self.cache = {LPN : priority in {True or False}} self.high = # of entries with priority """ -class DataCache(object): + + +class DataCache: def __init__(self, size, slot_size, method="LRU", priority=False, threshold=0.2): - - assert(method in ["LRU","FIFO","LFU","LIFO","MRU"]) - method2cache = {"LRU": LRUCache(), + assert method in ["LRU", "FIFO", "LFU", "LIFO", "MRU"] + method2cache = { + "LRU": LRUCache(), "FIFO": FIFOCache(), "LFU": LFUCache(), "LIFO": LIFOCache(), - "MRU": MRUCache() + "MRU": MRUCache(), } self.cache = method2cache[method] self.slot_size = slot_size self.capacity = max(1, self.bytes2slots(size)) self.priority = priority - self.high = 0. + self.high = 0.0 self.threshold = threshold def bytes2slots(self, size): - return max(int(size / self.slot_size), 0) - + return max(int(old_div(size, self.slot_size)), 0) + def read(self, lpn): if lpn not in self.cache: self._write(lpn) @@ -210,7 +218,6 @@ def set_priority(self, lpn, priority): if old_priority and not priority: self.high -= 1 - def _write(self, lpn): # self.cache[lpn] = False # self.cache.move_to_head(lpn, self.cache[lpn]) @@ -219,7 +226,7 @@ def _write(self, lpn): self.cache._write(lpn) def evict(self, n): - assert(n <= self.capacity) + assert n <= self.capacity self.cache.evict(n) # for _ in range(n): # # guarantee to evict low priority @@ -230,21 +237,22 @@ def evict(self, n): # # break # # else: # lpn, priority = self.cache.popitem(last=False) - # if priority: - # self.high -= 1 + # if priority: + # self.high -= 1 def resize(self, size): - assert(size >= 0) + assert size >= 0 old_capacity = self.capacity self.capacity = self.bytes2slots(size) if self.capacity < old_capacity: self.evict(old_capacity - self.capacity) -class WriteBuffer(): +class WriteBuffer: UNASSIGNED, ASSIGNED = "UNASSIGNED", "ASSIGNED" - def __init__(self, flush_size, size, filtering = 1.0, assign_ppn=True): - assert(flush_size <= size) + + def __init__(self, flush_size, size, filtering=1.0, assign_ppn=True): + assert flush_size <= size self.assign_ppn = assign_ppn self.flush_size = flush_size self.size = size @@ -262,20 +270,19 @@ def peek(self, lpn, move=False): elif lpn in self.unassigned: if move: self.unassigned.move_to_head(lpn, self.unassigned[lpn]) - return WriteBuffer.UNASSIGNED + return WriteBuffer.UNASSIGNED else: - return None + return None def read(self, lpn): return self.peek(lpn, move=True) - def write(self, lpn): if not self.peek(lpn): self.unassigned[lpn] = "" else: self.counter += 1 - #print("hit", self.counter) + # print("hit", self.counter) if self.length > self.size: writeback, _ = self.assigned.popitem(last=False) @@ -288,7 +295,6 @@ def write(self, lpn): else: return (None, None) - def should_assign_page(self): return len(self.unassigned) >= self.flush_size @@ -296,17 +302,29 @@ def should_assign_page(self): @staticmethod def split_into_frame(frame_length, entries): split_results = defaultdict(list) - for lpn, ppn in entries.items(): + for lpn, ppn in list(entries.items()): split_results[lpn // frame_length].append((lpn, ppn)) return split_results def flush_unassigned(self): - assert(len(self.unassigned) == self.flush_size) + assert len(self.unassigned) == self.flush_size split_results = self.split_into_frame(256, self.unassigned) - per_frame_count = dict(sorted([(k, len(v)) for k,v in split_results.items()], key=lambda x:x[1], reverse=True)) - cumsum = np.cumsum(per_frame_count.values()) - index = np.argwhere(cumsum<=self.flush_size*self.filtering)[-1][0] - lpa_to_flush = sorted(filter(lambda x: x // 256 in per_frame_count.keys()[:index+1], self.unassigned)) + per_frame_count = dict( + sorted( + [(k, len(v)) for k, v in list(split_results.items())], + key=lambda x: x[1], + reverse=True, + ) + ) + cumsum = np.cumsum(list(per_frame_count.values())) + index = np.argwhere(cumsum <= self.flush_size * self.filtering)[-1][0] + lpa_to_flush = sorted( + [ + x + for x in self.unassigned + if x // 256 in list(per_frame_count.keys())[: index + 1] + ] + ) for lpa in lpa_to_flush: self.assigned[lpa] = "" @@ -320,9 +338,9 @@ def flush_unassigned(self): return lpa_to_flush def update_assigned(self, mappings): - assert(self.assign_ppn) - for lpn, ppn in mappings.items(): - assert(lpn in self.assigned) + assert self.assign_ppn + for lpn, ppn in list(mappings.items()): + assert lpn in self.assigned self.mapping[lpn] = ppn @property @@ -330,8 +348,7 @@ def length(self): return len(self.assigned) + len(self.unassigned) - -class SimpleWriteBuffer(): +class SimpleWriteBuffer: def __init__(self, size): # assert(size <= max_size) # self.max_size = max(size, max_size) @@ -347,7 +364,7 @@ def write(self, lpn): self.buffer.append(lpn) else: self.counter += 1 - #print("hit", self.counter) + # print("hit", self.counter) def should_flush(self): return self.length >= self.size @@ -362,20 +379,25 @@ def length(self): return len(self.buffer) - - -class RWCache(object): - def __init__(self, size, slot_size, flush_size, filter_ratio, preconditioning=0.2, method="LRU"): - - assert(method in ["LRU","FIFO","LFU","LIFO","MRU"]) +class RWCache: + def __init__( + self, + size, + slot_size, + flush_size, + filter_ratio, + preconditioning=0.2, + method="LRU", + ): + assert method in ["LRU", "FIFO", "LFU", "LIFO", "MRU"] self.slot_size = slot_size if flush_size > size: flush_size = size filter_ratio = 1.0 log_msg("Adjusting flush size to become the data cache size") - self.capacity = max(int(size / self.slot_size), 1) - assert(flush_size <= size) - self.flush_size = max(int(flush_size / self.slot_size), 1) + self.capacity = max(int(old_div(size, self.slot_size)), 1) + assert flush_size <= size + self.flush_size = max(int(old_div(flush_size, self.slot_size)), 1) self.cache = self.method2cache(method) self.clean = self.method2cache(method) @@ -387,8 +409,8 @@ def __init__(self, size, slot_size, flush_size, filter_ratio, preconditioning=0. self.counter = defaultdict(float) for i in range(self.capacity): - dummy_lpn = "dummy"+str(i) - if i <= int(preconditioning*self.capacity): + dummy_lpn = "dummy" + str(i) + if i <= int(preconditioning * self.capacity): self.cache._write(dummy_lpn, state="A") self.assigned._write(dummy_lpn) else: @@ -408,7 +430,7 @@ def method2cache(self, method): return LIFOCache() elif method == "MRU": return MRUCache() - + def read(self, lpn): writeback, wb_ppn = False, None if lpn not in self.cache: @@ -430,7 +452,7 @@ def read(self, lpn): if writeback: self.counter["writebacks"] += 1 - + return hit, writeback, wb_ppn def write(self, lpn): @@ -455,9 +477,9 @@ def write(self, lpn): if writeback: self.counter["writebacks"] += 1 - + return writeback, wb_ppn - + def evict(self, state_preference): writeback, wb_ppn = False, None if self.should_evict(): @@ -481,31 +503,30 @@ def evict(self, state_preference): elif state == "C": self.cache.evict_one() lpn, state = self.clean.evict_lpn(lpn) - + ## we still can hit U here if the size of unassigned is smaller than flush_size elif state == "U": if state_preference == "A" and len(self.assigned) == 0: state_preference = "C" elif state_preference == "C" and len(self.clean) == 0: state_preference = "A" - + if state_preference == "A": lpn, _ = self.assigned.evict_one() _, state = self.cache.evict_lpn(lpn) - assert(state == state_preference) + assert state == state_preference writeback, wb_ppn = True, self.dirty_page_mapping[lpn] del self.dirty_page_mapping[lpn] elif state_preference == "C": lpn, _ = self.clean.evict_one() _, state = self.cache.evict_lpn(lpn) - assert(state == state_preference) - + assert state == state_preference + return writeback, wb_ppn - def update_assigned(self, mappings): - for lpn, ppn in mappings.items(): - assert(lpn in self.assigned) + for lpn, ppn in list(mappings.items()): + assert lpn in self.assigned self.dirty_page_mapping[lpn] = ppn def select_unassigned(self): @@ -517,17 +538,28 @@ def select_unassigned(self): candidates.append(lpn) if i >= self.flush_size: break - per_frame_count = dict(sorted([(k, len(v)) for k,v in split_results.items()], key=lambda x:x[1], reverse=True)) - cumsum = np.cumsum(per_frame_count.values()) + per_frame_count = dict( + sorted( + [(k, len(v)) for k, v in list(split_results.items())], + key=lambda x: x[1], + reverse=True, + ) + ) + cumsum = np.cumsum(list(per_frame_count.values())) ## select index as the last logical group to satisfy the flush size requirement - index = np.argwhere(cumsum<=self.flush_size*self.filter_ratio)[-1][0] - lpa_to_flush = sorted(filter(lambda x: x // frame_length in per_frame_count.keys()[:index+1], candidates)) + index = np.argwhere(cumsum <= self.flush_size * self.filter_ratio)[-1][0] + lpa_to_flush = sorted( + [ + x + for x in candidates + if x // frame_length in list(per_frame_count.keys())[: index + 1] + ] + ) # lpa_to_flush = list(filter(lambda x: x // frame_length in per_frame_count.keys()[:index+1], candidates)) return lpa_to_flush - def flush_unassigned(self): # evicted = self.unassigned.evict(self.flush_size) # lpn_to_flush = zip(*evicted)[0] @@ -537,33 +569,47 @@ def flush_unassigned(self): self.assigned[lpn] = "" del self.unassigned[lpn] self.cache[lpn] = "A" - + return lpn_to_flush def should_assign_page(self): - return self.cache.peak()[1] == "U" and len(self.cache) >= self.capacity and len(self.unassigned) >= self.flush_size + return ( + self.cache.peak()[1] == "U" + and len(self.cache) >= self.capacity + and len(self.unassigned) >= self.flush_size + ) def should_evict(self): return len(self.cache) > self.capacity def record_counter(self): - for k, v in self.counter.items(): + for k, v in list(self.counter.items()): log_msg(k, v) - + num_writes = self.counter["write hits"] + self.counter["write misses"] num_reads = self.counter["read hits"] + self.counter["read misses"] if num_writes > 0: - log_msg("write miss ratio", self.counter["writebacks"] / num_writes) + log_msg("write miss ratio", old_div(self.counter["writebacks"], num_writes)) if num_reads > 0: - log_msg("read miss ratio", self.counter["read misses"] / num_reads) - - log_msg("Clean slots", len(self.clean), "Dirty slots (assigned)", len(self.assigned), "Dirty slots (unassigned)", len(self.unassigned)) - - est_overall_latency = (self.counter["read misses"]*20 + self.counter["writebacks"]*200) / (num_writes + num_reads) + log_msg("read miss ratio", old_div(self.counter["read misses"], num_reads)) + + log_msg( + "Clean slots", + len(self.clean), + "Dirty slots (assigned)", + len(self.assigned), + "Dirty slots (unassigned)", + len(self.unassigned), + ) + + est_overall_latency = old_div( + (self.counter["read misses"] * 20 + self.counter["writebacks"] * 200), + (num_writes + num_reads), + ) if num_reads > 0: - est_read_latency = self.counter["read misses"]*20 / num_reads - log_msg("Estimated read latency", round(est_read_latency,3), "us") + est_read_latency = old_div(self.counter["read misses"] * 20, num_reads) + log_msg("Estimated read latency", round(est_read_latency, 3), "us") if num_writes > 0: - est_write_latency = self.counter["writebacks"]*200 / num_writes - log_msg("Estimated write latency", round(est_write_latency,3), "us") - log_msg("Estimated overall latency", round(est_overall_latency,3), "us") + est_write_latency = old_div(self.counter["writebacks"] * 200, num_writes) + log_msg("Estimated write latency", round(est_write_latency, 3), "us") + log_msg("Estimated overall latency", round(est_overall_latency, 3), "us") diff --git a/wiscsee/wiscsim/devblockpool.py b/wiscsee/wiscsim/devblockpool.py index 925d8dd..716f3e4 100644 --- a/wiscsee/wiscsim/devblockpool.py +++ b/wiscsee/wiscsim/devblockpool.py @@ -1,10 +1,19 @@ -from tagblockpool import * -from ftlsim_commons import random_channel_id +from past.utils import old_div +from .tagblockpool import * +from .ftlsim_commons import random_channel_id from utilities import utils -class MultiChannelBlockPoolBase(object): - def __init__(self, n_channels, n_blocks_per_channel, n_pages_per_block, tags, - leveling_factor=2, leveling_diff=10): + +class MultiChannelBlockPoolBase: + def __init__( + self, + n_channels, + n_blocks_per_channel, + n_pages_per_block, + tags, + leveling_factor=2, + leveling_diff=10, + ): self.n_channels = n_channels self.n_blocks_per_channel = n_blocks_per_channel self.n_blocks_per_dev = n_blocks_per_channel * n_channels @@ -14,8 +23,9 @@ def __init__(self, n_channels, n_blocks_per_channel, n_pages_per_block, tags, self.tags = tags self._channel_pool = [ - ChannelBlockPool(n_blocks_per_channel, tags, n_pages_per_block, i) \ - for i in range(n_channels)] + ChannelBlockPool(n_blocks_per_channel, tags, n_pages_per_block, i) + for i in range(n_channels) + ] # TODO: each tag has its own _next_channel self._next_channel = random_channel_id(self.n_channels) @@ -51,9 +61,9 @@ def get_blocks_of_tag(self, tag, channel_id=None): ret = [] if channel_id is None: - channels = self._channel_pool + channels = self._channel_pool else: - channels = [self._channel_pool[channel_id]] + channels = [self._channel_pool[channel_id]] for pool in channels: blocks = pool.get_blocks_of_tag(tag) @@ -81,16 +91,16 @@ def get_wear_status(self): """ nblocks = self.total_blocks * 0.1 - top_total, top_count = self.get_top_or_bottom_erasure_total( - 'top', nblocks) + top_total, top_count = self.get_top_or_bottom_erasure_total("top", nblocks) top_average = float(top_total) / top_count bottom_total, bottom_count = self.get_top_or_bottom_erasure_total( - 'bottom', nblocks) + "bottom", nblocks + ) bottom_average = float(bottom_total) / bottom_count diff = top_average - bottom_average if bottom_total == 0: - factor = float('inf') + factor = float("inf") else: factor = float(top_total) / bottom_total @@ -99,10 +109,10 @@ def get_wear_status(self): def need_wear_leveling(self): factor, diff = self.get_wear_status() - print factor, self.leveling_factor - print diff, self.leveling_diff + print((factor, self.leveling_factor)) + print((diff, self.leveling_diff)) - if factor > self.leveling_factor and diff > self.leveling_diff: + if factor > self.leveling_factor and diff > self.leveling_diff: return True else: return False @@ -132,9 +142,8 @@ def get_erasure_count(self): global_counter = Counter() for pool in self._channel_pool: erasure_cnt = pool.get_erasure_count() - for blocknum, erase_cnt in erasure_cnt.items(): - global_blocknum = self._channel_to_global( - pool.channel_id, blocknum) + for blocknum, erase_cnt in list(erasure_cnt.items()): + global_blocknum = self._channel_to_global(pool.channel_id, blocknum) global_counter[global_blocknum] = erasure_cnt[blocknum] return global_counter @@ -156,8 +165,7 @@ def pick(self, tag, channel_id=None, choice=LEAST_ERASED): else: cur_channel = channel_id - block_off = self._channel_pool[cur_channel].pick(tag=tag, - choice=choice) + block_off = self._channel_pool[cur_channel].pick(tag=tag, choice=choice) if block_off is None: return None @@ -176,12 +184,12 @@ def _channel_to_global(self, channel_id, blocknum): def _global_to_channel(self, blocknum): "return channel_id, block_offset" assert blocknum < self.total_blocks - return blocknum / self.n_blocks_per_channel, \ - blocknum % self.n_blocks_per_channel + return old_div( + blocknum, self.n_blocks_per_channel + ), blocknum % self.n_blocks_per_channel def _blocks_channel_to_global(self, channel_id, blocks): - return [self._channel_to_global(channel_id, block) - for block in blocks] + return [self._channel_to_global(channel_id, block) for block in blocks] def _ppn_channel_to_global(self, channel_id, ppn): return channel_id * self.n_pages_per_channel + ppn @@ -194,6 +202,7 @@ class MultiChannelBlockPool(MultiChannelBlockPoolBase): """ This is for DFTL """ + def current_blocks(self): "return all current block numbers" blocknums = [] @@ -201,8 +210,7 @@ def current_blocks(self): for tag in self.tags: cur_blk_objs = pool.get_cur_block_obj(tag) for obj in cur_blk_objs: - global_blk = self._channel_to_global(pool.channel_id, - obj.blocknum) + global_blk = self._channel_to_global(pool.channel_id, obj.blocknum) blocknums.append(global_blk) return blocknums @@ -217,8 +225,8 @@ def next_ppns(self, n, tag, block_index, stripe_size, choice=LEAST_ERASED): current block before going to the next. """ remaining = n - if stripe_size == 'infinity': - stripe_size = float('inf') + if stripe_size == "infinity": + stripe_size = float("inf") ret_ppns = [] empty_channels = set() @@ -226,9 +234,12 @@ def next_ppns(self, n, tag, block_index, stripe_size, choice=LEAST_ERASED): cur_channel_id = self._next_channel req = min(remaining, stripe_size) ppns = self._next_ppns_in_channel( - channel_id=cur_channel_id, - n=req, tag=tag, block_index=block_index, - choice=choice) + channel_id=cur_channel_id, + n=req, + tag=tag, + block_index=block_index, + choice=choice, + ) if len(ppns) == 0: # channel out of space empty_channels.add(cur_channel_id) @@ -245,8 +256,9 @@ def next_ppns(self, n, tag, block_index, stripe_size, choice=LEAST_ERASED): return ret_ppns - def _next_ppns_in_channel(self, channel_id, n, tag, block_index, - choice=LEAST_ERASED): + def _next_ppns_in_channel( + self, channel_id, n, tag, block_index, choice=LEAST_ERASED + ): """ Return ppns we can find. If returning [], it means this channel is out of space. @@ -256,12 +268,14 @@ def _next_ppns_in_channel(self, channel_id, n, tag, block_index, ret_ppns = [] while remaining > 0: - ppnlist = channel_pool.next_ppns_from_cur_block(n=remaining, - tag=tag, block_index=block_index) + ppnlist = channel_pool.next_ppns_from_cur_block( + n=remaining, tag=tag, block_index=block_index + ) if len(ppnlist) == 0: - new_block = channel_pool.pick_and_move(src=TFREE, dst=tag, - choice=choice) + new_block = channel_pool.pick_and_move( + src=TFREE, dst=tag, choice=choice + ) if new_block == None: # this channel is out of space of this tag break @@ -274,11 +288,9 @@ def _next_ppns_in_channel(self, channel_id, n, tag, block_index, class ChannelBlockPool(BlockPoolWithCurBlocks): def __init__(self, n, tags, n_pages_per_block, channel_id): - super(ChannelBlockPool, self).__init__(n, tags, n_pages_per_block) + super().__init__(n, tags, n_pages_per_block) self.channel_id = channel_id class TagOutOfSpaceError(RuntimeError): pass - - diff --git a/wiscsee/wiscsim/dftldes.py b/wiscsee/wiscsim/dftldes.py index a80c398..fafbc01 100644 --- a/wiscsee/wiscsim/dftldes.py +++ b/wiscsee/wiscsim/dftldes.py @@ -1,3 +1,4 @@ +from past.utils import old_div import bitarray from collections import deque, Counter, defaultdict import csv @@ -6,7 +7,7 @@ import itertools import random import os -import Queue +import queue import sys import simpy import numpy as np @@ -14,28 +15,30 @@ import bidict import config -import flash -import ftlbuilder -from lrulist import LruDict, SegmentedLruCache, LruCache -import recorder +from . import flash +from . import ftlbuilder +from .lrulist import LruDict, SegmentedLruCache, LruCache +from . import recorder from utilities import utils from commons import * -from ftlsim_commons import * +from .ftlsim_commons import * from .blkpool import BlockPool, MOST_ERASED, LEAST_ERASED from .bitmap import FlashBitmap2 -from datacache import * +from .datacache import * from wiscsim.utils import * - -UNINITIATED, MISS = ('UNINIT', 'MISS') -DATA_BLOCK, TRANS_BLOCK = ('data_block', 'trans_block') +UNINITIATED, MISS = ("UNINIT", "MISS") +DATA_BLOCK, TRANS_BLOCK = ("data_block", "trans_block") random.seed(0) -LOGICAL_READ, LOGICAL_WRITE, LOGICAL_DISCARD = ('LOGICAL_READ', \ - 'LOGICAL_WRITE', 'LOGICAL_DISCARD') +LOGICAL_READ, LOGICAL_WRITE, LOGICAL_DISCARD = ( + "LOGICAL_READ", + "LOGICAL_WRITE", + "LOGICAL_DISCARD", +) -PURPOSE_GC = 'PURPOSE_GC' -PURPOSE_WEAR_LEVEL = 'PURPOSE_WEAR_LEVEL' +PURPOSE_GC = "PURPOSE_GC" +PURPOSE_WEAR_LEVEL = "PURPOSE_WEAR_LEVEL" # # - translation pages @@ -61,7 +64,7 @@ # trans clean include: # erasing translation block # move translation page during gc (including read and write) -TRANS_CLEAN = "trans.clean" #read/write are for moving pages +TRANS_CLEAN = "trans.clean" # read/write are for moving pages # clean_data_block() # update_mapping_in_batch() @@ -75,12 +78,10 @@ DATA_CLEANING = "data.cleaning" - -class Ftl(object): +class Ftl: def __init__(self, confobj, recorderobj, flashcontrollerobj, env): if not isinstance(confobj, Config): - raise TypeError("confobj is not Config. it is {}". - format(type(confobj).__name__)) + raise TypeError(f"confobj is not Config. it is {type(confobj).__name__}") self.conf = confobj self.recorder = recorderobj @@ -90,38 +91,38 @@ def __init__(self, confobj, recorderobj, flashcontrollerobj, env): self.block_pool = BlockPool(confobj) self.oob = OutOfBandAreas(confobj) - self._directory = GlobalTranslationDirectory(self.conf, - self.oob, self.block_pool) + self._directory = GlobalTranslationDirectory( + self.conf, self.oob, self.block_pool + ) - self._trans_page_locks = LockPool(self.env) + self._trans_page_locks = LockPool(self.env) self._mappings = MappingCache( - ftl = self, - confobj = self.conf, - block_pool = self.block_pool, - flashobj = self.flash, - oobobj = self.oob, - recorderobj = self.recorder, - envobj = self.env, - directory = self._directory, - mapping_on_flash = MappingOnFlash(self.conf), - trans_page_locks = self._trans_page_locks - ) - - self.n_sec_per_page = self.conf.page_size \ - / self.conf['sector_size'] + ftl=self, + confobj=self.conf, + block_pool=self.block_pool, + flashobj=self.flash, + oobobj=self.oob, + recorderobj=self.recorder, + envobj=self.env, + directory=self._directory, + mapping_on_flash=MappingOnFlash(self.conf), + trans_page_locks=self._trans_page_locks, + ) + + self.n_sec_per_page = old_div(self.conf.page_size, self.conf["sector_size"]) self._cleaner = Cleaner( - conf = self.conf, - flash = self.flash, - oob = self.oob, - block_pool = self.block_pool, - mappings = self._mappings, - directory = self._directory, - rec = self.recorder, - env = self.env, - trans_page_locks = self._trans_page_locks - ) + conf=self.conf, + flash=self.flash, + oob=self.oob, + block_pool=self.block_pool, + mappings=self._mappings, + directory=self._directory, + rec=self.recorder, + env=self.env, + trans_page_locks=self._trans_page_locks, + ) self._check_segment_config() @@ -133,22 +134,26 @@ def __init__(self, confobj, recorderobj, flashcontrollerobj, env): self.pre_read_bytes = 0 self.display_interval = 1000 * MB self.rw_events = 0 - self.datacache = DataCache(self.conf['cache_size'], self.conf.page_size) - self.buffer = WriteBuffer(self.conf.n_pages_per_block, self.conf.n_pages_per_block, assign_ppn=False) + self.datacache = DataCache(self.conf["cache_size"], self.conf.page_size) + self.buffer = WriteBuffer( + self.conf.n_pages_per_block, self.conf.n_pages_per_block, assign_ppn=False + ) # self.buffer = SimpleWriteBuffer(self.conf.n_pages_per_block) self.read_latencies = [] self.write_latencies = [] - self.waf = {"request" : 0, "actual" : 0} - + self.waf = {"request": 0, "actual": 0} def recorder_enabled(self, enable=True): pass def _check_segment_config(self): - if self.conf['segment_bytes'] % (self.conf.n_pages_per_block \ - * self.conf.page_size) != 0: - print 'WARNING: segment should be multiple of block size' + if ( + self.conf["segment_bytes"] + % (self.conf.n_pages_per_block * self.conf.page_size) + != 0 + ): + print("WARNING: segment should be multiple of block size") def _ppns_to_write(self, ext, new_mappings): ppns = [] @@ -160,12 +165,12 @@ def _ppns_to_write(self, ext, new_mappings): def get_ppns_to_write(self, ext): exts_by_seg = split_ext_by_segment(self.conf.n_pages_per_segment, ext) mapping = {} - for seg_id, seg_ext in exts_by_seg.items(): + for seg_id, seg_ext in list(exts_by_seg.items()): # ppns = self.block_pool.next_data_page_to_program_striped( ppns = self.block_pool.next_n_data_pages_to_program_unstriped( - n=seg_ext.lpn_count, seg_id=seg_id, - choice=LEAST_ERASED) - mapping.update( dict(zip(seg_ext.lpn_iter(), ppns)) ) + n=seg_ext.lpn_count, seg_id=seg_id, choice=LEAST_ERASED + ) + mapping.update(dict(list(zip(seg_ext.lpn_iter(), ppns)))) return mapping def flush_trans_cache(self): @@ -180,34 +185,47 @@ def purge_trans_cache(self): def end_ssd(self): if len(self.read_latencies) > 0: - log_msg("End-to-end read response time per page: %.2fus" % (np.average(self.read_latencies) / 1000)) + log_msg( + f"End-to-end read response time per page: {old_div(np.average(self.read_latencies), 1000):.2f}us" + ) if len(self.write_latencies) > 0: - log_msg("End-to-end write response time per page: %.2fus" % (np.average(self.write_latencies) / 1000)) + log_msg( + f"End-to-end write response time per page: {old_div(np.average(self.write_latencies), 1000):.2f}us" + ) - log_msg("Write Amplification Factor: %.2f; Actual: %d; Request: %d" % (self.waf['actual'] / float(self.waf['request']), self.waf['actual'], self.waf['request'])) + log_msg( + f"Write Amplification Factor: {self.waf['actual'] / float(self.waf['request']):.2f}; Actual: {self.waf['actual']}; Request: {self.waf['request']}" + ) def write_ext(self, extent): req_size = extent.lpn_count * self.conf.page_size - self.recorder.add_to_general_accumulater('traffic', 'write', req_size) + self.recorder.add_to_general_accumulater("traffic", "write", req_size) self.written_bytes += req_size self.rw_events += 1 if self.written_bytes > self.pre_written_bytes + self.display_interval: - log_msg('Event', self.rw_events, 'Written (MB)', self.written_bytes / MB, 'writing', round(float(req_size) / MB, 2)) + log_msg( + "Event", + self.rw_events, + "Written (MB)", + old_div(self.written_bytes, MB), + "writing", + round(float(req_size) / MB, 2), + ) sys.stdout.flush() self.pre_written_bytes = self.written_bytes op_id = self.recorder.get_unique_num() procs = [] - start_time = self.env.now # <----- start + start_time = self.env.now # <----- start pages_to_write = [] for lpn in extent.lpn_iter(): writeback, _ = self.buffer.write(lpn) self.datacache.invalidate(lpn) - + # different from learnedftl, here we do not need to assign ppns as a group; but we still need to call flush_unassigned to enable eviction if self.buffer.should_assign_page(): exts = self.buffer.flush_unassigned() - + if writeback: wb_extent = Extent(writeback, 1) exts_in_mvpngroup = split_ext_to_mvpngroups(self.conf, wb_extent) @@ -215,8 +233,10 @@ def write_ext(self, extent): for ext_single_m_vpn in exts_in_mvpngroup: ppns_of_ext = self._ppns_to_write(ext_single_m_vpn, new_mappings) p = self.env.process( - self._write_single_mvpngroup(ext_single_m_vpn, ppns_of_ext, - tag = op_id)) + self._write_single_mvpngroup( + ext_single_m_vpn, ppns_of_ext, tag=op_id + ) + ) procs.append(p) # if len(procs) > 0: @@ -227,25 +247,31 @@ def write_ext(self, extent): # if op_id > 48: # exit() - end_time = self.env.now # <----- end + end_time = self.env.now # <----- end self.write_latencies.append(end_time - start_time) self.waf["request"] += extent.lpn_count # self.waf["actual"] += pages_to_write - write_timeline(self.conf, self.recorder, - op_id = op_id, op = 'write_ext', arg = extent.lpn_start, - start_time = start_time, end_time = end_time) - - def _write_single_mvpngroup(self, ext_single_m_vpn, ppns_to_write, - tag=None): + write_timeline( + self.conf, + self.recorder, + op_id=op_id, + op="write_ext", + arg=extent.lpn_start, + start_time=start_time, + end_time=end_time, + ) + + def _write_single_mvpngroup(self, ext_single_m_vpn, ppns_to_write, tag=None): m_vpn = self.conf.lpn_to_m_vpn(ext_single_m_vpn.lpn_start) p_relocate = self.env.process( self._update_metadata_for_relocating_lpns( - ext_single_m_vpn.lpn_iter(), - new_ppns=ppns_to_write, tag=tag)) + ext_single_m_vpn.lpn_iter(), new_ppns=ppns_to_write, tag=tag + ) + ) p_w_user = self.env.process(self._program_user_data(ppns_to_write, tag)) @@ -255,11 +281,11 @@ def _program_user_data(self, ppns_to_write, tag=None): start_time = self.env.now op_id = self.recorder.get_unique_num() - channel_tag = self.recorder.get_tag('write_user', tag) + channel_tag = self.recorder.get_tag("write_user", tag) self.waf["actual"] += len(ppns_to_write) yield self.env.process( - self.flash.rw_ppns(ppns_to_write, 'write', - tag = channel_tag)) + self.flash.rw_ppns(ppns_to_write, "write", tag=channel_tag) + ) # write_timeline(self.conf, self.recorder, # op_id = op_id, op = 'write_user_data', arg = len(ppns_to_write), @@ -269,16 +295,14 @@ def _update_metadata_for_relocating_lpns(self, lpns, new_ppns, tag=None): """ This may be parallelized. """ - old_ppns = yield self.env.process( - self._mappings.lpns_to_ppns(lpns, tag)) + old_ppns = yield self.env.process(self._mappings.lpns_to_ppns(lpns, tag)) for lpn, old_ppn, new_ppn in zip(lpns, old_ppns, new_ppns): yield self.env.process( - self._update_metadata_for_relocating_lpn( - lpn, old_ppn, new_ppn, tag)) + self._update_metadata_for_relocating_lpn(lpn, old_ppn, new_ppn, tag) + ) - def _update_metadata_for_relocating_lpn(self, lpn, old_ppn, new_ppn, - tag=None): + def _update_metadata_for_relocating_lpn(self, lpn, old_ppn, new_ppn, tag=None): """ contents of lpn used to be in old_ppn, but now it is in new_ppn. This function adjust all metadata to reflect the change. @@ -303,8 +327,7 @@ def _update_metadata_for_relocating_lpn(self, lpn, old_ppn, new_ppn, raise NotImplementedError() """ # mappings in cache - yield self.env.process( - self._mappings.update(lpn=lpn, ppn=new_ppn, tag=tag)) + yield self.env.process(self._mappings.update(lpn=lpn, ppn=new_ppn, tag=tag)) # mappings on flash # handled by _mappings @@ -314,19 +337,29 @@ def _update_metadata_for_relocating_lpn(self, lpn, old_ppn, new_ppn, # oob state # oob ppn->lpn/vpn - self.oob.relocate_data_page(lpn=lpn, old_ppn=old_ppn, new_ppn=new_ppn, - update_time=True) + self.oob.relocate_data_page( + lpn=lpn, old_ppn=old_ppn, new_ppn=new_ppn, update_time=True + ) # blockpool # should be handled when we got new_ppn def read_ext(self, extent): req_size = extent.lpn_count * self.conf.page_size - self.recorder.add_to_general_accumulater('traffic', 'read', req_size) + self.recorder.add_to_general_accumulater("traffic", "read", req_size) self.read_bytes += req_size self.rw_events += 1 if self.read_bytes > self.pre_read_bytes + self.display_interval: - log_msg('Event', self.rw_events, 'Read (MB)', self.read_bytes / MB, 'reading', round(float(req_size) / MB, 2), "Misses", self._mappings.misses)#"Miss ratio %.2f" % (self._mappings.misses / float(self._mappings.misses + self._mappings.hits))) + log_msg( + "Event", + self.rw_events, + "Read (MB)", + old_div(self.read_bytes, MB), + "reading", + round(float(req_size) / MB, 2), + "Misses", + self._mappings.misses, + ) # "Miss ratio %.2f" % (self._mappings.misses / float(self._mappings.misses + self._mappings.hits))) sys.stdout.flush() self.pre_read_bytes = self.read_bytes @@ -336,11 +369,12 @@ def read_ext(self, extent): op_id = self.recorder.get_unique_num() start_time = self.env.now - procs = [] + procs = [] for ext_single_m_vpn in ext_list: p = self.env.process( - self._read_single_mvpngroup(ext_single_m_vpn, tag=op_id)) - + self._read_single_mvpngroup(ext_single_m_vpn, tag=op_id) + ) + procs.append(p) ret = yield simpy.events.AllOf(self.env, procs) @@ -349,11 +383,17 @@ def read_ext(self, extent): end_time = self.env.now if lpns_to_read > 0: - self.read_latencies.append((end_time - start_time)/lpns_to_read) - - write_timeline(self.conf, self.recorder, - op_id = op_id, op = 'read_ext', arg = extent.lpn_start, - start_time = start_time, end_time = end_time) + self.read_latencies.append(old_div((end_time - start_time), lpns_to_read)) + + write_timeline( + self.conf, + self.recorder, + op_id=op_id, + op="read_ext", + arg=extent.lpn_start, + start_time=start_time, + end_time=end_time, + ) # self.env.exit(total_read) @@ -366,29 +406,36 @@ def _read_single_mvpngroup(self, ext_single_m_vpn, tag=None): lpns_to_read.append(lpn) ppns_to_read = yield self.env.process( - self._mappings.lpns_to_ppns(lpns_to_read, - tag=tag)) + self._mappings.lpns_to_ppns(lpns_to_read, tag=tag) + ) ppns_to_read = remove_invalid_ppns(ppns_to_read) op_id = self.recorder.get_unique_num() start_time = self.env.now yield self.env.process( - self.flash.rw_ppns(ppns_to_read, 'read', - tag=self.recorder.get_tag('read_user', tag))) + self.flash.rw_ppns( + ppns_to_read, "read", tag=self.recorder.get_tag("read_user", tag) + ) + ) # write_timeline(self.conf, self.recorder, # op_id=op_id, op='read_user_data', arg=len(ppns_to_read), # start_time = start_time, end_time = self.env.now) - - self.env.exit(len(ppns_to_read)) + + return len(ppns_to_read) def discard_ext(self, extent): req_size = extent.lpn_count * self.conf.page_size - self.recorder.add_to_general_accumulater('traffic', 'discard', req_size) + self.recorder.add_to_general_accumulater("traffic", "discard", req_size) self.discarded_bytes += req_size if self.discarded_bytes > self.pre_discarded_bytes + self.display_interval: - log_msg('Discarded (MB)', self.pre_discarded_bytes / MB, 'discarding', round(float(req_size) / MB, 2)) + log_msg( + "Discarded (MB)", + old_div(self.pre_discarded_bytes, MB), + "discarding", + round(float(req_size) / MB, 2), + ) sys.stdout.flush() self.pre_discarded_bytes = self.discarded_bytes @@ -405,12 +452,14 @@ def _discard_single_mvpngroup(self, ext_single_m_vpn): m_vpn = self.conf.lpn_to_m_vpn(ext_single_m_vpn.lpn_start) ppns_to_invalidate = yield self.env.process( - self._mappings.lpns_to_ppns(ext_single_m_vpn.lpn_iter())) + self._mappings.lpns_to_ppns(ext_single_m_vpn.lpn_iter()) + ) ppns_to_invalidate = remove_invalid_ppns(ppns_to_invalidate) - mapping_dict = dict(itertools.izip_longest( - ppns_to_invalidate, (), fillvalue = UNINITIATED)) + mapping_dict = dict( + itertools.zip_longest(ppns_to_invalidate, (), fillvalue=UNINITIATED) + ) self._mappings.update_batch(mapping_dict) self.oob.invalidate_ppns(ppns_to_invalidate) @@ -420,8 +469,8 @@ def is_cleaning_needed(self): def is_wear_leveling_needed(self): factor, diff = self.block_pool.get_wear_status() - self.recorder.append_to_value_list('wear_diff', diff) - print 'ddddddddddddddddddddiiiiiiiiiiifffffffffff', diff + self.recorder.append_to_value_list("wear_diff", diff) + print(("ddddddddddddddddddddiiiiiiiiiiifffffffffff", diff)) return self.block_pool.need_wear_leveling() @@ -434,30 +483,31 @@ def level_wear(self): def snapshot_valid_ratios(self): victim_blocks = VictimBlocks(self.conf, self.block_pool, self.oob) ratios = victim_blocks.get_valid_ratio_counter_of_used_blocks() - self.recorder.append_to_value_list('ftl_func_valid_ratios', - ratios) + self.recorder.append_to_value_list("ftl_func_valid_ratios", ratios) def snapshot_user_traffic(self): return - self.recorder.append_to_value_list('ftl_func_user_traffic', - {'timestamp': self.env.now/float(SEC), - 'write_traffic_size': self.written_bytes, - 'read_traffic_size': self.read_bytes, - 'discard_traffic_size': self.discarded_bytes, - }, - ) + self.recorder.append_to_value_list( + "ftl_func_user_traffic", + { + "timestamp": self.env.now / float(SEC), + "write_traffic_size": self.written_bytes, + "read_traffic_size": self.read_bytes, + "discard_traffic_size": self.discarded_bytes, + }, + ) def snapshot_erasure_count_dist(self): dist = self.block_pool.get_erasure_count_dist() - print self.env.now - print dist - self.recorder.append_to_value_list('ftl_func_erasure_count_dist', - dist) + print(self.env.now) + print(dist) + self.recorder.append_to_value_list("ftl_func_erasure_count_dist", dist) def remove_invalid_ppns(ppns): return [ppn for ppn in ppns if not ppn in (UNINITIATED, MISS)] + def split_ext_to_mvpngroups(conf, extent): """ return a list of extents, each belongs to one m_vpn @@ -467,15 +517,13 @@ def split_ext_to_mvpngroups(conf, extent): cur_m_vpn = conf.lpn_to_m_vpn(lpn) if i == 0: # intialization - cur_group_extent = Extent(lpn_start = extent.lpn_start, - lpn_count = 1) + cur_group_extent = Extent(lpn_start=extent.lpn_start, lpn_count=1) group_extent_list.append(cur_group_extent) else: if cur_m_vpn == last_m_vpn: cur_group_extent.lpn_count += 1 else: - cur_group_extent = Extent(lpn_start = lpn, - lpn_count = 1) + cur_group_extent = Extent(lpn_start=lpn, lpn_count=1) group_extent_list.append(cur_group_extent) last_m_vpn = cur_m_vpn @@ -487,10 +535,11 @@ class MappingDict(dict): """ Used to map lpn->ppn """ + pass -class FlashTransmitMixin(object): +class FlashTransmitMixin: def _write_back(self, m_vpn, tag=None): assert m_vpn in self._trans_page_locks.locked_addrs @@ -498,13 +547,14 @@ def _write_back(self, m_vpn, tag=None): # We have to mark it clean before writing it back because # if we do it after writing flash, the cache may already changed - self._lpn_table.mark_clean_multiple(mapping_in_cache.keys()) + self._lpn_table.mark_clean_multiple(list(mapping_in_cache.keys())) if len(mapping_in_cache) < self.conf.n_mapping_entries_per_page: # Not all mappings are in cache - self.recorder.count_me("translation", 'read_trans_page-for-write-back') + self.recorder.count_me("translation", "read_trans_page-for-write-back") mapping_in_flash = yield self.env.process( - self._read_translation_page(m_vpn, tag)) + self._read_translation_page(m_vpn, tag) + ) latest_mapping = mapping_in_flash latest_mapping.update(mapping_in_cache) else: @@ -512,13 +562,13 @@ def _write_back(self, m_vpn, tag=None): latest_mapping = mapping_in_cache yield self.env.process( - self.__update_mapping_on_flash(m_vpn, latest_mapping, tag)) + self.__update_mapping_on_flash(m_vpn, latest_mapping, tag) + ) def _read_translation_page(self, m_vpn, tag=None): lpns = self.conf.m_vpn_to_lpns(m_vpn) mapping_dict = self.mapping_on_flash.lpns_to_ppns(lpns) - # as if we readlly read from flash m_ppn = self.directory.m_vpn_to_m_ppn(m_vpn) @@ -526,21 +576,23 @@ def _read_translation_page(self, m_vpn, tag=None): start_time = self.env.now yield self.env.process( - self.flash.rw_ppn_extent(m_ppn, 1, 'read', - tag = self.recorder.get_tag('read_trans', tag))) + self.flash.rw_ppn_extent( + m_ppn, 1, "read", tag=self.recorder.get_tag("read_trans", tag) + ) + ) # write_timeline(self.conf, self.recorder, # op_id = op_id, op = 'read_trans_page', arg = m_vpn, # start_time = start_time, end_time = self.env.now) - self.env.exit(mapping_dict) + return mapping_dict def __update_mapping_on_flash(self, m_vpn, mapping_dict, tag=None): """ mapping_dict should only has lpns belonging to m_vpn """ # mapping_dict has to have all and only the entries of m_vpn - lpn_sample = mapping_dict.keys()[0] + lpn_sample = list(mapping_dict.keys())[0] tmp_m_vpn = self.conf.lpn_to_m_vpn(lpn_sample) assert tmp_m_vpn == m_vpn assert len(mapping_dict) == self.conf.n_mapping_entries_per_page @@ -559,16 +611,19 @@ def __program_translation_page(self, m_vpn, tag=None): self.ftl.waf["actual"] += 1 yield self.env.process( - self.flash.rw_ppn_extent(new_m_ppn, 1, 'write', - tag = self.recorder.get_tag('prog_trans', tag))) + self.flash.rw_ppn_extent( + new_m_ppn, 1, "write", tag=self.recorder.get_tag("prog_trans", tag) + ) + ) # write_timeline(self.conf, self.recorder, # op_id = op_id, op = 'prog_trans_page', arg = m_vpn, # start_time = start_time, end_time = self.env.now) - self.oob.relocate_trans_page(m_vpn=m_vpn, old_ppn=old_m_ppn, - new_ppn = new_m_ppn, update_time=True) - self.directory.update_mapping(m_vpn = m_vpn, m_ppn = new_m_ppn) + self.oob.relocate_trans_page( + m_vpn=m_vpn, old_ppn=old_m_ppn, new_ppn=new_m_ppn, update_time=True + ) + self.directory.update_mapping(m_vpn=m_vpn, m_ppn=new_m_ppn) assert self.oob.states.is_page_valid(old_m_ppn) == False assert self.oob.states.is_page_valid(new_m_ppn) == True @@ -576,31 +631,28 @@ def __program_translation_page(self, m_vpn, tag=None): assert self.directory.m_vpn_to_m_ppn(m_vpn) == new_m_ppn -class InsertMixin(object): +class InsertMixin: def _insert_new_mapping(self, lpn, ppn, tag=None): - assert not self._lpn_table.has_lpn(lpn), 'lpn: {}'.format(lpn) + assert not self._lpn_table.has_lpn(lpn), f"lpn: {lpn}" # no free space for this insertion, free and lock 1 - locked_rows = yield self.env.process( - self.__add_locked_room_for_insert(tag=tag)) + locked_rows = yield self.env.process(self.__add_locked_room_for_insert(tag=tag)) locked_row_id = locked_rows[0] # even if you inserting and loading of the same lpn can be # serialize, but loading of lpns in the same mvpn which # also brings lpn to memory is not serialized. - self._lpn_table.add_lpn(rowid = locked_row_id, - lpn = lpn, ppn = ppn, dirty = True) + self._lpn_table.add_lpn(rowid=locked_row_id, lpn=lpn, ppn=ppn, dirty=True) def _add_to_free(self, lpn, ppn): locked_row_id = self._lpn_table.lock_free_row() - self._lpn_table.add_lpn(rowid = locked_row_id, - lpn = lpn, ppn = ppn, dirty = True) + self._lpn_table.add_lpn(rowid=locked_row_id, lpn=lpn, ppn=ppn, dirty=True) def __add_locked_room_for_insert(self, tag=None): locked_row_ids = [] row_id = yield self.env.process(self.__evict_entry_for_insert(tag)) locked_row_ids.append(row_id) - self.env.exit(locked_row_ids) + return locked_row_ids def __evict_entry_for_insert(self, tag=None): victim_rows = self._victim_row(1, avoid_m_vpns=[]) @@ -610,17 +662,18 @@ def __evict_entry_for_insert(self, tag=None): yield self._concurrent_trans_quota.get(1) # lock m_vpn - m_vpn = self.conf.lpn_to_m_vpn(lpn = victim_row.lpn) + m_vpn = self.conf.lpn_to_m_vpn(lpn=victim_row.lpn) tp_req = self._trans_page_locks.get_request(m_vpn) yield tp_req self._trans_page_locks.locked_addrs.add(m_vpn) - #if victim_row.dirty == True: - self.recorder.count_me('translation', 'write-back-dirty-for-insert') + # if victim_row.dirty == True: + self.recorder.count_me("translation", "write-back-dirty-for-insert") yield self.env.process(self._write_back(m_vpn, tag)) - assert self._lpn_table.has_lpn(victim_row.lpn), \ - "lpn_table does not has lpn {}.".format(victim_row.lpn) + assert self._lpn_table.has_lpn(victim_row.lpn), ( + f"lpn_table does not has lpn {victim_row.lpn}." + ) # Hmm... victim_row could be updated when you are writing them back # this is OK because we have written the old mapping to flash # and the new one in cache is marked dirty, we will @@ -629,7 +682,7 @@ def __evict_entry_for_insert(self, tag=None): assert victim_row.state == USED_AND_HOLD victim_row.state = USED - self.recorder.count_me('translation', 'delete-lpn-in-table-for-insert') + self.recorder.count_me("translation", "delete-lpn-in-table-for-insert") locked_row_id = self._lpn_table.delete_lpn_and_lock(victim_row.lpn) self._trans_page_locks.release_request(m_vpn, tp_req) @@ -637,10 +690,10 @@ def __evict_entry_for_insert(self, tag=None): yield self._concurrent_trans_quota.put(1) - self.env.exit(locked_row_id) + return locked_row_id -class LoadMixin(object): +class LoadMixin: def _load_missing(self, m_vpn, wanted_lpn, tag=None): """ Return True if we really load flash page @@ -658,12 +711,15 @@ def _load_missing(self, m_vpn, wanted_lpn, tag=None): if n_more > 0: more_locked_rows = yield self.env.process( - self.__add_locked_room_for_load(n_more, loading_m_vpn=m_vpn, - tag=tag)) + self.__add_locked_room_for_load( + n_more, loading_m_vpn=m_vpn, tag=tag + ) + ) locked_rows += more_locked_rows yield self.env.process( - self.__load_to_locked_space(m_vpn, locked_rows, tag=tag)) + self.__load_to_locked_space(m_vpn, locked_rows, tag=tag) + ) loaded = True else: @@ -676,7 +732,7 @@ def _load_missing(self, m_vpn, wanted_lpn, tag=None): yield self._concurrent_trans_quota.put(2) - self.env.exit((loaded, ppn)) + return (loaded, ppn) def __add_locked_room_for_load(self, n_needed, loading_m_vpn, tag=None): # locked_row_ids = [] @@ -686,23 +742,25 @@ def __add_locked_room_for_load(self, n_needed, loading_m_vpn, tag=None): # locked_row_ids.append(row_id) locked_row_ids = yield self.env.process( - self.__evict_entry_for_load(n_needed, loading_m_vpn, tag)) - self.env.exit(locked_row_ids) + self.__evict_entry_for_load(n_needed, loading_m_vpn, tag) + ) + return locked_row_ids def __evict_entry_for_load(self, n_needed, loading_m_vpn, tag=None): locked_row_ids = [] - victim_rows = self._victim_row(n_needed, - [loading_m_vpn] + list(self._trans_page_locks.locked_addrs)) + victim_rows = self._victim_row( + n_needed, [loading_m_vpn] + list(self._trans_page_locks.locked_addrs) + ) m_vpns = defaultdict(list) for victim_row in victim_rows: victim_row.state = USED_AND_HOLD - m_vpn = self.conf.lpn_to_m_vpn(lpn = victim_row.lpn) + m_vpn = self.conf.lpn_to_m_vpn(lpn=victim_row.lpn) m_vpns[m_vpn].append(victim_row) - #tp_req = self._trans_page_locks.get_request(m_vpn) - #yield tp_req - for m_vpn, victims in m_vpns.items(): + # tp_req = self._trans_page_locks.get_request(m_vpn) + # yield tp_req + for m_vpn, victims in list(m_vpns.items()): tp_req = self._trans_page_locks.get_request(m_vpn) yield tp_req self._trans_page_locks.locked_addrs.add(m_vpn) @@ -710,60 +768,63 @@ def __evict_entry_for_load(self, n_needed, loading_m_vpn, tag=None): written_back = False for victim_row in victims: if victim_row.dirty and not written_back: - self.recorder.count_me('translation', 'write-back-dirty-for-load') + self.recorder.count_me("translation", "write-back-dirty-for-load") yield self.env.process(self._write_back(m_vpn, tag)) written_back = True # after writing back, this lpn could already been deleted # by another _evict_entry()? - assert self._lpn_table.has_lpn(victim_row.lpn), \ - "lpn_table does not has lpn {}.".format(victim_row.lpn) + assert self._lpn_table.has_lpn(victim_row.lpn), ( + f"lpn_table does not has lpn {victim_row.lpn}." + ) # assert victim_row.dirty == False, repr(victim_row) assert victim_row.state == USED_AND_HOLD victim_row.state = USED # This is the only place that we delete a lpn - self.recorder.count_me('translation', 'delete-lpn-in-table-for-load') + self.recorder.count_me("translation", "delete-lpn-in-table-for-load") locked_row_id = self._lpn_table.delete_lpn_and_lock(victim_row.lpn) locked_row_ids.append(locked_row_id) self._trans_page_locks.release_request(m_vpn, tp_req) self._trans_page_locks.locked_addrs.remove(m_vpn) - self.env.exit(locked_row_ids) + return locked_row_ids def __load_to_locked_space(self, m_vpn, locked_rows, tag=None): """ It should not call _write_back() directly or indirectly as it will deadlock. """ - self.recorder.count_me('translation', 'read-trans-for-load') - mapping_dict = yield self.env.process( - self._read_translation_page(m_vpn, tag)) + self.recorder.count_me("translation", "read-trans-for-load") + mapping_dict = yield self.env.process(self._read_translation_page(m_vpn, tag)) uncached_mapping = self.__get_uncached_mappings(mapping_dict) n_needed = len(uncached_mapping) needed_rows = locked_rows[:n_needed] unused_rows = locked_rows[n_needed:] - self._lpn_table.add_lpns(needed_rows, uncached_mapping, False, - as_least_recent = True) + self._lpn_table.add_lpns( + needed_rows, uncached_mapping, False, as_least_recent=True + ) self._lpn_table.unlock_free_rows(unused_rows) def __get_uncached_mappings(self, mapping_dict): uncached_mapping = {} - for lpn, ppn in mapping_dict.items(): + for lpn, ppn in list(mapping_dict.items()): if not self._lpn_table.has_lpn(lpn): uncached_mapping[lpn] = ppn return uncached_mapping -class FlushMixin(object): + +class FlushMixin: """ Write back all dirty entries in translation cache Flush has to be run alone without other processes. Run barrier before calling flush """ + def _flush(self, tag=None): for lpn, row in self._lpn_table.least_to_most_lpn_items(): if row.dirty is True: @@ -774,7 +835,7 @@ def _flush(self, tag=None): yield tp_req self._trans_page_locks.locked_addrs.add(m_vpn) - self.recorder.count_me('translation', 'write-back-dirty-for-flush') + self.recorder.count_me("translation", "write-back-dirty-for-flush") yield self.env.process(self._write_back(m_vpn, tag)) self._trans_page_locks.release_request(m_vpn, tp_req) @@ -785,8 +846,20 @@ class MappingCache(FlashTransmitMixin, InsertMixin, LoadMixin, FlushMixin): """ TODO: should separate operations that do/do not change recency """ - def __init__(self, ftl, confobj, block_pool, flashobj, oobobj, recorderobj, - envobj, directory, mapping_on_flash, trans_page_locks): + + def __init__( + self, + ftl, + confobj, + block_pool, + flashobj, + oobobj, + recorderobj, + envobj, + directory, + mapping_on_flash, + trans_page_locks, + ): self.ftl = ftl self.conf = confobj self.flash = flashobj @@ -801,20 +874,23 @@ def __init__(self, ftl, confobj, block_pool, flashobj, oobobj, recorderobj, self._trans_page_locks = trans_page_locks - n_cache_tps = self.conf.n_cache_entries / self.conf.n_mapping_entries_per_page + n_cache_tps = old_div( + self.conf.n_cache_entries, self.conf.n_mapping_entries_per_page + ) log_msg(self.conf.n_cache_entries, self.conf.n_mapping_entries_per_page) capsize = max(n_cache_tps - 1, 2) - log_msg('max number of tps in cache', n_cache_tps) + log_msg("max number of tps in cache", n_cache_tps) - self._concurrent_trans_quota = simpy.Container(self.env, init=capsize, - capacity=capsize) + self._concurrent_trans_quota = simpy.Container( + self.env, init=capsize, capacity=capsize + ) self._m_vpn_interface_lock = LockPool(self.env) self.hits = 1 self.misses = 0 def update_batch(self, mapping_dict, tag=None): - for lpn, ppn in mapping_dict.items(): + for lpn, ppn in list(mapping_dict.items()): yield self.env.process(self.update(lpn, ppn, tag)) def update(self, lpn, ppn, tag=None): @@ -826,14 +902,14 @@ def update(self, lpn, ppn, tag=None): yield req if self._lpn_table.has_lpn(lpn): - #if ppn != UNINITIATED: + # if ppn != UNINITIATED: # self.hits += 1 - self.recorder.count_me('translation', 'overwrite-in-cache') + self.recorder.count_me("translation", "overwrite-in-cache") self._lpn_table.overwrite_lpn(lpn, ppn, dirty=True) else: # self.misses += 1 if self._lpn_table.n_free_rows() > 0: - self.recorder.count_me('translation', 'insert-to-free') + self.recorder.count_me("translation", "insert-to-free") self._add_to_free(lpn, ppn) else: self.misses += 1 @@ -850,7 +926,7 @@ def lpns_to_ppns(self, lpns, tag=None): for lpn in lpns: ppn = yield self.env.process(self.lpn_to_ppn(lpn, tag)) ppns.append(ppn) - self.env.exit(ppns) + return ppns def lpn_to_ppn(self, lpn, tag=None): """ @@ -869,7 +945,8 @@ def lpn_to_ppn(self, lpn, tag=None): # self.misses += 1 # print(lpn, ppn) loaded, ppn = yield self.env.process( - self._load_missing(m_vpn, wanted_lpn=lpn, tag=tag)) + self._load_missing(m_vpn, wanted_lpn=lpn, tag=tag) + ) assert ppn != MISS else: if ppn != UNINITIATED: @@ -884,7 +961,7 @@ def lpn_to_ppn(self, lpn, tag=None): self.recorder.count_me("Mapping_Cache", "hit") self._m_vpn_interface_lock.release_request(m_vpn, req) - self.env.exit(ppn) + return ppn def flush(self): yield self.env.process(self._flush()) @@ -892,7 +969,7 @@ def flush(self): def drop(self): "flush before dropping, otherwise mapping will be lost" for lpn, row in self._lpn_table.least_to_most_lpn_items(): - self.recorder.count_me('translation', 'delete-lpn-in-table-for-drop') + self.recorder.count_me("translation", "delete-lpn-in-table-for-drop") self._lpn_table.delete_lpn_and_lock(lpn) self._lpn_table.free_row(lpn) @@ -909,20 +986,26 @@ def _victim_row(self, n, avoid_m_vpns): got += 1 if got >= n: return rows - log_msg("Cannot find a victim. Current stats: {}"\ - ", avoid_m_vpns: {}.\n" - .format(str(self._lpn_table.stats()), avoid_m_vpns)) + log_msg( + f"Cannot find a victim. Current stats: {str(self._lpn_table.stats())}" + f", avoid_m_vpns: {avoid_m_vpns}.\n" + ) exit() -FREE, FREE_AND_LOCKED, USED, USED_AND_LOCKED, USED_AND_HOLD = \ - 'FREE', 'FREE_AND_LOCKED', 'USED', 'USED_AND_LOCKED', 'USED_AND_HOLD' +FREE, FREE_AND_LOCKED, USED, USED_AND_LOCKED, USED_AND_HOLD = ( + "FREE", + "FREE_AND_LOCKED", + "USED", + "USED_AND_LOCKED", + "USED_AND_HOLD", +) + -class LpnTable(object): +class LpnTable: def __init__(self, n_rows): self._n_rows = n_rows - # lpns to Row instances, it is a dict # {lpn1: row1, lpn2: row2, ...} # self._lpn_to_row = SegmentedLruCache(n_rows, 0.5) @@ -935,7 +1018,7 @@ def __init__(self, n_rows): def _fresh_rows(self): for i in range(self._n_rows): - self._rows.append(Row(lpn = None, ppn = None, dirty = False, state = FREE, rowid = i)) + self._rows.append(Row(lpn=None, ppn=None, dirty=False, state=FREE, rowid=i)) # stripe = int((1000**4/4096) / self._n_rows) # for i in range(self._n_rows): # self._rows.append(Row(lpn = i*stripe, ppn = i*stripe, dirty = True, state = FREE_AND_LOCKED, rowid = i)) @@ -969,7 +1052,7 @@ def lock_free_row(self): """FREE TO FREE_AND_LOCKED""" if len(self.free_list) > 0: row = self._rows[self.free_list.pop(0)] - assert(row.state == FREE) + assert row.state == FREE row.state = FREE_AND_LOCKED return row.rowid return None @@ -995,15 +1078,15 @@ def lock_free_rows(self, n): got = 0 for index in self.free_list[:n]: - row = self._rows[index] - assert(row.state == FREE) + row = self._rows[index] + assert row.state == FREE row.state = FREE_AND_LOCKED row_ids.append(row.rowid) got += 1 if got == n: break - + del self.free_list[:n] return row_ids @@ -1047,14 +1130,15 @@ def unhold_used_rows(self, row_ids): for rowid in row_ids: self.unhold_used_row(rowid) - def add_lpns(self, row_ids, mapping_dict, dirty, as_least_recent = False): - assert len(row_ids) == len(mapping_dict), \ - "{} == {}".format(len(row_ids), len(mapping_dict)) - for row_id, (lpn, ppn) in zip(row_ids, mapping_dict.items()): + def add_lpns(self, row_ids, mapping_dict, dirty, as_least_recent=False): + assert len(row_ids) == len(mapping_dict), ( + f"{len(row_ids)} == {len(mapping_dict)}" + ) + for row_id, (lpn, ppn) in zip(row_ids, list(mapping_dict.items())): self.add_lpn(row_id, lpn, ppn, dirty, as_least_recent) - def add_lpn(self, rowid, lpn, ppn, dirty, as_least_recent = False): - assert self.has_lpn(lpn) == False, "lpn is {}.".format(lpn) + def add_lpn(self, rowid, lpn, ppn, dirty, as_least_recent=False): + assert self.has_lpn(lpn) == False, f"lpn is {lpn}." row = self._rows[rowid] @@ -1109,7 +1193,7 @@ def delete_lpn_and_lock(self, lpn): return row.rowid def has_lpn(self, lpn): - return self._lpn_to_row.has_key(lpn) + return lpn in self._lpn_to_row # try: # row = self._lpn_to_row.peek(lpn) # except KeyError: @@ -1125,8 +1209,9 @@ class LpnTableMvpn(LpnTable): """ With addition supports related to m_vpn """ + def __init__(self, conf): - super(LpnTableMvpn, self).__init__(conf.n_cache_entries) + super().__init__(conf.n_cache_entries) self.conf = conf def least_to_most_lpn_items(self): @@ -1137,8 +1222,7 @@ def needed_space_for_m_vpn(self, m_vpn): return self.conf.n_mapping_entries_per_page - len(cached_mappings) def get_m_vpn_mappings(self, m_vpn): - """ return all the mappings of m_vpn that are in cache - """ + """return all the mappings of m_vpn that are in cache""" rows = self._rows_of_m_vpn(m_vpn) mapping_dict = {} for row in rows: @@ -1175,7 +1259,7 @@ def get_un_cached_lpn_of_m_vpn(self, m_vpn): return uncached_lpns -class _Row(object): +class _Row: def __init__(self, lpn, ppn, dirty, state, rowid): self.lpn = lpn self.ppn = ppn @@ -1189,7 +1273,7 @@ def clear_data(self): self.dirty = None -class Row(object): +class Row: def __init__(self, lpn, ppn, dirty, state, rowid): self._lpn = lpn self._ppn = ppn @@ -1198,8 +1282,9 @@ def __init__(self, lpn, ppn, dirty, state, rowid): self._rowid = rowid def _assert_modification_allowed(self): - assert self._state in (FREE_AND_LOCKED, USED, USED_AND_HOLD), \ - "current state {}".format(self._state) + assert self._state in (FREE_AND_LOCKED, USED, USED_AND_HOLD), ( + f"current state {self._state}" + ) @property def lpn(self): @@ -1244,22 +1329,19 @@ def state(self, state_value): """ # check state transition if state_value == FREE: - assert self._state == FREE_AND_LOCKED, \ - "current state {}".format(self._state) + assert self._state == FREE_AND_LOCKED, f"current state {self._state}" elif state_value == FREE_AND_LOCKED: - assert self._state in (FREE, USED), \ - "current state {}".format(self._state) + assert self._state in (FREE, USED), f"current state {self._state}" elif state_value == USED: - assert self._state in (FREE_AND_LOCKED, USED_AND_LOCKED, USED_AND_HOLD), \ - "current state {}".format(self._state) + assert self._state in (FREE_AND_LOCKED, USED_AND_LOCKED, USED_AND_HOLD), ( + f"current state {self._state}" + ) elif state_value == USED_AND_LOCKED: - assert self._state == USED, \ - "current state {}".format(self._state) + assert self._state == USED, f"current state {self._state}" elif state_value == USED_AND_HOLD: - assert self._state == USED, \ - "current state {}".format(self._state) + assert self._state == USED, f"current state {self._state}" else: - raise RuntimeError("{} is not a valid state".format(state_value)) + raise RuntimeError(f"{state_value} is not a valid state") self._state = state_value @property @@ -1272,33 +1354,36 @@ def clear_data(self): self.dirty = None def __repr__(self): - return "lpn:{}, ppn:{}, dirty:{}, rowid:{}".format(self.lpn, - self.ppn, self.dirty, self._rowid) + return ( + f"lpn:{self.lpn}, ppn:{self.ppn}, dirty:{self.dirty}, rowid:{self._rowid}" + ) -class CacheEntryData(object): +class CacheEntryData: """ This is a helper class that store entry data for a LPN """ + def __init__(self, lpn, ppn, dirty): self.lpn = lpn self.ppn = ppn self.dirty = dirty def __repr__(self): - return "lpn:{}, ppn:{}, dirty:{}".format(self.lpn, - self.ppn, self.dirty) + return f"lpn:{self.lpn}, ppn:{self.ppn}, dirty:{self.dirty}" -class MappingOnFlash(object): +class MappingOnFlash: """ This mapping table is for data pages, not for translation pages. GMT should have entries as many as the number of pages in flash """ + def __init__(self, confobj): if not isinstance(confobj, config.Config): - raise TypeError("confobj is not conf.Config. it is {}". - format(type(confobj).__name__)) + raise TypeError( + f"confobj is not conf.Config. it is {type(confobj).__name__}" + ) self.conf = confobj @@ -1320,7 +1405,7 @@ def update(self, lpn, ppn): self.entries[lpn] = ppn def batch_update(self, mapping_dict): - for lpn, ppn in mapping_dict.items(): + for lpn, ppn in list(mapping_dict.items()): self.update(lpn, ppn) def lpns_to_ppns(self, lpns): @@ -1331,14 +1416,15 @@ def lpns_to_ppns(self, lpns): return d def __repr__(self): - return "global mapping table: {}".format(repr(self.entries)) + return f"global mapping table: {repr(self.entries)}" -class GlobalTranslationDirectory(object): +class GlobalTranslationDirectory: """ This is an in-memory data structure. It is only for book keeping. It used to remeber thing so that we don't lose it. """ + def __init__(self, confobj, oob, block_pool): self.conf = confobj @@ -1375,8 +1461,9 @@ def _initialize(self): # Note that we don't actually read or write flash self.add_mapping(m_vpn=m_vpn, m_ppn=m_ppn) # update oob of the translation page - self.oob.relocate_trans_page(m_vpn=m_vpn, old_ppn=UNINITIATED, - new_ppn=m_ppn, update_time=True) + self.oob.relocate_trans_page( + m_vpn=m_vpn, old_ppn=UNINITIATED, new_ppn=m_ppn, update_time=True + ) def m_vpn_to_m_ppn(self, m_vpn): """ @@ -1385,13 +1472,12 @@ def m_vpn_to_m_ppn(self, m_vpn): try: return self.mapping[m_vpn] except: - print("cmt mapping entry not enough", m_vpn, len(self.mapping)) + print(("cmt mapping entry not enough", m_vpn, len(self.mapping))) exit(0) def add_mapping(self, m_vpn, m_ppn): - if self.mapping.has_key(m_vpn): - raise RuntimeError("self.mapping already has m_vpn:{}"\ - .format(m_vpn)) + if m_vpn in self.mapping: + raise RuntimeError(f"self.mapping already has m_vpn:{m_vpn}") self.mapping[m_vpn] = m_ppn def update_mapping(self, m_vpn, m_ppn): @@ -1408,9 +1494,11 @@ def lpn_to_m_ppn(self, lpn): def __repr__(self): return repr(self.mapping) -class WearLevelingVictimBlocks(object): - TYPE_DATA = 'TYPE_DATA' - TYPE_TRANS = 'TYPE_TRANS' + +class WearLevelingVictimBlocks: + TYPE_DATA = "TYPE_DATA" + TYPE_TRANS = "TYPE_TRANS" + def __init__(self, conf, block_pool, oob, n_victims): self._conf = conf self._block_pool = block_pool @@ -1450,16 +1538,17 @@ def iterator_verbose(self): break -class VictimBlocks(object): - TYPE_DATA = 'TYPE_DATA' - TYPE_TRANS = 'TYPE_TRANS' +class VictimBlocks: + TYPE_DATA = "TYPE_DATA" + TYPE_TRANS = "TYPE_TRANS" + def __init__(self, conf, block_pool, oob): self._conf = conf self._block_pool = block_pool self._oob = oob def iterator(self): - for (_, _, block_num) in self.iterator_verbose(): + for _, _, block_num in self.iterator_verbose(): yield block_num def __str__(self): @@ -1480,7 +1569,7 @@ def get_valid_ratio_counter_of_used_blocks(self): counter = Counter() for block in used_blocks: valid_ratio = self._oob.states.block_valid_ratio(block) - ratio_str = "{0:.2f}".format(valid_ratio) + ratio_str = f"{valid_ratio:.2f}" counter[ratio_str] += 1 return counter @@ -1505,12 +1594,12 @@ def _form_tuples(self, used_blocks, block_type): if valid_ratio == 1: # skip all-valid blocks continue - if valid_ratio > self._conf['max_victim_valid_ratio']: + if valid_ratio > self._conf["max_victim_valid_ratio"]: # If valid ratio is too big, moving it does not provide # too much benefit. continue - victim_candidates.append( (valid_ratio, block_type, block) ) + victim_candidates.append((valid_ratio, block_type, block)) return victim_candidates @@ -1518,15 +1607,26 @@ def _victim_candidates(self): used_data_blocks = self._block_pool.data_usedblocks used_trans_blocks = self._block_pool.trans_usedblocks - candidate_tuples = self._form_tuples(used_data_blocks, self.TYPE_DATA) + \ - self._form_tuples(used_trans_blocks, self.TYPE_TRANS) + candidate_tuples = self._form_tuples( + used_data_blocks, self.TYPE_DATA + ) + self._form_tuples(used_trans_blocks, self.TYPE_TRANS) return candidate_tuples -class Cleaner(object): - def __init__(self, conf, flash, oob, block_pool, mappings, directory, rec, - env, trans_page_locks): +class Cleaner: + def __init__( + self, + conf, + flash, + oob, + block_pool, + mappings, + directory, + rec, + env, + trans_page_locks, + ): self.conf = conf self.flash = flash self.oob = oob @@ -1541,31 +1641,31 @@ def __init__(self, conf, flash, oob, block_pool, mappings, directory, rec, self._trans_page_locks = trans_page_locks self._datablockcleaner = DataBlockCleaner( - conf = self.conf, - flash = self.flash, - oob = self.oob, - block_pool = self.block_pool, - mappings = self.mappings, - rec = self.recorder, - env = self.env - ) + conf=self.conf, + flash=self.flash, + oob=self.oob, + block_pool=self.block_pool, + mappings=self.mappings, + rec=self.recorder, + env=self.env, + ) self._transblockcleaner = TransBlockCleaner( - conf = self.conf, - flash = self.flash, - oob = self.oob, - block_pool = self.block_pool, - mappings = self.mappings, - directory = self.directory, - rec = self.recorder, - env = self.env, - trans_page_locks = self._trans_page_locks - ) + conf=self.conf, + flash=self.flash, + oob=self.oob, + block_pool=self.block_pool, + mappings=self.mappings, + directory=self.directory, + rec=self.recorder, + env=self.env, + trans_page_locks=self._trans_page_locks, + ) # limit number of cleaner processes # self.n_cleaners = self.conf.n_channels_per_dev * 64 - self.n_cleaners = self.conf['n_gc_procs'] - print 'n_cleaners:', self.n_cleaners + self.n_cleaners = self.conf["n_gc_procs"] + print(("n_cleaners:", self.n_cleaners)) self._block_cleaner_res = simpy.Resource(self.env, capacity=self.n_cleaners) self.n_victim_per_batch = self.conf.n_channels_per_dev * 2 @@ -1576,7 +1676,7 @@ def __init__(self, conf, flash, oob, block_pool, mappings, directory, rec, self.gc_time_recorded = False def assert_threshold_sanity(self): - if self.conf['do_not_check_gc_setting'] is True: + if self.conf["do_not_check_gc_setting"] is True: return # check high threshold @@ -1584,18 +1684,23 @@ def assert_threshold_sanity(self): # than the file system size because the translation page overhead min_high = 1 / float(self.conf.over_provisioning) if self.conf.GC_high_threshold_ratio < min_high: - raise RuntimeError("GC_high_threshold_ratio is too low. "\ - "When file system is full, we will consistently try to "\ - "collect garbage. Too much overhead.") + raise RuntimeError( + "GC_high_threshold_ratio is too low. " + "When file system is full, we will consistently try to " + "collect garbage. Too much overhead." + ) # check minimum spare blocks # we need some free blocks to use for GC - n_spare_blocks = (1.0 - self.conf.GC_high_threshold_ratio) * \ - self.conf.n_blocks_per_dev + n_spare_blocks = ( + 1.0 - self.conf.GC_high_threshold_ratio + ) * self.conf.n_blocks_per_dev if n_spare_blocks < 32: - raise RuntimeError("We may not have spare blocks to use when "\ - "cleaning (for translation page, etc.). # of spare blocks:" \ - " {}.".format(n_spare_blocks)) + raise RuntimeError( + "We may not have spare blocks to use when " + "cleaning (for translation page, etc.). # of spare blocks:" + f" {n_spare_blocks}." + ) def is_cleaning_needed(self): return self.block_pool.used_ratio() > self.conf.GC_high_threshold_ratio @@ -1610,11 +1715,12 @@ def level_wear(self): req = self._cleaner_res.request() yield req - print 'start wear leveling....' - print self.block_pool.get_erasure_count_dist() + print("start wear leveling....") + print(self.block_pool.get_erasure_count_dist()) - victim_blocks = WearLevelingVictimBlocks(self.conf, - self.block_pool, self.oob, 0.1 * self.conf.n_blocks_per_dev) + victim_blocks = WearLevelingVictimBlocks( + self.conf, self.block_pool, self.oob, 0.1 * self.conf.n_blocks_per_dev + ) all_victim_tuples = list(victim_blocks.iterator_verbose()) batches = utils.group_to_batches(all_victim_tuples, self.n_victim_per_batch) @@ -1622,8 +1728,8 @@ def level_wear(self): for batch in batches: yield self.env.process(self._clean_batch(batch, purpose=PURPOSE_WEAR_LEVEL)) - print 'after wear leveling' - print self.block_pool.get_erasure_count_dist() + print("after wear leveling") + print(self.block_pool.get_erasure_count_dist()) self._cleaner_res.release(req) def clean(self): @@ -1635,14 +1741,17 @@ def clean(self): yield req victim_blocks = VictimBlocks(self.conf, self.block_pool, self.oob) - self.recorder.append_to_value_list('clean_func_valid_ratio_snapshot', - victim_blocks.get_valid_ratio_counter_of_used_blocks()) + self.recorder.append_to_value_list( + "clean_func_valid_ratio_snapshot", + victim_blocks.get_valid_ratio_counter_of_used_blocks(), + ) if self.gc_time_recorded == False: - self.recorder.set_result_by_one_key('gc_trigger_timestamp', - self.env.now / float(SEC)) + self.recorder.set_result_by_one_key( + "gc_trigger_timestamp", self.env.now / float(SEC) + ) self.gc_time_recorded = True - print 'GC time recorded!........!' + print("GC time recorded!........!") all_victim_tuples = list(victim_blocks.iterator_verbose()) batches = utils.group_to_batches(all_victim_tuples, self.n_victim_per_batch) @@ -1657,8 +1766,7 @@ def clean(self): def _clean_batch(self, victim_tuples, purpose): procs = [] for valid_ratio, block_type, block_num in victim_tuples: - p = self.env.process( - self._clean_block(block_type, block_num, purpose)) + p = self.env.process(self._clean_block(block_type, block_num, purpose)) procs.append(p) yield simpy.AllOf(self.env, procs) @@ -1668,20 +1776,19 @@ def _clean_block(self, block_type, block_num, purpose): yield req if block_type == VictimBlocks.TYPE_DATA: - yield self.env.process( - self._datablockcleaner.clean(block_num, purpose)) + yield self.env.process(self._datablockcleaner.clean(block_num, purpose)) elif block_type == VictimBlocks.TYPE_TRANS: - yield self.env.process( - self._transblockcleaner.clean(block_num, purpose)) + yield self.env.process(self._transblockcleaner.clean(block_num, purpose)) self._block_cleaner_res.release(req) -class DataBlockCleaner(object): +class DataBlockCleaner: """ Note that this class does not maintain any private state. It is a set of methods to change other states. """ + def __init__(self, conf, flash, oob, block_pool, mappings, rec, env): self.conf = conf self.flash = flash @@ -1694,7 +1801,7 @@ def __init__(self, conf, flash, oob, block_pool, mappings, rec, env): self.gcid = 0 def log(self, blocknum): - if self.conf['write_gc_log'] is False: + if self.conf["write_gc_log"] is False: return valid_ratio = self.oob.states.block_valid_ratio(blocknum) @@ -1706,20 +1813,22 @@ def log(self, blocknum): try: lpn = self.oob.ppn_to_lpn_or_mvpn(ppn) except KeyError: - lpn = 'NA' - - self.recorder.write_file('gc.log', - gcid=self.gcid, - blocknum=blocknum, - lpn=lpn, - valid=self.oob.states.is_page_valid(ppn)) + lpn = "NA" + + self.recorder.write_file( + "gc.log", + gcid=self.gcid, + blocknum=blocknum, + lpn=lpn, + valid=self.oob.states.is_page_valid(ppn), + ) self.gcid += 1 - def clean(self, blocknum, purpose = PURPOSE_GC): - ''' + def clean(self, blocknum, purpose=PURPOSE_GC): + """ for each valid page, move it to another block invalidate pages in blocknum and erase block - ''' + """ assert blocknum in self.block_pool.used_blocks # assert blocknum not in self.block_pool.current_blocks() @@ -1731,8 +1840,10 @@ def clean(self, blocknum, purpose = PURPOSE_GC): yield self.env.process(self._clean_page(ppn, purpose)) yield self.env.process( - self.flash.erase_pbn_extent(blocknum, 1, - tag=self.recorder.get_tag('erase.data.gc', None))) + self.flash.erase_pbn_extent( + blocknum, 1, tag=self.recorder.get_tag("erase.data.gc", None) + ) + ) self.recorder.count_me("gc", "erase.data.block") self.oob.erase_block(blocknum) @@ -1750,8 +1861,10 @@ def _clean_page(self, ppn, purpose): self.recorder.count_me("wearleveling", "user.page.moves") yield self.env.process( - self.flash.rw_ppn_extent(ppn, 1, 'read', - tag=self.recorder.get_tag('read.data.gc', None))) + self.flash.rw_ppn_extent( + ppn, 1, "read", tag=self.recorder.get_tag("read.data.gc", None) + ) + ) if purpose == PURPOSE_GC: choice = LEAST_ERASED @@ -1760,14 +1873,15 @@ def _clean_page(self, ppn, purpose): new_ppn = self.block_pool.next_gc_data_page_to_program(choice) yield self.env.process( - self.flash.rw_ppn_extent(new_ppn, 1, 'write', - tag=self.recorder.get_tag('write.data.gc', None))) + self.flash.rw_ppn_extent( + new_ppn, 1, "write", tag=self.recorder.get_tag("write.data.gc", None) + ) + ) lpn = self.oob.ppn_to_lpn_or_mvpn(ppn) # mappings in cache - yield self.env.process( - self.mappings.update(lpn=lpn, ppn=new_ppn, tag=None)) + yield self.env.process(self.mappings.update(lpn=lpn, ppn=new_ppn, tag=None)) # mappings on flash # handled by self.mappings @@ -1776,8 +1890,9 @@ def _clean_page(self, ppn, purpose): # handled by self.mappings # oob state - self.oob.relocate_data_page(lpn=lpn, old_ppn=ppn, new_ppn=new_ppn, - update_time=False) + self.oob.relocate_data_page( + lpn=lpn, old_ppn=ppn, new_ppn=new_ppn, update_time=False + ) # oob ppn->lpn/vpn # handled above @@ -1786,14 +1901,25 @@ def _clean_page(self, ppn, purpose): # handled by next_gc_data_page_to_program -class TransBlockCleaner(object): +class TransBlockCleaner: """ Note that this class does not maintain any private state. It is a set of methods to change other states. TODO: some code is duplicated with DataBlockCleaner """ - def __init__(self, conf, flash, oob, block_pool, mappings, directory, rec, - env, trans_page_locks): + + def __init__( + self, + conf, + flash, + oob, + block_pool, + mappings, + directory, + rec, + env, + trans_page_locks, + ): self.conf = conf self.flash = flash self.oob = oob @@ -1804,7 +1930,7 @@ def __init__(self, conf, flash, oob, block_pool, mappings, directory, rec, self.env = env self._trans_page_locks = trans_page_locks - def clean(self, blocknum, purpose = PURPOSE_GC): + def clean(self, blocknum, purpose=PURPOSE_GC): assert blocknum in self.block_pool.used_blocks # assert blocknum not in self.block_pool.current_blocks() @@ -1814,8 +1940,10 @@ def clean(self, blocknum, purpose = PURPOSE_GC): yield self.env.process(self._clean_page(ppn, purpose)) yield self.env.process( - self.flash.erase_pbn_extent(blocknum, 1, - tag=self.recorder.get_tag('erase.trans.gc', None))) + self.flash.erase_pbn_extent( + blocknum, 1, tag=self.recorder.get_tag("erase.trans.gc", None) + ) + ) self.recorder.count_me("gc", "erase.trans.block") self.oob.erase_block(blocknum) @@ -1836,8 +1964,10 @@ def _clean_page(self, ppn, purpose): self._trans_page_locks.locked_addrs.add(m_vpn) yield self.env.process( - self.flash.rw_ppn_extent(ppn, 1, 'read', - tag=self.recorder.get_tag('read.trans.gc', None))) + self.flash.rw_ppn_extent( + ppn, 1, "read", tag=self.recorder.get_tag("read.trans.gc", None) + ) + ) if purpose == PURPOSE_GC: choice = LEAST_ERASED @@ -1846,9 +1976,10 @@ def _clean_page(self, ppn, purpose): new_ppn = self.block_pool.next_gc_translation_page_to_program(choice) yield self.env.process( - self.flash.rw_ppn_extent(new_ppn, 1, 'write', - tag=self.recorder.get_tag('write.trans.gc', None))) - + self.flash.rw_ppn_extent( + new_ppn, 1, "write", tag=self.recorder.get_tag("write.trans.gc", None) + ) + ) # mappings in cache # mapping cache is only for data pages, so we don't need to update @@ -1860,8 +1991,9 @@ def _clean_page(self, ppn, purpose): self.directory.update_mapping(m_vpn=m_vpn, m_ppn=new_ppn) # oob state - self.oob.relocate_trans_page(m_vpn=m_vpn, old_ppn=ppn, new_ppn=new_ppn, - update_time=False) + self.oob.relocate_trans_page( + m_vpn=m_vpn, old_ppn=ppn, new_ppn=new_ppn, update_time=False + ) # oob ppn->lpn/vpn # handled above @@ -1873,7 +2005,7 @@ def _clean_page(self, ppn, purpose): self._trans_page_locks.locked_addrs.remove(m_vpn) -class OutOfBandAreas(object): +class OutOfBandAreas: """ From Wiscsee: It is used to hold page state and logical page number of a page. It is not necessary to implement it as list. But the interface should @@ -1883,6 +2015,7 @@ class OutOfBandAreas(object): and lpn_of_phy_page. Jinghan: I will temporarily keep the data strcutures as it is. Now we only use the p2l mapping """ + def __init__(self, confobj): self.conf = confobj @@ -1950,12 +2083,14 @@ def erase_block(self, flash_block): pass def relocate_data_page(self, lpn, old_ppn, new_ppn, update_time=True): - self._relocate_page(virtual_pn=lpn, old_ppn=old_ppn, new_ppn=new_ppn, - update_time=update_time) + self._relocate_page( + virtual_pn=lpn, old_ppn=old_ppn, new_ppn=new_ppn, update_time=update_time + ) def relocate_trans_page(self, m_vpn, old_ppn, new_ppn, update_time=True): - self._relocate_page(virtual_pn=m_vpn, old_ppn=old_ppn, new_ppn=new_ppn, - update_time=update_time) + self._relocate_page( + virtual_pn=m_vpn, old_ppn=old_ppn, new_ppn=new_ppn, update_time=update_time + ) def _relocate_page(self, virtual_pn, old_ppn, new_ppn, update_time=True): """ @@ -1993,73 +2128,73 @@ def validate_ppn(self, ppn): def data_page_move(self, lpn, old_ppn, new_ppn): # move data page does not change the content's timestamp, so # we copy - self.copy_timestamp(src_ppn = old_ppn, dst_ppn = new_ppn) + self.copy_timestamp(src_ppn=old_ppn, dst_ppn=new_ppn) self.relocate_data_page(lpn, old_ppn, new_ppn, update_time=False) def lpns_of_block(self, flash_block): s, e = self.conf.block_to_page_range(flash_block) lpns = [] for ppn in range(s, e): - lpns.append(self.ppn_to_lpn_mvpn.get(ppn, 'NA')) + lpns.append(self.ppn_to_lpn_mvpn.get(ppn, "NA")) return lpns class Config(config.ConfigNCQFTL): - def __init__(self, confdic = None): - super(Config, self).__init__(confdic) + def __init__(self, confdic=None): + super().__init__(confdic) local_items = { # number of bytes per entry in mapping_on_flash - "translation_page_entry_bytes": 4, # 32 bits - "cache_entry_bytes": 8, # 4 bytes for lpn, 4 bytes for ppn + "translation_page_entry_bytes": 4, # 32 bits + "cache_entry_bytes": 8, # 4 bytes for lpn, 4 bytes for ppn "GC_high_threshold_ratio": 0.95, "GC_low_threshold_ratio": 0.7, - "over_provisioning": 1.28, #TODO: this is not used - "mapping_cache_bytes": None, # cmt: cached mapping table + "over_provisioning": 1.28, # TODO: this is not used + "mapping_cache_bytes": None, # cmt: cached mapping table "do_not_check_gc_setting": False, "write_gc_log": True, - } + } self.update(local_items) - self['segment_bytes'] = 1*TB + self["segment_bytes"] = 1 * TB # self['keeping_all_tp_entries'] = False - self['keeping_all_tp_entries'] = True + self["keeping_all_tp_entries"] = True def get_segment_id(self, lpn): - return (lpn * self.page_size) / self['segment_bytes'] + return old_div((lpn * self.page_size), self["segment_bytes"]) @property def n_pages_per_segment(self): - return self['segment_bytes'] / self.page_size + return old_div(self["segment_bytes"], self.page_size) @property def keeping_all_tp_entries(self): - return self['keeping_all_tp_entries'] + return self["keeping_all_tp_entries"] @keeping_all_tp_entries.setter def keeping_all_tp_entries(self, value): - self['keeping_all_tp_entries'] = value + self["keeping_all_tp_entries"] = value @property def n_mapping_entries_per_page(self): - return self.page_size / self['translation_page_entry_bytes'] + return old_div(self.page_size, self["translation_page_entry_bytes"]) @property def mapping_cache_bytes(self): - return self['mapping_cache_bytes'] + return self["mapping_cache_bytes"] @mapping_cache_bytes.setter def mapping_cache_bytes(self, value): - self['mapping_cache_bytes'] = value + self["mapping_cache_bytes"] = value @property def n_cache_entries(self): - return self.mapping_cache_bytes / self['cache_entry_bytes'] + return old_div(self.mapping_cache_bytes, self["cache_entry_bytes"]) @n_cache_entries.setter def n_cache_entries(self, value): - self.mapping_cache_bytes = value * self['cache_entry_bytes'] + self.mapping_cache_bytes = value * self["cache_entry_bytes"] @property def cache_mapped_data_bytes(self): @@ -2067,57 +2202,59 @@ def cache_mapped_data_bytes(self): @cache_mapped_data_bytes.setter def cache_mapped_data_bytes(self, data_bytes): - self.n_cache_entries = data_bytes / self.page_size + self.n_cache_entries = old_div(data_bytes, self.page_size) if self.n_cache_entries % self.n_mapping_entries_per_page != 0: - print "WARNING: size of mapping cache is not aligned with "\ + print( + "WARNING: size of mapping cache is not aligned with " "translation page size." + ) @property def translation_page_entry_bytes(self): - return self['translation_page_entry_bytes'] + return self["translation_page_entry_bytes"] @property def over_provisioning(self): - return self['over_provisioning'] + return self["over_provisioning"] @over_provisioning.setter def over_provisioning(self, value): - self['over_provisioning'] = value + self["over_provisioning"] = value @property def GC_high_threshold_ratio(self): - return self['GC_high_threshold_ratio'] + return self["GC_high_threshold_ratio"] @GC_high_threshold_ratio.setter def GC_high_threshold_ratio(self, value): - self['GC_high_threshold_ratio'] = value + self["GC_high_threshold_ratio"] = value @property def GC_low_threshold_ratio(self): - return self['GC_low_threshold_ratio'] + return self["GC_low_threshold_ratio"] @GC_low_threshold_ratio.setter def GC_low_threshold_ratio(self, value): - self['GC_low_threshold_ratio'] = value + self["GC_low_threshold_ratio"] = value def sec_ext_to_page_ext(self, sector, count): """ The sector extent has to be aligned with page return page_start, page_count """ - page = sector / self.n_secs_per_page - page_end = (sector + count) / self.n_secs_per_page + page = old_div(sector, self.n_secs_per_page) + page_end = old_div((sector + count), self.n_secs_per_page) page_count = page_end - page if (sector + count) % self.n_secs_per_page != 0: page_count += 1 return page, page_count def lpn_to_m_vpn(self, lpn): - return lpn / self.n_mapping_entries_per_page + return old_div(lpn, self.n_mapping_entries_per_page) def m_vpn_to_lpns(self, m_vpn): start_lpn = m_vpn * self.n_mapping_entries_per_page - return range(start_lpn, start_lpn + self.n_mapping_entries_per_page) + return list(range(start_lpn, start_lpn + self.n_mapping_entries_per_page)) def total_translation_pages(self): """ @@ -2128,17 +2265,17 @@ def total_translation_pages(self): entry_bytes = self.translation_page_entry_bytes flash_page_size = self.page_size # play the ceiling trick - return (n_entries * entry_bytes + \ - (flash_page_size -1)) / flash_page_size + return old_div( + (n_entries * entry_bytes + (flash_page_size - 1)), flash_page_size + ) def write_timeline(conf, recorder, op_id, op, arg, start_time, end_time): - recorder.write_file('timeline.txt', - op_id = op_id, op = op, arg = arg, - start_time = start_time, end_time = end_time) - - - - - - + recorder.write_file( + "timeline.txt", + op_id=op_id, + op=op, + arg=arg, + start_time=start_time, + end_time=end_time, + ) diff --git a/wiscsee/wiscsim/dftlext.py b/wiscsee/wiscsim/dftlext.py index 6ccf2a9..22ee902 100644 --- a/wiscsee/wiscsim/dftlext.py +++ b/wiscsee/wiscsim/dftlext.py @@ -1,19 +1,21 @@ +from past.builtins import cmp +from past.utils import old_div import bitarray from collections import deque, Counter import csv import datetime import random import os -import Queue +import queue import sys import bidict import config -import flash -import ftlbuilder -import lrulist -import recorder +from . import flash +from . import ftlbuilder +from . import lrulist +from . import recorder from utilities import utils from .blkpool import BlockPool from .bitmap import FlashBitmap2 @@ -192,44 +194,45 @@ Timestamp can be easilty obtained from TimestampTble. """ -UNINITIATED, MISS = ('UNINIT', 'MISS') -DATA_BLOCK, TRANS_BLOCK = ('data_block', 'trans_block') +UNINITIATED, MISS = ("UNINIT", "MISS") +DATA_BLOCK, TRANS_BLOCK = ("data_block", "trans_block") random.seed(0) + class Config(config.ConfigNCQFTL): - def __init__(self, confdic = None): - super(Config, self).__init__(confdic) + def __init__(self, confdic=None): + super().__init__(confdic) local_items = { # number of bytes per entry in mapping_on_flash - "translation_page_entry_bytes": 4, # 32 bits - "cache_entry_bytes": 8, # 4 bytes for lpn, 4 bytes for ppn + "translation_page_entry_bytes": 4, # 32 bits + "cache_entry_bytes": 8, # 4 bytes for lpn, 4 bytes for ppn "GC_threshold_ratio": 0.95, "GC_low_threshold_ratio": 0.9, "over_provisioning": 1.28, - "mapping_cache_bytes": None # cmt: cached mapping table - } + "mapping_cache_bytes": None, # cmt: cached mapping table + } self.update(local_items) @property def n_mapping_entries_per_page(self): - return self.page_size / self['translation_page_entry_bytes'] + return old_div(self.page_size, self["translation_page_entry_bytes"]) @property def mapping_cache_bytes(self): - return self['mapping_cache_bytes'] + return self["mapping_cache_bytes"] @mapping_cache_bytes.setter def mapping_cache_bytes(self, value): - self['mapping_cache_bytes'] = value + self["mapping_cache_bytes"] = value @property def n_cache_entries(self): - return self.mapping_cache_bytes / self['cache_entry_bytes'] + return old_div(self.mapping_cache_bytes, self["cache_entry_bytes"]) @n_cache_entries.setter def n_cache_entries(self, value): - self.mapping_cache_bytes = value * self['cache_entry_bytes'] + self.mapping_cache_bytes = value * self["cache_entry_bytes"] @property def cache_mapped_data_bytes(self): @@ -237,40 +240,48 @@ def cache_mapped_data_bytes(self): @cache_mapped_data_bytes.setter def cache_mapped_data_bytes(self, data_bytes): - self.n_cache_entries = data_bytes / self.page_size + self.n_cache_entries = old_div(data_bytes, self.page_size) if self.n_cache_entries % self.n_mapping_entries_per_page != 0: - print "WARNING: size of mapping cache is not aligned with "\ + print( + "WARNING: size of mapping cache is not aligned with " "translation page size." + ) + @property def translation_page_entry_bytes(self): - return self['translation_page_entry_bytes'] + return self["translation_page_entry_bytes"] @property def over_provisioning(self): - return self['over_provisioning'] + return self["over_provisioning"] @property def GC_threshold_ratio(self): - return self['GC_threshold_ratio'] + return self["GC_threshold_ratio"] @property def GC_low_threshold_ratio(self): - return self['GC_low_threshold_ratio'] + return self["GC_low_threshold_ratio"] -class GlobalHelper(object): +class GlobalHelper: """ In case you need some global variables. We put all global stuff here so it is easier to manage. (And you know all the bad things you did :) """ + def __init__(self, confobj): self.timeline = Timeline(confobj) -LOGICAL_READ, LOGICAL_WRITE, LOGICAL_DISCARD = ('LOGICAL_READ', \ - 'LOGICAL_WRITE', 'LOGICAL_DISCARD') +LOGICAL_READ, LOGICAL_WRITE, LOGICAL_DISCARD = ( + "LOGICAL_READ", + "LOGICAL_WRITE", + "LOGICAL_DISCARD", +) -class Timeline(object): + +class Timeline: """ This is intended for global use. It maintains a table like: @@ -287,6 +298,7 @@ class Timeline(object): 3. go to 1. """ + def __init__(self, confobj): self.table = [] self.conf = confobj @@ -294,9 +306,9 @@ def __init__(self, confobj): self.timestamp = 0 self.name_map = { - 'flash.read': 'page_read_time', # milli sec - 'flash.write': 'page_prog_time', # milli sec - 'flash.erasure': 'block_erase_time' # milli sec + "flash.read": "page_read_time", # milli sec + "flash.write": "page_prog_time", # milli sec + "flash.erasure": "block_erase_time", # milli sec } def turn_on(self): @@ -309,10 +321,15 @@ def add_logical_op(self, sector, count, op): if not self.ON: return - self.table.append( {'sector':sector, 'count':count, 'operation': op, - 'start_timestamp': self.timestamp, - 'end_timestamp': self.timestamp} - ) + self.table.append( + { + "sector": sector, + "count": count, + "operation": op, + "start_timestamp": self.timestamp, + "end_timestamp": self.timestamp, + } + ) def incr_time_stamp(self, op, count): """ @@ -322,43 +339,47 @@ def incr_time_stamp(self, op, count): return opname = self.name_map[op] - time = self.conf['flash_config'][opname] * count + time = self.conf["flash_config"][opname] * count self.timestamp += time last_row = self.table[-1] - last_row['end_timestamp'] = self.timestamp + last_row["end_timestamp"] = self.timestamp def save(self): - path = os.path.join(self.conf['result_dir'], 'timeline.txt') + path = os.path.join(self.conf["result_dir"], "timeline.txt") utils.table_to_file(self.table, path) def block_to_channel_block(conf, blocknum): n_blocks_per_channel = conf.n_blocks_per_channel - channel = blocknum / n_blocks_per_channel + channel = old_div(blocknum, n_blocks_per_channel) block_off = blocknum % n_blocks_per_channel return channel, block_off + def channel_block_to_block(conf, channel, block_off): n_blocks_per_channel = conf.n_blocks_per_channel return channel * n_blocks_per_channel + block_off + def page_to_channel_page(conf, pagenum): """ pagenum is in the context of device """ n_pages_per_channel = conf.n_pages_per_channel - channel = pagenum / n_pages_per_channel + channel = old_div(pagenum, n_pages_per_channel) page_off = pagenum % n_pages_per_channel return channel, page_off + def channel_page_to_page(conf, channel, page_off): """ Translate channel, page_off to pagenum in context of device """ return channel * conf.n_pages_per_channel + page_off -class OutOfBandAreas(object): + +class OutOfBandAreas: """ It is used to hold page state and logical page number of a page. It is not necessary to implement it as list. But the interface should @@ -367,6 +388,7 @@ class OutOfBandAreas(object): events, and react accordingly to this event. The action may involve state and lpn_of_phy_page. """ + def __init__(self, confobj): self.conf = confobj @@ -421,11 +443,11 @@ def wipe_ppn(self, ppn): # It is OK to delay it until we erase the block # try: - # del self.ppn_to_lpn_mvpn[ppn] + # del self.ppn_to_lpn_mvpn[ppn] # except KeyError: - # # it is OK that the key does not exist, for example, - # # when discarding without writing to it - # pass + # # it is OK that the key does not exist, for example, + # # when discarding without writing to it + # pass def erase_block(self, flash_block): self.states.erase_block(flash_block) @@ -465,32 +487,33 @@ def new_lba_write(self, lpn, old_ppn, new_ppn): def data_page_move(self, lpn, old_ppn, new_ppn): # move data page does not change the content's timestamp, so # we copy - self.timestamp_copy(src_ppn = old_ppn, dst_ppn = new_ppn) + self.timestamp_copy(src_ppn=old_ppn, dst_ppn=new_ppn) self.new_write(lpn, old_ppn, new_ppn) def lpns_of_block(self, flash_block): s, e = self.conf.block_to_page_range(flash_block) lpns = [] for ppn in range(s, e): - lpns.append(self.ppn_to_lpn_mvpn.get(ppn, 'NA')) + lpns.append(self.ppn_to_lpn_mvpn.get(ppn, "NA")) return lpns -class CacheEntryData(object): + +class CacheEntryData: """ This is a helper class that store entry data for a LPN """ + def __init__(self, lpn, ppn, dirty): self.lpn = lpn self.ppn = ppn self.dirty = dirty def __repr__(self): - return "lpn:{}, ppn:{}, dirty:{}".format(self.lpn, - self.ppn, self.dirty) + return f"lpn:{self.lpn}, ppn:{self.ppn}, dirty:{self.dirty}" -class CachedMappingTable(object): +class CachedMappingTable: """ When do we need batched update? - do we need it when cleaning translation pages? NO. cleaning translation @@ -507,15 +530,23 @@ class CachedMappingTable(object): entry in CMT. In that case, we need to find all the CMT entries in the same translation page with the victim entry. """ + def __init__(self, confobj): self.conf = confobj - self.entry_bytes = 8 # lpn + ppn + self.entry_bytes = 8 # lpn + ppn max_bytes = self.conf.mapping_cache_bytes - self.max_n_entries = (max_bytes + self.entry_bytes - 1) / \ - self.entry_bytes - print 'cache max entries', self.max_n_entries, \ - self.max_n_entries * 4096 / 2**20, 'MB' + self.max_n_entries = old_div( + (max_bytes + self.entry_bytes - 1), self.entry_bytes + ) + print( + ( + "cache max entries", + self.max_n_entries, + old_div(self.max_n_entries * 4096, 2**20), + "MB", + ) + ) # self.entries = {} # self.entries = lrulist.LruCache() @@ -531,14 +562,15 @@ def lpn_to_ppn(self, lpn): def add_new_entry(self, lpn, ppn, dirty): "dirty is a boolean" - if self.entries.has_key(lpn): - raise RuntimeError("{}:{} already exists in CMT entries.".format( - lpn, self.entries[lpn].ppn)) - self.entries[lpn] = CacheEntryData(lpn = lpn, ppn = ppn, dirty = dirty) + if lpn in self.entries: + raise RuntimeError( + f"{lpn}:{self.entries[lpn].ppn} already exists in CMT entries." + ) + self.entries[lpn] = CacheEntryData(lpn=lpn, ppn=ppn, dirty=dirty) def update_entry(self, lpn, ppn, dirty): "You may end up remove the old one" - self.entries[lpn] = CacheEntryData(lpn = lpn, ppn = ppn, dirty = dirty) + self.entries[lpn] = CacheEntryData(lpn=lpn, ppn=ppn, dirty=dirty) def overwrite_entry(self, lpn, ppn, dirty): "lpn must exist" @@ -551,7 +583,7 @@ def remove_entry_by_lpn(self, lpn): def victim_entry(self): # lpn = random.choice(self.entries.keys()) classname = type(self.entries).__name__ - if classname in ('SegmentedLruCache', 'LruCache'): + if classname in ("SegmentedLruCache", "LruCache"): lpn = self.entries.victim_key() else: raise RuntimeError("You need to specify victim selection") @@ -568,18 +600,20 @@ def __repr__(self): return repr(self.entries) -class MappingOnFlash(object): +class MappingOnFlash: """ This mapping table is for data pages, not for translation pages. GMT should have entries as many as the number of pages in flash """ + def __init__(self, confobj, flashobj): """ flashobj is the flash device that we may operate on. """ if not isinstance(confobj, config.Config): - raise TypeError("confobj is not conf.Config. it is {}". - format(type(confobj).__name__)) + raise TypeError( + f"confobj is not conf.Config. it is {type(confobj).__name__}" + ) self.conf = confobj @@ -605,7 +639,7 @@ def total_translation_pages(self): entry_bytes = self.conf.translation_page_entry_bytes flash_page_size = self.conf.page_size # play the ceiling trick - return (entries * entry_bytes + (flash_page_size -1))/flash_page_size + return old_div((entries * entry_bytes + (flash_page_size - 1)), flash_page_size) def lpn_to_ppn(self, lpn): """ @@ -619,14 +653,15 @@ def update(self, lpn, ppn): self.entries[lpn] = ppn def __repr__(self): - return "global mapping table: {}".format(repr(self.entries)) + return f"global mapping table: {repr(self.entries)}" -class GlobalTranslationDirectory(object): +class GlobalTranslationDirectory: """ This is an in-memory data structure. It is only for book keeping. It used to remeber thing so that we don't lose it. """ + def __init__(self, confobj): self.conf = confobj @@ -649,9 +684,8 @@ def m_vpn_to_m_ppn(self, m_vpn): return self.mapping[m_vpn] def add_mapping(self, m_vpn, m_ppn): - if self.mapping.has_key(m_vpn): - raise RuntimeError("self.mapping already has m_vpn:{}"\ - .format(m_vpn)) + if m_vpn in self.mapping: + raise RuntimeError(f"self.mapping already has m_vpn:{m_vpn}") self.mapping[m_vpn] = m_ppn def update_mapping(self, m_vpn, m_ppn): @@ -662,11 +696,11 @@ def remove_mapping(self, m_vpn): def m_vpn_of_lpn(self, lpn): "Find the virtual translation page that holds lpn" - return lpn / self.n_entries_per_page + return old_div(lpn, self.n_entries_per_page) def m_vpn_to_lpns(self, m_vpn): start_lpn = m_vpn * self.n_entries_per_page - return range(start_lpn, start_lpn + self.n_entries_per_page) + return list(range(start_lpn, start_lpn + self.n_entries_per_page)) def m_ppn_of_lpn(self, lpn): m_vpn = self.m_vpn_of_lpn(lpn) @@ -677,13 +711,14 @@ def __repr__(self): return repr(self.mapping) -class MappingManager(object): +class MappingManager: """ This class is the supervisor of all the mappings. When initializing, it register CMT and GMT with it and provides higher level operations on top of them. This class should act as a coordinator of all the mapping data structures. """ + def __init__(self, confobj, block_pool, flashobj, oobobj, recorderobj): self.conf = confobj @@ -710,26 +745,22 @@ def ppns_for_writing(self, lpns): ppns.append(new_ppn) # CMT # lpn must be in cache thanks to self.mapping_manager.lpn_to_ppn() - self.cached_mapping_table.overwrite_entry( - lpn = lpn, ppn = new_ppn, dirty = True) + self.cached_mapping_table.overwrite_entry(lpn=lpn, ppn=new_ppn, dirty=True) # OOB - self.oob.new_lba_write(lpn = lpn, old_ppn = old_ppn, - new_ppn = new_ppn) + self.oob.new_lba_write(lpn=lpn, old_ppn=old_ppn, new_ppn=new_ppn) return ppns def ppns_for_reading(self, lpns): - """ - """ + """ """ ppns = [] for lpn in lpns: ppn = self.lpn_to_ppn(lpn) # request lpn must match oob[ppn].lpn - if ppn != 'UNINIT': + if ppn != "UNINIT": # assert_flash_data_startswith_oob_lpn(self.conf, self.flash, - # self.oob, ppn) - check_data(self, self.conf, self.flash, self.oob, ppn, - lpn) + # self.oob, ppn) + check_data(self, self.conf, self.flash, self.oob, ppn, lpn) ppns.append(ppn) return ppns @@ -766,13 +797,12 @@ def load_mapping_entry_to_cache(self, lpn): m_ppn = self.directory.m_ppn_of_lpn(lpn) # read it up, this operation is just for statistics - self.flash.read_pages(ppns = [m_ppn], tag = TRANS_CACHE) + self.flash.read_pages(ppns=[m_ppn], tag=TRANS_CACHE) # Now we have all the entries of m_ppn in memory, we need to put # the mapping of lpn->ppn to CMT ppn = self.mapping_on_flash.lpn_to_ppn(lpn) - self.cached_mapping_table.add_new_entry(lpn = lpn, ppn = ppn, - dirty = False) + self.cached_mapping_table.add_new_entry(lpn=lpn, ppn=ppn, dirty=False) return ppn @@ -793,10 +823,9 @@ def initialize_mappings(self): # Note that we don't actually read or write flash self.directory.add_mapping(m_vpn=m_vpn, m_ppn=m_ppn) # update oob of the translation page - self.oob.new_write(lpn = m_vpn, old_ppn = UNINITIATED, - new_ppn = m_ppn) + self.oob.new_write(lpn=m_vpn, old_ppn=UNINITIATED, new_ppn=m_ppn) - def update_entry(self, lpn, new_ppn, tag = "NA"): + def update_entry(self, lpn, new_ppn, tag="NA"): """ Update mapping of lpn to be lpn->new_ppn everywhere if necessary. @@ -818,15 +847,14 @@ def update_entry(self, lpn, new_ppn, tag = "NA"): cached_ppn = self.cached_mapping_table.lpn_to_ppn(lpn) if cached_ppn != MISS: # in cache - self.cached_mapping_table.overwrite_entry(lpn = lpn, - ppn = new_ppn, dirty = False) + self.cached_mapping_table.overwrite_entry(lpn=lpn, ppn=new_ppn, dirty=False) m_vpn = self.directory.m_vpn_of_lpn(lpn) # batch_entries may be empty batch_entries = self.dirty_entries_of_translation_page(m_vpn) - new_mappings = {lpn:new_ppn} # lpn->new_ppn may not be in cache + new_mappings = {lpn: new_ppn} # lpn->new_ppn may not be in cache for entry in batch_entries: new_mappings[entry.lpn] = entry.ppn @@ -843,7 +871,7 @@ def evict_cache_entry(self): If the entry is dirty, write it back to GMT. If it is not dirty, simply remove it. """ - self.recorder.count_me('cache', 'evict') + self.recorder.count_me("cache", "evict") vic_lpn, vic_entrydata = self.cached_mapping_table.victim_entry() @@ -859,7 +887,7 @@ def batch_write_back(self, m_vpn): """ Write dirty entries in a translation page with a flash read and a flash write. """ - self.recorder.count_me('cache', 'batch_write_back') + self.recorder.count_me("cache", "batch_write_back") batch_entries = self.dirty_entries_of_translation_page(m_vpn) @@ -868,7 +896,7 @@ def batch_write_back(self, m_vpn): new_mappings[entry.lpn] = entry.ppn # update translation page - self.recorder.count_me('batch.size', len(new_mappings)) + self.recorder.count_me("batch.size", len(new_mappings)) self.update_translation_page_on_flash(m_vpn, new_mappings, TRANS_CACHE) # mark them as clean @@ -880,7 +908,7 @@ def dirty_entries_of_translation_page(self, m_vpn): Get all dirty entries in translation page m_vpn. """ retlist = [] - for entry_lpn, dataentry in self.cached_mapping_table.entries.items(): + for entry_lpn, dataentry in list(self.cached_mapping_table.entries.items()): if dataentry.dirty == True: tmp_m_vpn = self.directory.m_vpn_of_lpn(entry_lpn) if tmp_m_vpn == m_vpn: @@ -905,28 +933,27 @@ def update_translation_page_on_flash(self, m_vpn, new_mappings, tag): # update GMT on flash if len(new_mappings) < self.conf.n_mapping_entries_per_page: # need to read some mappings - self.flash.read_pages(ppns = [old_m_ppn], tag = tag) + self.flash.read_pages(ppns=[old_m_ppn], tag=tag) else: - self.recorder.count_me('cache', 'saved.1.read') + self.recorder.count_me("cache", "saved.1.read") - pass # modify in memory. Since we are a simulator, we don't do anything + pass # modify in memory. Since we are a simulator, we don't do anything new_m_ppn = self.block_pool.next_translation_page_to_program() # update flash - self.flash.write_pages(ppns = [new_m_ppn], ppn_data = None, tag = tag) + self.flash.write_pages(ppns=[new_m_ppn], ppn_data=None, tag=tag) # update our fake 'on-flash' GMT - for lpn, new_ppn in new_mappings.items(): - self.mapping_on_flash.update(lpn = lpn, ppn = new_ppn) + for lpn, new_ppn in list(new_mappings.items()): + self.mapping_on_flash.update(lpn=lpn, ppn=new_ppn) # OOB, keep m_vpn as lpn - self.oob.new_write(lpn = m_vpn, old_ppn = old_m_ppn, - new_ppn = new_m_ppn) + self.oob.new_write(lpn=m_vpn, old_ppn=old_m_ppn, new_ppn=new_m_ppn) # update GTD so we can find it - self.directory.update_mapping(m_vpn = m_vpn, m_ppn = new_m_ppn) + self.directory.update_mapping(m_vpn=m_vpn, m_ppn=new_m_ppn) -class GcDecider(object): +class GcDecider: """ It is used to decide wheter we should do garbage collection. @@ -935,6 +962,7 @@ class GcDecider(object): Later, use low water mark and progress to decide. If we haven't make progress in 10 times, stop GC """ + def __init__(self, confobj, block_pool, recorderobj): self.conf = confobj self.block_pool = block_pool @@ -947,29 +975,29 @@ def __init__(self, confobj, block_pool, recorderobj): min_high = 1 / float(self.conf.over_provisioning) if self.conf.GC_threshold_ratio < min_high: hi_watermark_ratio = min_high - print 'High watermark is reset to {}. It was {}'.format( - hi_watermark_ratio, self.conf.GC_threshold_ratio) + print( + f"High watermark is reset to {hi_watermark_ratio}. It was {self.conf.GC_threshold_ratio}" + ) else: hi_watermark_ratio = self.conf.GC_threshold_ratio - print 'Using user defined high watermark', hi_watermark_ratio + print(("Using user defined high watermark", hi_watermark_ratio)) - self.high_watermark = hi_watermark_ratio * \ - self.conf.n_blocks_per_dev + self.high_watermark = hi_watermark_ratio * self.conf.n_blocks_per_dev - min_low = 0.8 * 1 / self.conf.over_provisioning + min_low = old_div(0.8 * 1, self.conf.over_provisioning) if self.conf.GC_low_threshold_ratio < min_low: low_watermark_ratio = min_low - print 'Low watermark is reset to {}. It was {}'.format( - low_watermark_ratio, self.conf.GC_low_threshold_ratio) + print( + f"Low watermark is reset to {low_watermark_ratio}. It was {self.conf.GC_low_threshold_ratio}" + ) else: low_watermark_ratio = self.conf.GC_low_threshold_ratio - print 'Using user defined low watermark', low_watermark_ratio + print(("Using user defined low watermark", low_watermark_ratio)) - self.low_watermark = low_watermark_ratio * \ - self.conf.n_blocks_per_dev + self.low_watermark = low_watermark_ratio * self.conf.n_blocks_per_dev - print 'High watermark', self.high_watermark - print 'Low watermark', self.low_watermark + print(("High watermark", self.high_watermark)) + print(("Low watermark", self.low_watermark)) self.call_index = -1 self.last_used_blocks = None @@ -998,13 +1026,13 @@ def need_cleaning(self): else: if self.freezed_too_long(n_used_blocks): ret = False - print 'freezed too long, stop GC' - self.recorder.count_me("GC", 'freezed_too_long') + print("freezed too long, stop GC") + self.recorder.count_me("GC", "freezed_too_long") else: # Is it higher than low watermark? ret = n_used_blocks > self.low_watermark if ret == False: - self.recorder.count_me("GC", 'below_lowerwatermark') + self.recorder.count_me("GC", "below_lowerwatermark") # We were able to bring used block to below lower # watermark. It means we still have a lot free space # We don't need to worry about frequent GC. @@ -1025,8 +1053,9 @@ def raise_high_watermark(self): """ return - self.high_watermark = min(self.high_watermark * 1.01, - self.conf.n_blocks_per_dev * 0.95) + self.high_watermark = min( + self.high_watermark * 1.01, self.conf.n_blocks_per_dev * 0.95 + ) def lower_high_watermark(self): """ @@ -1034,8 +1063,7 @@ def lower_high_watermark(self): """ return - self.high_watermark = max(self.high_watermark_orig, - self.high_watermark / 1.01) + self.high_watermark = max(self.high_watermark_orig, self.high_watermark / 1.01) def improved(self, cur_n_used_blocks): """ @@ -1065,10 +1093,11 @@ def freezed_too_long(self, cur_n_used_blocks): return ret -class BlockInfo(object): +class BlockInfo: """ This is for sorting blocks to clean the victim. """ + def __init__(self, block_type, block_num, value): self.block_type = block_type self.block_num = block_num @@ -1080,9 +1109,10 @@ def __comp__(self, other): # return cmp(self.value, other.value) -class GarbageCollector(object): - def __init__(self, confobj, flashobj, oobobj, block_pool, mapping_manager, - recorderobj): +class GarbageCollector: + def __init__( + self, confobj, flashobj, oobobj, block_pool, mapping_manager, recorderobj + ): self.conf = confobj self.flash = flashobj self.oob = oobobj @@ -1103,21 +1133,26 @@ def try_gc(self): if self.decider.call_index == 0: triggered = True self.recorder.count_me("GC", "invoked") - print 'GC is triggerred', self.block_pool.used_ratio(), \ - 'freeblocks:', len(self.block_pool.freeblocks) + print( + ( + "GC is triggerred", + self.block_pool.used_ratio(), + "freeblocks:", + len(self.block_pool.freeblocks), + ) + ) block_iter = self.victim_blocks_iter() blk_cnt = 0 try: - blockinfo = block_iter.next() + blockinfo = next(block_iter) except StopIteration: - print 'GC stoped from StopIteration exception' + print("GC stoped from StopIteration exception") self.recorder.count_me("GC", "StopIteration") # high utilization, raise watermarkt to reduce GC attempts self.decider.raise_high_watermark() # nothing to be cleaned break - victim_type, victim_block = (blockinfo.block_type, - blockinfo.block_num) + victim_type, victim_block = (blockinfo.block_type, blockinfo.block_num) if victim_type == DATA_BLOCK: self.clean_data_block(victim_block) elif victim_type == TRANS_BLOCK: @@ -1125,9 +1160,16 @@ def try_gc(self): blk_cnt += 1 if triggered: - print 'GC is finished', self.block_pool.used_ratio(), \ - blk_cnt, 'collected', \ - 'freeblocks:', len(self.block_pool.freeblocks) + print( + ( + "GC is finished", + self.block_pool.used_ratio(), + blk_cnt, + "collected", + "freeblocks:", + len(self.block_pool.freeblocks), + ) + ) # raise RuntimeError("intentional exit") def clean_data_block(self, flash_block): @@ -1148,8 +1190,7 @@ def clean_data_block(self, flash_block): self.erase_block(flash_block, DATA_CLEANING) def clean_trans_block(self, flash_block): - self.move_valid_pages(flash_block, - self.move_trans_page_to_new_location) + self.move_valid_pages(flash_block, self.move_trans_page_to_new_location) # mark block as free self.block_pool.move_used_trans_block_to_free(flash_block) # it handles oob and flash @@ -1184,25 +1225,23 @@ def move_data_page_to_new_location(self, ppn): old_ppn = ppn # read the the data page - pagedata = self.flash.read_pages(ppns = [old_ppn], - tag = DATA_CLEANING)[0] + pagedata = self.flash.read_pages(ppns=[old_ppn], tag=DATA_CLEANING)[0] # find the mapping lpn = self.oob.translate_ppn_to_lpn(old_ppn) # write to new page new_ppn = self.block_pool.next_gc_data_page_to_program() - self.flash.write_pages(ppns = [new_ppn], ppn_data = [pagedata], - tag = DATA_CLEANING) + self.flash.write_pages(ppns=[new_ppn], ppn_data=[pagedata], tag=DATA_CLEANING) # update new page and old page's OOB self.oob.data_page_move(lpn, old_ppn, new_ppn) if lpn == 6127: - print 'we will move 6127' + print("we will move 6127") exit(1) - return {'lpn':lpn, 'old_ppn':old_ppn, 'new_ppn':new_ppn} + return {"lpn": lpn, "old_ppn": old_ppn, "new_ppn": new_ppn} def group_changes(self, changes): """ @@ -1219,7 +1258,7 @@ def group_changes(self, changes): # Put the mapping changes into groups, each group belongs to one mvpn groups = {} for change in changes: - m_vpn = self.mapping_manager.directory.m_vpn_of_lpn(change['lpn']) + m_vpn = self.mapping_manager.directory.m_vpn_of_lpn(change["lpn"]) group = groups.setdefault(m_vpn, []) group.append(change) @@ -1227,22 +1266,22 @@ def group_changes(self, changes): def update_flash_mappings(self, m_vpn, changes_list): # update translation page on flash - new_mappings = {change['lpn']:change['new_ppn'] - for change in changes_list} + new_mappings = {change["lpn"]: change["new_ppn"] for change in changes_list} self.mapping_manager.update_translation_page_on_flash( - m_vpn, new_mappings, TRANS_UPDATE_FOR_DATA_GC) + m_vpn, new_mappings, TRANS_UPDATE_FOR_DATA_GC + ) def update_cache_mappings(self, changes_in_cache): # some mappings are in flash and some in cache # we can set mappings in cache as dirty=False since # they are consistent with flash for change in changes_in_cache: - lpn = change['lpn'] - old_ppn = change['old_ppn'] - new_ppn = change['new_ppn'] - self.mapping_manager.cached_mapping_table\ - .overwrite_entry( - lpn = lpn, ppn = new_ppn, dirty = False) + lpn = change["lpn"] + old_ppn = change["old_ppn"] + new_ppn = change["new_ppn"] + self.mapping_manager.cached_mapping_table.overwrite_entry( + lpn=lpn, ppn=new_ppn, dirty=False + ) def apply_mvpn_changes(self, m_vpn, changes_list): """ @@ -1263,17 +1302,17 @@ def apply_mvpn_changes(self, m_vpn, changes_list): some_in_cache = False some_in_flash = False for change in changes_list: - lpn = change['lpn'] - old_ppn = change['old_ppn'] - new_ppn = change['new_ppn'] + lpn = change["lpn"] + old_ppn = change["old_ppn"] + new_ppn = change["new_ppn"] - cached_ppn = self.mapping_manager\ - .cached_mapping_table.lpn_to_ppn(lpn) + cached_ppn = self.mapping_manager.cached_mapping_table.lpn_to_ppn(lpn) if cached_ppn != MISS: # lpn is in cache some_in_cache = True self.mapping_manager.cached_mapping_table.overwrite_entry( - lpn = lpn, ppn = new_ppn, dirty = True) + lpn=lpn, ppn=new_ppn, dirty=True + ) changes_in_cache.append(change) else: # lpn is not in cache, mark it and update later in batch @@ -1305,8 +1344,9 @@ def update_mapping_in_batch(self, changes): # Put the mapping changes into groups, each group belongs to one mvpn groups = self.group_changes(changes) - for m_vpn, changes_list in groups.items(): + for m_vpn, changes_list in list(groups.items()): self.apply_mvpn_changes(m_vpn, changes_list) + def move_trans_page_to_new_location(self, m_ppn): """ 1. read the trans page @@ -1318,19 +1358,17 @@ def move_trans_page_to_new_location(self, m_ppn): m_vpn = self.oob.translate_ppn_to_lpn(old_m_ppn) - self.flash.read_pages(ppns = [old_m_ppn], tag = TRANS_CLEAN) + self.flash.read_pages(ppns=[old_m_ppn], tag=TRANS_CLEAN) # write to new page new_m_ppn = self.block_pool.next_gc_translation_page_to_program() - self.flash.write_pages(ppns = [new_m_ppn], ppn_data = None, - tag = TRANS_CLEAN) + self.flash.write_pages(ppns=[new_m_ppn], ppn_data=None, tag=TRANS_CLEAN) # update new page and old page's OOB self.oob.new_write(m_vpn, old_m_ppn, new_m_ppn) # update GTD - self.mapping_manager.directory.update_mapping(m_vpn = m_vpn, - m_ppn = new_m_ppn) + self.mapping_manager.directory.update_mapping(m_vpn=m_vpn, m_ppn=new_m_ppn) def benefit_cost(self, blocknum, current_time): """ @@ -1350,15 +1388,15 @@ def benefit_cost(self, blocknum, current_time): last_inv_time = self.oob.last_inv_time_of_block.get(blocknum, None) if last_inv_time == None: - print blocknum + print(blocknum) raise RuntimeError( - "blocknum {} has never been invalidated."\ - "valid ratio:{}." - .format(blocknum, valid_ratio)) + f"blocknum {blocknum} has never been invalidated." + f"valid ratio:{valid_ratio}." + ) age = current_time - self.oob.last_inv_time_of_block[blocknum] age = age.total_seconds() - bene_cost = age * ( 1 - valid_ratio ) / ( 2 * valid_ratio ) + bene_cost = old_div(age * (1 - valid_ratio), (2 * valid_ratio)) return bene_cost, valid_ratio @@ -1368,17 +1406,17 @@ def victim_blocks_iter(self): """ current_blocks = self.block_pool.current_blocks() current_time = datetime.datetime.now() - priority_q = Queue.PriorityQueue() + priority_q = queue.PriorityQueue() for usedblocks, block_type in ( (self.block_pool.data_usedblocks, DATA_BLOCK), - (self.block_pool.trans_usedblocks, TRANS_BLOCK)): + (self.block_pool.trans_usedblocks, TRANS_BLOCK), + ): for blocknum in usedblocks: if blocknum in current_blocks: continue - bene_cost, valid_ratio = self.benefit_cost(blocknum, - current_time) + bene_cost, valid_ratio = self.benefit_cost(blocknum, current_time) if bene_cost == 0: # valid_ratio must be zero, we definitely don't @@ -1386,37 +1424,38 @@ def victim_blocks_iter(self): # free pages from it continue - blk_info = BlockInfo(block_type = block_type, - block_num = blocknum, value = bene_cost) + blk_info = BlockInfo( + block_type=block_type, block_num=blocknum, value=bene_cost + ) blk_info.valid_ratio = valid_ratio if blk_info.valid_ratio > 0: lpns = self.oob.lpns_of_block(blocknum) s, e = self.conf.block_to_page_range(blocknum) - ppns = range(s, e) + ppns = list(range(s, e)) - ppn_states = [self.oob.states.page_state_human(ppn) - for ppn in ppns] - blk_info.mappings = zip(ppns, lpns, ppn_states) + ppn_states = [self.oob.states.page_state_human(ppn) for ppn in ppns] + blk_info.mappings = list(zip(ppns, lpns, ppn_states)) priority_q.put(blk_info) while not priority_q.empty(): - b_info = priority_q.get() + b_info = priority_q.get() # record the information of victim block - self.recorder.count_me('block.info.valid_ratio', - round(b_info.valid_ratio, 2)) - self.recorder.count_me('block.info.bene_cost', - round(b_info.value)) - - if self.conf['record_bad_victim_block'] == True and \ - b_info.valid_ratio > 0: - self.recorder.write_file('bad_victim_blocks', - block_type = b_info.block_type, - block_num = b_info.block_num, - bene_cost = b_info.value, - valid_ratio = round(b_info.valid_ratio, 2)) + self.recorder.count_me( + "block.info.valid_ratio", round(b_info.valid_ratio, 2) + ) + self.recorder.count_me("block.info.bene_cost", round(b_info.value)) + + if self.conf["record_bad_victim_block"] == True and b_info.valid_ratio > 0: + self.recorder.write_file( + "bad_victim_blocks", + block_type=b_info.block_type, + block_num=b_info.block_num, + bene_cost=b_info.value, + valid_ratio=round(b_info.valid_ratio, 2), + ) # lpn ppn ppn_states blocknum for ppn, lpn, ppn_state in b_info.mappings: @@ -1425,16 +1464,17 @@ def victim_blocks_iter(self): else: lpn_timestamp = -1 - self.recorder.write_file('bad.block.mappings', - ppn = ppn, - lpn = lpn, - ppn_state = ppn_state, - block_num = b_info.block_num, - valid_ratio = b_info.valid_ratio, - block_type = b_info.block_type, - victim_block_seqid = self.victim_block_seqid, - lpn_timestamp = lpn_timestamp - ) + self.recorder.write_file( + "bad.block.mappings", + ppn=ppn, + lpn=lpn, + ppn_state=ppn_state, + block_num=b_info.block_num, + valid_ratio=b_info.valid_ratio, + block_type=b_info.block_type, + victim_block_seqid=self.victim_block_seqid, + lpn_timestamp=lpn_timestamp, + ) self.victim_block_seqid += 1 @@ -1449,16 +1489,19 @@ def erase_block(self, blocknum, tag): # set page states to ERASED and in-OOB lpn to nothing self.oob.erase_block(blocknum) - self.flash.erase_blocks(pbns = [blocknum], tag = tag) + self.flash.erase_blocks(pbns=[blocknum], tag=tag) + def dec_debug(function): def wrapper(self, lpn): ret = function(self, lpn) if lpn == 38356: - print function.__name__, 'lpn:', lpn, 'ret:', ret + print((function.__name__, "lpn:", lpn, "ret:", ret)) return ret + return wrapper + # # - translation pages # - cache miss read (trans.cache.load) @@ -1483,7 +1526,7 @@ def wrapper(self, lpn): # trans clean include: # erasing translation block # move translation page during gc (including read and write) -TRANS_CLEAN = "trans.clean" #read/write are for moving pages +TRANS_CLEAN = "trans.clean" # read/write are for moving pages # clean_data_block() # update_mapping_in_batch() @@ -1496,17 +1539,18 @@ def wrapper(self, lpn): # move data page during gc (including read and write) DATA_CLEANING = "data.cleaning" + class Dftl(ftlbuilder.FtlBuilder): """ The implementation literally follows DFtl paper. This class is a coordinator of other coordinators and data structures """ + def __init__(self, confobj, recorderobj, flashobj): - super(Dftl, self).__init__(confobj, recorderobj, flashobj) + super().__init__(confobj, recorderobj, flashobj) if not isinstance(confobj, Config): - raise TypeError("confobj is not Config. it is {}". - format(type(confobj).__name__)) + raise TypeError(f"confobj is not Config. it is {type(confobj).__name__}") # bitmap has been created parent class # Change: we now don't put the bitmap here @@ -1523,29 +1567,28 @@ def __init__(self, confobj, recorderobj, flashobj): ###### the managers ###### self.mapping_manager = MappingManager( - confobj = self.conf, - block_pool = self.block_pool, - flashobj = self.flash, + confobj=self.conf, + block_pool=self.block_pool, + flashobj=self.flash, oobobj=self.oob, - recorderobj = recorderobj - ) + recorderobj=recorderobj, + ) self.garbage_collector = GarbageCollector( - confobj = self.conf, - flashobj = self.flash, + confobj=self.conf, + flashobj=self.flash, oobobj=self.oob, - block_pool = self.block_pool, - mapping_manager = self.mapping_manager, - recorderobj = recorderobj - ) + block_pool=self.block_pool, + mapping_manager=self.mapping_manager, + recorderobj=recorderobj, + ) # We should initialize Globaltranslationdirectory in Dftl self.mapping_manager.initialize_mappings() - self.n_sec_per_page = self.conf.page_size \ - / self.conf['sector_size'] + self.n_sec_per_page = old_div(self.conf.page_size, self.conf["sector_size"]) - def lba_discard(self, lpn, pid = None): + def lba_discard(self, lpn, pid=None): """ block_pool: no need to update @@ -1563,12 +1606,12 @@ def lba_discard(self, lpn, pid = None): no updates needed updates should be done by GC """ - self.recorder.put('logical_discard', lpn, 'user') + self.recorder.put("logical_discard", lpn, "user") # self.recorder.write_file('lba.trace.txt', - # timestamp = self.oob.timestamp(), - # operation = 'discard', - # lpn = lpn + # timestamp = self.oob.timestamp(), + # operation = 'discard', + # lpn = lpn # ) ppn = self.mapping_manager.lpn_to_ppn(lpn) @@ -1576,8 +1619,9 @@ def lba_discard(self, lpn, pid = None): return # flash page ppn has valid data - self.mapping_manager.cached_mapping_table.overwrite_entry(lpn = lpn, - ppn = UNINITIATED, dirty = True) + self.mapping_manager.cached_mapping_table.overwrite_entry( + lpn=lpn, ppn=UNINITIATED, dirty=True + ) # OOB self.oob.wipe_ppn(ppn) @@ -1595,13 +1639,15 @@ def sec_read(self, sector, count): It returns an array of data. """ lpn_start, lpn_count = self.conf.sec_ext_to_page_ext(sector, count) - self.global_helper.timeline.add_logical_op(sector = sector, count = count, - op = 'LOGICAL_READ') + self.global_helper.timeline.add_logical_op( + sector=sector, count=count, op="LOGICAL_READ" + ) ppns_to_read = self.mapping_manager.ppns_for_reading( - range(lpn_start, lpn_start + lpn_count)) + list(range(lpn_start, lpn_start + lpn_count)) + ) - data = self.flash.read_pages(ppns = ppns_to_read, tag = DATA_USER) + data = self.flash.read_pages(ppns=ppns_to_read, tag=DATA_USER) data = self.page_to_sec_items(data) self.check_read(sector, count, data) @@ -1609,16 +1655,12 @@ def sec_read(self, sector, count): return data def check_read(self, sector, sector_count, data): - for sec, sec_data in zip( - range(sector, sector + sector_count), data): + for sec, sec_data in zip(list(range(sector, sector + sector_count)), data): if sec_data == None: continue if not sec_data.startswith(str(sec)): - msg = "request: sec {} count {}\n".format(sector, sector_count) - msg += "INFTL: Data is not correct. Got: {read}, "\ - "sector={sec}".format( - read = sec_data, - sec = sec) + msg = f"request: sec {sector} count {sector_count}\n" + msg += f"INFTL: Data is not correct. Got: {sec_data}, sector={sec}" # print msg raise RuntimeError(msg) @@ -1636,8 +1678,8 @@ def sec_to_page_items(self, data): if data == None: return None - sec_per_page = self.conf.page_size / self.conf['sector_size'] - n_pages = len(data) / sec_per_page + sec_per_page = old_div(self.conf.page_size, self.conf["sector_size"]) + n_pages = old_div(len(data), sec_per_page) new_data = [] for page in range(n_pages): @@ -1648,26 +1690,28 @@ def sec_to_page_items(self, data): return new_data - def sec_write(self, sector, count, data = None): + def sec_write(self, sector, count, data=None): lpn_start, lpn_count = self.conf.sec_ext_to_page_ext(sector, count) - self.global_helper.timeline.add_logical_op(sector = sector, count = count, - op = 'LOGICAL_WRITE') + self.global_helper.timeline.add_logical_op( + sector=sector, count=count, op="LOGICAL_WRITE" + ) ppns_to_write = self.mapping_manager.ppns_for_writing( - range(lpn_start, lpn_start + lpn_count)) + list(range(lpn_start, lpn_start + lpn_count)) + ) ppn_data = self.sec_to_page_items(data) - self.flash.write_pages(ppns = ppns_to_write, ppn_data = ppn_data, - tag = DATA_USER) + self.flash.write_pages(ppns=ppns_to_write, ppn_data=ppn_data, tag=DATA_USER) self.garbage_collector.try_gc() def sec_discard(self, sector, count): lpn_start, lpn_count = self.conf.sec_ext_to_page_ext(sector, count) - self.global_helper.timeline.add_logical_op(sector = sector, count = count, - op = 'LOGICAL_DISCARD') + self.global_helper.timeline.add_logical_op( + sector=sector, count=count, op="LOGICAL_DISCARD" + ) for lpn in range(lpn_start, lpn_start + lpn_count): self.lba_discard(lpn) @@ -1686,7 +1730,7 @@ def get_type(self): def assert_oob_lpn_eq_req_lpn(mapping_manager, oob, ppn, req_lpn): - if ppn == 'UNINIT': + if ppn == "UNINIT": return if ppn == None: @@ -1694,12 +1738,9 @@ def assert_oob_lpn_eq_req_lpn(mapping_manager, oob, ppn, req_lpn): oob_lpn = oob.translate_ppn_to_lpn(ppn) if oob_lpn != req_lpn: - msg = "oob_lpn {oob_lpn} != req_lpn {req_lpn}. PPN={ppn}\n"\ - .format(oob_lpn = oob_lpn, req_lpn = req_lpn, ppn = ppn) - msg += "Mapping: lpn {} -> ppn {}.\n".format( - 5434, mapping_manager.lpn_to_ppn(5434)) - msg += "Mapping: lpn {} -> ppn {}.\n".format( - 6127, mapping_manager.lpn_to_ppn(6127)) + msg = f"oob_lpn {oob_lpn} != req_lpn {req_lpn}. PPN={ppn}\n" + msg += f"Mapping: lpn {5434} -> ppn {mapping_manager.lpn_to_ppn(5434)}.\n" + msg += f"Mapping: lpn {6127} -> ppn {mapping_manager.lpn_to_ppn(6127)}.\n" raise RuntimeError(msg) @@ -1708,28 +1749,29 @@ def assert_flash_data_startswith_oob_lpn(conf, flash, oob, ppn): if ppn == None: return - if ppn == 'UNINIT': + if ppn == "UNINIT": return flashdata = flash.flash_backend.data oob_lpn = oob.translate_ppn_to_lpn(ppn) sec, sec_count = conf.page_ext_to_sec_ext(oob_lpn, 1) - for sec_num, data in zip(range(sec, sec+sec_count), flashdata[ppn]): + for sec_num, data in zip(list(range(sec, sec + sec_count)), flashdata[ppn]): if not data.startswith(str(sec_num)): - msg = "Flash data does not match its stored sec num"\ - "flash data: {}, ppn: {}. oob_lpn: {} sec:{}".format( - flashdata[ppn], ppn, oob_lpn, - list(range(sec, sec+sec_count)) ) - print msg + msg = ( + "Flash data does not match its stored sec num" + f"flash data: {flashdata[ppn]}, ppn: {ppn}. oob_lpn: {oob_lpn} sec:{list(range(sec, sec + sec_count))}" + ) + print(msg) raise RuntimeError(msg) -def check_data(mapping_manager, conf, flash, oob, ppn, req_lpn = None): + +def check_data(mapping_manager, conf, flash, oob, ppn, req_lpn=None): assert_oob_lpn_eq_req_lpn(mapping_manager, oob, ppn, req_lpn) assert_flash_data_startswith_oob_lpn(conf, flash, oob, ppn) -class ParallelFlash(object): - def __init__(self, confobj, recorderobj, globalhelper = None): +class ParallelFlash: + def __init__(self, confobj, recorderobj, globalhelper=None): self.conf = confobj self.recorder = recorderobj self.global_helper = globalhelper @@ -1741,7 +1783,7 @@ def get_max_channel_page_count(self, ppns): """ pbns = [] for ppn in ppns: - if ppn == 'UNINIT': + if ppn == "UNINIT": # skip it so unitialized ppn does not involve flash op continue block, _ = self.conf.page_to_block_off(ppn) @@ -1770,12 +1812,11 @@ def read_pages(self, ppns, tag): lpns are the corresponding lpns of ppns, we pass them in for checking """ max_count = self.get_max_channel_page_count(ppns) - self.global_helper.timeline.incr_time_stamp('flash.read', - max_count) + self.global_helper.timeline.incr_time_stamp("flash.read", max_count) data = [] for ppn in ppns: - data.append( self.flash_backend.page_read(ppn, tag) ) + data.append(self.flash_backend.page_read(ppn, tag)) return data def write_pages(self, ppns, ppn_data, tag): @@ -1787,8 +1828,7 @@ def write_pages(self, ppns, ppn_data, tag): queue. """ max_count = self.get_max_channel_page_count(ppns) - self.global_helper.timeline.incr_time_stamp('flash.write', - max_count) + self.global_helper.timeline.incr_time_stamp("flash.write", max_count) # save the data to flash if ppn_data == None: @@ -1796,14 +1836,11 @@ def write_pages(self, ppns, ppn_data, tag): self.flash_backend.page_write(ppn, tag) else: for ppn, item in zip(ppns, ppn_data): - self.flash_backend.page_write(ppn, tag, data = item) + self.flash_backend.page_write(ppn, tag, data=item) def erase_blocks(self, pbns, tag): max_count = self.get_max_channel_block_count(pbns) - self.global_helper.timeline.incr_time_stamp('flash.erasure', - max_count) + self.global_helper.timeline.incr_time_stamp("flash.erasure", max_count) for block in pbns: - self.flash_backend.block_erase(block, cat = tag) - - + self.flash_backend.block_erase(block, cat=tag) diff --git a/wiscsee/wiscsim/flash.py b/wiscsee/wiscsim/flash.py index b50ce91..0848c98 100644 --- a/wiscsee/wiscsim/flash.py +++ b/wiscsee/wiscsim/flash.py @@ -1,28 +1,28 @@ import simpy -class SimpleFlash(object): - def __init__(self, recorder, confobj = None): +class SimpleFlash: + def __init__(self, recorder, confobj=None): self.recorder = recorder self.conf = confobj - self.data = {} # ppn -> contents stored in a flash page + self.data = {} # ppn -> contents stored in a flash page def page_read(self, pagenum, cat): - self.recorder.put('physical_read', pagenum, cat) + self.recorder.put("physical_read", pagenum, cat) content = self.data.get(pagenum, None) return content - def page_write(self, pagenum, cat, data = None): - self.recorder.put('physical_write', pagenum, cat) + def page_write(self, pagenum, cat, data=None): + self.recorder.put("physical_write", pagenum, cat) if data != None: self.data[pagenum] = data def block_erase(self, blocknum, cat): # print 'block_erase', blocknum, cat - self.recorder.put('phy_block_erase', blocknum, cat) + self.recorder.put("phy_block_erase", blocknum, cat) ppn_start, ppn_end = self.conf.block_to_page_range(blocknum) for ppn in range(ppn_start, ppn_end): @@ -32,24 +32,25 @@ def block_erase(self, blocknum, cat): # ignore key error pass -class Flash(object): - def __init__(self, recorder, confobj = None, globalhelper = None): + +class Flash: + def __init__(self, recorder, confobj=None, globalhelper=None): self.recorder = recorder # If you enable store data, you must provide confobj - self.store_data = True # whether store data to self.data[] - self.data = {} # ppn -> contents stored in a flash page + self.store_data = True # whether store data to self.data[] + self.data = {} # ppn -> contents stored in a flash page self.conf = confobj def page_read(self, pagenum, cat): - self.recorder.count_me(cat, 'physical_read') + self.recorder.count_me(cat, "physical_read") if self.store_data == True: content = self.data.get(pagenum, None) return content - def page_write(self, pagenum, cat, data = None): - self.recorder.count_me(cat, 'physical_write') + def page_write(self, pagenum, cat, data=None): + self.recorder.count_me(cat, "physical_write") # we only put data to self.data when the caller specify data if self.store_data == True: @@ -58,7 +59,7 @@ def page_write(self, pagenum, cat, data = None): def block_erase(self, blocknum, cat): # print 'block_erase', blocknum, cat - self.recorder.count_me(cat, 'phy_block_erase') + self.recorder.count_me(cat, "phy_block_erase") if self.store_data == True: ppn_start, ppn_end = self.conf.block_to_page_range(blocknum) @@ -68,6 +69,3 @@ def block_erase(self, blocknum, cat): except KeyError: # ignore key error pass - - - diff --git a/wiscsee/wiscsim/ftlbuilder.py b/wiscsee/wiscsim/ftlbuilder.py index 68c6af4..609602a 100644 --- a/wiscsee/wiscsim/ftlbuilder.py +++ b/wiscsee/wiscsim/ftlbuilder.py @@ -1,30 +1,34 @@ import config -import flash -import recorder +from . import flash +from . import recorder -class FtlBuilder(object): + +class FtlBuilder: def __init__(self, confobj, recorderobj, flashobj): if not isinstance(confobj, config.Config): - raise TypeError('confobj is not of type config.Config, it is {}'. - format(type(confobj).__name__)) + raise TypeError( + f"confobj is not of type config.Config, it is {type(confobj).__name__}" + ) if not isinstance(recorderobj, recorder.Recorder): - raise TypeError('recorder is not of type recorder.Recorder, "\ - "it is{}'.format(type(recorderobj).__name__)) + raise TypeError( + f'recorder is not of type recorder.Recorder, "\ + "it is{type(recorderobj).__name__}' + ) if not isinstance(flashobj, flash.Flash): - raise TypeError('flash is not of type flash.Flash'. - format(type(flashobj).__name__)) + raise TypeError("flash is not of type flash.Flash") self.conf = confobj self.recorder = recorderobj self.flash = flashobj - if self.conf['workload_src'] == config.LBAGENERATOR: + if self.conf["workload_src"] == config.LBAGENERATOR: self.recorder.enable() - elif self.conf['workload_src'] == config.WLRUNNER: + elif self.conf["workload_src"] == config.WLRUNNER: self.recorder.disable() else: - raise RuntimeError("workload_src:{} is not supported".format( - self.conf['workload_src'])) + raise RuntimeError( + "workload_src:{} is not supported".format(self.conf["workload_src"]) + ) def lba_read(self, page_num): raise NotImplementedError @@ -65,5 +69,3 @@ def post_processing(self): def get_type(self): return "FtlBuilder" - - diff --git a/wiscsee/wiscsim/ftlcounter.py b/wiscsee/wiscsim/ftlcounter.py index 862613b..6062b70 100644 --- a/wiscsee/wiscsim/ftlcounter.py +++ b/wiscsee/wiscsim/ftlcounter.py @@ -1,19 +1,20 @@ +from past.utils import old_div import bitarray from collections import deque, Counter import csv import datetime import random import os -import Queue +import queue import sys import bidict import config -import flash -import ftlbuilder -import lrulist -import recorder +from . import flash +from . import ftlbuilder +from . import lrulist +from . import recorder from utilities import utils from .blkpool import BlockPool from .bitmap import FlashBitmap2 @@ -26,39 +27,39 @@ class Config(config.ConfigNCQFTL): - def __init__(self, confdic = None): - super(Config, self).__init__(confdic) + def __init__(self, confdic=None): + super().__init__(confdic) local_items = { # number of bytes per entry in mapping_on_flash - "translation_page_entry_bytes": 4, # 32 bits - "cache_entry_bytes": 8, # 4 bytes for lpn, 4 bytes for ppn + "translation_page_entry_bytes": 4, # 32 bits + "cache_entry_bytes": 8, # 4 bytes for lpn, 4 bytes for ppn "GC_threshold_ratio": 0.95, "GC_low_threshold_ratio": 0.9, "over_provisioning": 1.28, - "mapping_cache_bytes": None # cmt: cached mapping table - } + "mapping_cache_bytes": None, # cmt: cached mapping table + } self.update(local_items) @property def n_mapping_entries_per_page(self): - return self.page_size / self['translation_page_entry_bytes'] + return old_div(self.page_size, self["translation_page_entry_bytes"]) @property def mapping_cache_bytes(self): - return self['mapping_cache_bytes'] + return self["mapping_cache_bytes"] @mapping_cache_bytes.setter def mapping_cache_bytes(self, value): - self['mapping_cache_bytes'] = value + self["mapping_cache_bytes"] = value @property def n_cache_entries(self): - return self.mapping_cache_bytes / self['cache_entry_bytes'] + return old_div(self.mapping_cache_bytes, self["cache_entry_bytes"]) @n_cache_entries.setter def n_cache_entries(self, value): - self.mapping_cache_bytes = value * self['cache_entry_bytes'] + self.mapping_cache_bytes = value * self["cache_entry_bytes"] @property def cache_mapped_data_bytes(self): @@ -66,33 +67,36 @@ def cache_mapped_data_bytes(self): @cache_mapped_data_bytes.setter def cache_mapped_data_bytes(self, data_bytes): - self.n_cache_entries = data_bytes / self.page_size + self.n_cache_entries = old_div(data_bytes, self.page_size) if self.n_cache_entries % self.n_mapping_entries_per_page != 0: - print "WARNING: size of mapping cache is not aligned with "\ + print( + "WARNING: size of mapping cache is not aligned with " "translation page size." + ) + @property def translation_page_entry_bytes(self): - return self['translation_page_entry_bytes'] + return self["translation_page_entry_bytes"] @property def over_provisioning(self): - return self['over_provisioning'] + return self["over_provisioning"] @property def GC_threshold_ratio(self): - return self['GC_threshold_ratio'] + return self["GC_threshold_ratio"] @property def GC_low_threshold_ratio(self): - return self['GC_low_threshold_ratio'] + return self["GC_low_threshold_ratio"] def sec_ext_to_page_ext(self, sector, count): """ The sector extent has to be aligned with page return page_start, page_count """ - page = sector / self.n_secs_per_page - page_end = (sector + count) / self.n_secs_per_page + page = old_div(sector, self.n_secs_per_page) + page_end = old_div((sector + count), self.n_secs_per_page) page_count = page_end - page if (sector + count) % self.n_secs_per_page != 0: page_count += 1 @@ -104,15 +108,14 @@ class Ftl(ftlbuilder.FtlBuilder): The implementation literally follows DFtl paper. This class is a coordinator of other coordinators and data structures """ + def __init__(self, confobj, recorderobj, flashobj): - super(Ftl, self).__init__(confobj, recorderobj, flashobj) + super().__init__(confobj, recorderobj, flashobj) if not isinstance(confobj, Config): - raise TypeError("confobj is not Config. it is {}". - format(type(confobj).__name__)) + raise TypeError(f"confobj is not Config. it is {type(confobj).__name__}") - self.n_sec_per_page = self.conf.page_size \ - / self.conf['sector_size'] + self.n_sec_per_page = old_div(self.conf.page_size, self.conf["sector_size"]) self.read_count = Counter() self.write_count = Counter() @@ -130,7 +133,7 @@ def sec_read(self, sector, count): for lpn in range(lpn_start, lpn_start + lpn_count): self.read_count[lpn] += 1 - def sec_write(self, sector, count, data = None): + def sec_write(self, sector, count, data=None): lpn_start, lpn_count = self.conf.sec_ext_to_page_ext(sector, count) self.total_write_bytes += lpn_count * self.conf.page_size @@ -155,12 +158,12 @@ def post_processing(self): """ self.record_traffic() - if self.conf['only_get_traffic'] == True: + if self.conf["only_get_traffic"] == True: return self.do_stats() - if self.conf['gen_ncq_depth_table'] is True: + if self.conf["gen_ncq_depth_table"] is True: self.gen_ncq_depth_table_from_event() def do_stats(self): @@ -168,18 +171,21 @@ def do_stats(self): self.dump_counts(lpns) - if self.conf['do_dump_lpn_sem'] is True: + if self.conf["do_dump_lpn_sem"] is True: self.dump_lpn_sem(lpns) def record_traffic(self): self.recorder.add_to_general_accumulater( - 'traffic_size', 'write', self.total_write_bytes) + "traffic_size", "write", self.total_write_bytes + ) self.recorder.add_to_general_accumulater( - 'traffic_size', 'read', self.total_read_bytes) + "traffic_size", "read", self.total_read_bytes + ) self.recorder.add_to_general_accumulater( - 'traffic_size', 'discard', self.total_discard_bytes) + "traffic_size", "discard", self.total_discard_bytes + ) def dump_counts(self, lpns): counters = [self.read_count, self.write_count, self.discard_count] @@ -189,85 +195,88 @@ def dump_counts(self, lpns): counts = [counter[lpn] for counter in counters] row = [lpn] + counts row = [str(x) for x in row] - row = ' '.join(row) + row = " ".join(row) table.append(row) - count_path = os.path.join(self.conf['result_dir'], 'lpn.count') - with open(count_path, 'w') as f: - f.write('lpn read write discard\n') + count_path = os.path.join(self.conf["result_dir"], "lpn.count") + with open(count_path, "w") as f: + f.write("lpn read write discard\n") for row in table: f.write(row) - f.write('\n') + f.write("\n") # self.clean_up() def dump_lpn_sem(self, lpns): - if self.conf['filesystem'] == 'ext4': + if self.conf["filesystem"] == "ext4": self.dump_lpn_sem_ext4(lpns) - elif self.conf['filesystem'] == 'f2fs': + elif self.conf["filesystem"] == "f2fs": self.dump_lpn_sem_f2fs(lpns) - elif self.conf['filesystem'] == 'xfs': + elif self.conf["filesystem"] == "xfs": self.dump_lpn_sem_xfs(lpns) else: - raise RuntimeError('{} is not supported'.format( - self.conf['filesystem'])) - + raise RuntimeError("{} is not supported".format(self.conf["filesystem"])) def dump_lpn_sem_ext4(self, lpns): classifier = LpnClassification( - lpns = lpns, - device_path = self.conf['device_path'], - result_dir = self.conf['result_dir'], - flash_page_size = 2048) + lpns=lpns, + device_path=self.conf["device_path"], + result_dir=self.conf["result_dir"], + flash_page_size=2048, + ) table = classifier.classify() - lpn_sem_path = os.path.join(self.conf['result_dir'], 'lpn_sem.out') - with open(lpn_sem_path, 'w') as f: + lpn_sem_path = os.path.join(self.conf["result_dir"], "lpn_sem.out") + with open(lpn_sem_path, "w") as f: f.write(utils.table_to_str(table, width=0)) def dump_lpn_sem_f2fs(self, lpns): classifier = F2FSLpnClassification( - lpns = lpns, - device_path = self.conf['device_path'], - result_dir = self.conf['result_dir'], - flash_page_size = 2048) + lpns=lpns, + device_path=self.conf["device_path"], + result_dir=self.conf["result_dir"], + flash_page_size=2048, + ) table = classifier.classify() - lpn_sem_path = os.path.join(self.conf['result_dir'], 'lpn_sem.out') - with open(lpn_sem_path, 'w') as f: + lpn_sem_path = os.path.join(self.conf["result_dir"], "lpn_sem.out") + with open(lpn_sem_path, "w") as f: f.write(utils.table_to_str(table, width=0)) def dump_lpn_sem_xfs(self, lpns): - file_ranges = get_range_table(self.conf['fs_mount_point']) + file_ranges = get_range_table(self.conf["fs_mount_point"]) classifier = XFSLpnClassification( - lpns = lpns, - device_path = self.conf['device_path'], - result_dir = self.conf['result_dir'], - flash_page_size = 2048, - external_ranges = file_ranges - ) + lpns=lpns, + device_path=self.conf["device_path"], + result_dir=self.conf["result_dir"], + flash_page_size=2048, + external_ranges=file_ranges, + ) table = classifier.classify() - lpn_sem_path = os.path.join(self.conf['result_dir'], 'lpn_sem.out') - with open(lpn_sem_path, 'w') as f: + lpn_sem_path = os.path.join(self.conf["result_dir"], "lpn_sem.out") + with open(lpn_sem_path, "w") as f: f.write(utils.table_to_str(table, width=0)) - def get_lpns(self): - lpns = list(set(self.read_count.keys() - + self.write_count.keys() - + self.discard_count.keys())) + lpns = list( + set( + list(self.read_count.keys()) + + list(self.write_count.keys()) + + list(self.discard_count.keys()) + ) + ) lpns = sorted(lpns) return lpns def clean_up(self): - with utils.cd(self.conf['result_dir']): - utils.shcmd('rm blkparse*', ignore_error=True) + with utils.cd(self.conf["result_dir"]): + utils.shcmd("rm blkparse*", ignore_error=True) def get_type(self): return "ftlcounter" @@ -284,52 +293,53 @@ def gen_ncq_depth_table_from_event(self): parser = EventNCQParser(event_workload_iter) table = parser.parse() - ncq_depth_table_path = os.path.join(self.conf['result_dir'], - 'ncq_depth_timeline.txt') - with open(ncq_depth_table_path, 'w') as f: + ncq_depth_table_path = os.path.join( + self.conf["result_dir"], "ncq_depth_timeline.txt" + ) + with open(ncq_depth_table_path, "w") as f: f.write(utils.table_to_str(table, width=0)) -class LpnClassification(object): +class LpnClassification: def __init__(self, lpns, device_path, result_dir, flash_page_size): self.device_path = device_path self.result_dir = result_dir self.flash_page_size = flash_page_size self.lpns = lpns - self.dumpe2fs_out_path = os.path.join(self.result_dir, 'dumpe2fs.out') - self.extents_path = os.path.join(self.result_dir, 'extents.json') + self.dumpe2fs_out_path = os.path.join(self.result_dir, "dumpe2fs.out") + self.extents_path = os.path.join(self.result_dir, "extents.json") self.fs_block_size = 4096 def classify(self): # extents = self._get_extents() # filepath_classifier = blockclassifiers.Ext4FileClassifier(extents, - # self.fs_block_size) + # self.fs_block_size) range_table = self._get_range_table() - classifier = blockclassifiers.Ext4BlockClassifier(range_table, - self.fs_block_size) + classifier = blockclassifiers.Ext4BlockClassifier( + range_table, self.fs_block_size + ) table = [] for lpn in self.lpns: - offset = lpn * self.flash_page_size + offset = lpn * self.flash_page_size sem = classifier.classify(offset) # if sem == 'UNKNOWN': - # sem = filepath_classifier.classify(offset) - row = {'lpn': lpn, - 'sem':sem} + # sem = filepath_classifier.classify(offset) + row = {"lpn": lpn, "sem": sem} table.append(row) return table def _get_extents(self): d = utils.load_json(self.extents_path) - extents = d['extents'] + extents = d["extents"] return extents def _get_range_table(self): - with open(self.dumpe2fs_out_path, 'r') as f: + with open(self.dumpe2fs_out_path) as f: text = f.read() header_text, bg_text = text.split("\n\n\n") @@ -338,26 +348,27 @@ def _get_range_table(self): j_start, j_end = self._get_journal_block_ext(header_text) if j_start != -1: - range_table.append( {'journal': (j_start, j_end)} ) + range_table.append({"journal": (j_start, j_end)}) return range_table def _get_journal_block_ext(self, header_text): header_dict = dumpe2fsparser.parse_header_text(header_text) - if header_dict.has_key('journal-inode') is not True: + if ("journal-inode" in header_dict) is not True: return -1, -1 - journal_inum = header_dict['journal-inode'] - journal_len = header_dict['journal-length'] + journal_inum = header_dict["journal-inode"] + journal_len = header_dict["journal-length"] - ext_text = ext4dumpextents.dump_extents_of_a_file(self.device_path, - '<{}>'.format(journal_inum)) + ext_text = ext4dumpextents.dump_extents_of_a_file( + self.device_path, f"<{journal_inum}>" + ) table = ext4dumpextents.parse_dump_extents_output(ext_text) - return table[0]['Physical_start'], table[0]['Physical_end'] + return table[0]["Physical_start"], table[0]["Physical_end"] -class F2FSLpnClassification(object): +class F2FSLpnClassification: def __init__(self, lpns, device_path, result_dir, flash_page_size): self.device_path = device_path self.result_dir = result_dir @@ -371,10 +382,9 @@ def classify(self): table = [] for lpn in self.lpns: - offset = lpn * self.flash_page_size + offset = lpn * self.flash_page_size sem = classifier.classify(offset) - row = {'lpn': lpn, - 'sem':sem} + row = {"lpn": lpn, "sem": sem} table.append(row) return table @@ -386,20 +396,23 @@ def _get_classifier(self): [SB: 1] [CP: 2] [SIT: 2] [NAT: 4] [SSA: 1] [MAIN: 502(OverProv:70 Resv:48)] """ range_table = [ - {'Superblock': (0, 2*MB)}, - {'Checkpoint': (2*MB, 4*MB)}, - {'SegInfoTab': (4*MB, 6*MB)}, - {'NodeAddrTab': (6*MB, 10*MB)}, - {'SegSummArea': (10*MB, 11*MB)}, - {'MainArea': (20*MB, 1024*MB)}, - ] + {"Superblock": (0, 2 * MB)}, + {"Checkpoint": (2 * MB, 4 * MB)}, + {"SegInfoTab": (4 * MB, 6 * MB)}, + {"NodeAddrTab": (6 * MB, 10 * MB)}, + {"SegSummArea": (10 * MB, 11 * MB)}, + {"MainArea": (20 * MB, 1024 * MB)}, + ] classifier = blockclassifiers.OffsetClassifier(range_table) return classifier -class XFSLpnClassification(object): - def __init__(self, lpns, device_path, result_dir, flash_page_size, external_ranges=None): + +class XFSLpnClassification: + def __init__( + self, lpns, device_path, result_dir, flash_page_size, external_ranges=None + ): self.device_path = device_path self.result_dir = result_dir self.flash_page_size = flash_page_size @@ -413,10 +426,9 @@ def classify(self): table = [] for lpn in self.lpns: - offset = lpn * self.flash_page_size + offset = lpn * self.flash_page_size sem = classifier.classify(offset) - row = {'lpn': lpn, - 'sem':sem} + row = {"lpn": lpn, "sem": sem} table.append(row) return table @@ -435,24 +447,53 @@ def _get_classifier(self): block 32777 (1/9) type freelist block 32778 (1/10) type freelist """ - range_table = [ - {'Journal': (131079*4*KB, 131079*4*KB + 10*MB)} - ] + range_table = [{"Journal": (131079 * 4 * KB, 131079 * 4 * KB + 10 * MB)}] - BLOCKSIZE = 4*KB + BLOCKSIZE = 4 * KB for i in range(8): - ag_start = i * 128*MB + ag_start = i * 128 * MB range_table.extend( [ - {'Superblock' : (ag_start + 0 * BLOCKSIZE, ag_start + 1 * BLOCKSIZE)}, - {'FreeBlockInfo' : (ag_start + 1 * BLOCKSIZE, ag_start + 2 * BLOCKSIZE)}, - {'InodeInfo' : (ag_start + 2 * BLOCKSIZE, ag_start + 3 * BLOCKSIZE)}, - {'FreeListInfo' : (ag_start + 3 * BLOCKSIZE, ag_start + 4 * BLOCKSIZE)}, - {'FreeSpTree1Root' : (ag_start + 4 * BLOCKSIZE, ag_start + 5 * BLOCKSIZE)}, - {'FreeSpTree2Root' : (ag_start + 5 * BLOCKSIZE, ag_start + 6 * BLOCKSIZE)}, - {'InodeTreeRoot' : (ag_start + 6 * BLOCKSIZE, ag_start + 7 * BLOCKSIZE)}, - {'FreeList' : (ag_start + 7 * BLOCKSIZE, ag_start + 11 * BLOCKSIZE)}, - ]) + { + "Superblock": ( + ag_start + 0 * BLOCKSIZE, + ag_start + 1 * BLOCKSIZE, + ) + }, + { + "FreeBlockInfo": ( + ag_start + 1 * BLOCKSIZE, + ag_start + 2 * BLOCKSIZE, + ) + }, + {"InodeInfo": (ag_start + 2 * BLOCKSIZE, ag_start + 3 * BLOCKSIZE)}, + { + "FreeListInfo": ( + ag_start + 3 * BLOCKSIZE, + ag_start + 4 * BLOCKSIZE, + ) + }, + { + "FreeSpTree1Root": ( + ag_start + 4 * BLOCKSIZE, + ag_start + 5 * BLOCKSIZE, + ) + }, + { + "FreeSpTree2Root": ( + ag_start + 5 * BLOCKSIZE, + ag_start + 6 * BLOCKSIZE, + ) + }, + { + "InodeTreeRoot": ( + ag_start + 6 * BLOCKSIZE, + ag_start + 7 * BLOCKSIZE, + ) + }, + {"FreeList": (ag_start + 7 * BLOCKSIZE, ag_start + 11 * BLOCKSIZE)}, + ] + ) if not self.external_ranges is None: range_table.extend(self.external_ranges) @@ -467,17 +508,16 @@ def get_range_table(dirpath): ret_table = [] for row in byte_ranges: - sem = os.path.basename(row['path']) - start = row['start_byte'] - end = start + row['size'] + sem = os.path.basename(row["path"]) + start = row["start_byte"] + end = start + row["size"] new_row = {sem: (start, end)} ret_table.append(new_row) return ret_table - -class EventNCQParser(object): +class EventNCQParser: def __init__(self, event_iter): self.event_iter = event_iter @@ -488,24 +528,24 @@ def parse(self): action = event.action.strip() pre_depth = depth - if action == 'D': + if action == "D": depth += 1 - elif action == 'C': + elif action == "C": depth -= 1 else: - raise RuntimeError('action has to be D or C') + raise RuntimeError("action has to be D or C") post_depth = depth - row = {'action': action, - 'operation': event.operation, - 'timestamp': event.timestamp, - 'offset': event.offset, - 'size': event.size, - 'pid': event.pid, - 'pre_depth': pre_depth, - 'post_depth': post_depth} + row = { + "action": action, + "operation": event.operation, + "timestamp": event.timestamp, + "offset": event.offset, + "size": event.size, + "pid": event.pid, + "pre_depth": pre_depth, + "post_depth": post_depth, + } table.append(row) return table - - diff --git a/wiscsee/wiscsim/ftlsim_commons.py b/wiscsee/wiscsim/ftlsim_commons.py index a86270f..2a0eb97 100644 --- a/wiscsee/wiscsim/ftlsim_commons.py +++ b/wiscsee/wiscsim/ftlsim_commons.py @@ -1,8 +1,10 @@ +from past.utils import old_div import simpy import random from collections import OrderedDict -class Extent(object): + +class Extent: def __init__(self, lpn_start, lpn_count): assert lpn_count > 0 self.lpn_start = lpn_start @@ -19,11 +21,10 @@ def end_lpn(self): return self.lpn_start + self.lpn_count def lpn_iter(self): - return range(self.lpn_start, self.end_lpn()) + return list(range(self.lpn_start, self.end_lpn())) def __str__(self): - return "lpn_start: {}, lpn_count: {}".format( - self.lpn_start, self.lpn_count) + return f"lpn_start: {self.lpn_start}, lpn_count: {self.lpn_count}" def __contains__(self, lpn): return lpn >= self.lpn_start and lpn < self.end_lpn() @@ -34,17 +35,16 @@ def __copy__(self): class CacheExtent(Extent): def __init__(self, lpn_start, lpn_count, in_cache): - super(CacheExtent, self).__init__(lpn_start, lpn_count) + super().__init__(lpn_start, lpn_count) self.in_cache = in_cache def __str__(self): - return "{}, in_cache: {}".format( - super(CacheExtent, self).__str__(), self.in_cache) + return f"{super().__str__()}, in_cache: {self.in_cache}" def display_extents(extent_list): for ext in extent_list: - print str(ext) + print(str(ext)) class SSDRequest(CacheExtent): @@ -53,41 +53,36 @@ def __init__(self, lpn_start, lpn_count, in_cache, operation): self.operation = operation def __str__(self): - return "{}, operation: {}".format( - super(CacheExtent, self).__str__(), self.operation) + return f"{super(CacheExtent, self).__str__()}, operation: {self.operation}" def create_ssd_request(conf, event): - lpn_start, lpn_count = conf.sec_ext_to_page_ext( - event.sector, event.sector_count) - return SSDRequest( - lpn_start, - lpn_count, - None, - event.operation) + lpn_start, lpn_count = conf.sec_ext_to_page_ext(event.sector, event.sector_count) + return SSDRequest(lpn_start, lpn_count, None, event.operation) -class NCQSingleQueue(object): +class NCQSingleQueue: """ User of the queue can take up to depth # of request without returning """ + def __init__(self, ncq_depth, simpy_env): self.ncq_depth = ncq_depth self.env = simpy_env - self.queue = OrderedDict() # simpy.Store(self.env) + self.queue = OrderedDict() # simpy.Store(self.env) # ssd need to grab a slot before get item from queue self.slots = simpy.Resource(self.env, capacity=ncq_depth) def hold_all_slots(self): held_slot_reqs = [] for i in range(self.ncq_depth): - slot_req = self.slots.request() + slot_req = self.slots.request() held_slot_reqs.append(slot_req) yield simpy.events.AllOf(self.env, held_slot_reqs) - self.env.exit(held_slot_reqs) + return held_slot_reqs def release_all_slots(self, held_slot_reqs): """Must be used in pair with hold_all_slots()""" @@ -104,7 +99,7 @@ def split_ext_by_segment(n_pages_per_segment, extent): cur_ext = None exts = {} for lpn in extent.lpn_iter(): - seg_id = lpn / n_pages_per_segment + seg_id = old_div(lpn, n_pages_per_segment) if seg_id == last_seg_id: cur_ext.lpn_count += 1 else: @@ -119,15 +114,14 @@ def split_ext_by_segment(n_pages_per_segment, extent): return exts -class LockPool(object): +class LockPool: def __init__(self, simpy_env): - self.resources = {} # addr: lock + self.resources = {} # addr: lock self.env = simpy_env self.locked_addrs = set() def get_request(self, addr): - res = self.resources.setdefault(addr, - simpy.Resource(self.env, capacity = 1)) + res = self.resources.setdefault(addr, simpy.Resource(self.env, capacity=1)) return res.request() def release_request(self, addr, request): @@ -137,10 +131,6 @@ def release_request(self, addr, request): random.seed(1) + def random_channel_id(n_channels_per_dev): return random.randint(0, n_channels_per_dev - 1) - - - - - diff --git a/wiscsee/wiscsim/gc_analysis.py b/wiscsee/wiscsim/gc_analysis.py index 10d2e3c..2076ed8 100644 --- a/wiscsee/wiscsim/gc_analysis.py +++ b/wiscsee/wiscsim/gc_analysis.py @@ -2,7 +2,7 @@ import csv from commons import * -from ftlsim_commons import * +from .ftlsim_commons import * from .host import Host from utilities import utils @@ -10,53 +10,58 @@ from pyreuse.fsutils import ext4dumpextents - -class GcLog(object): +class GcLog: def __init__(self, device_path, result_dir, flash_page_size): self.device_path = device_path self.result_dir = result_dir self.flash_page_size = flash_page_size - self.gclog_path = os.path.join(self.result_dir, 'gc.log') - self.dumpe2fs_out_path = os.path.join(self.result_dir, 'dumpe2fs.out') - self.extents_path = os.path.join(self.result_dir, 'extents.json') + self.gclog_path = os.path.join(self.result_dir, "gc.log") + self.dumpe2fs_out_path = os.path.join(self.result_dir, "dumpe2fs.out") + self.extents_path = os.path.join(self.result_dir, "extents.json") self.fs_block_size = 4096 def classify_lpn_in_gclog(self): extents = self._get_extents() - filepath_classifier = blockclassifiers.Ext4FileClassifier(extents, - self.fs_block_size) + filepath_classifier = blockclassifiers.Ext4FileClassifier( + extents, self.fs_block_size + ) range_table = self._get_range_table() - classifier = blockclassifiers.Ext4BlockClassifier(range_table, - self.fs_block_size) + classifier = blockclassifiers.Ext4BlockClassifier( + range_table, self.fs_block_size + ) new_table = [] - with open(self.gclog_path , 'rb') as f: + with open(self.gclog_path, "rb") as f: reader = csv.DictReader(f, skipinitialspace=True) for row in reader: - newrow = dict(zip(row.keys()[0].split(), row.values()[0].split())) - if newrow['lpn'] != 'NA': - offset = int(newrow['lpn']) * self.flash_page_size + newrow = dict( + list( + zip(list(row.keys())[0].split(), list(row.values())[0].split()) + ) + ) + if newrow["lpn"] != "NA": + offset = int(newrow["lpn"]) * self.flash_page_size sem = classifier.classify(offset) - if sem == 'UNKNOWN': + if sem == "UNKNOWN": sem = filepath_classifier.classify(offset) else: - sem = 'NA' - newrow['semantics'] = sem + sem = "NA" + newrow["semantics"] = sem new_table.append(newrow) - with open(self.gclog_path+'.parsed', 'w') as f: + with open(self.gclog_path + ".parsed", "w") as f: f.write(utils.table_to_str(new_table)) def _get_extents(self): d = utils.load_json(self.extents_path) - extents = d['extents'] + extents = d["extents"] return extents def _get_range_table(self): - with open(self.dumpe2fs_out_path, 'r') as f: + with open(self.dumpe2fs_out_path) as f: text = f.read() header_text, bg_text = text.split("\n\n\n") @@ -65,22 +70,21 @@ def _get_range_table(self): j_start, j_end = self._get_journal_block_ext(header_text) if j_start != -1: - range_table.append( {'journal': (j_start, j_end)} ) + range_table.append({"journal": (j_start, j_end)}) return range_table def _get_journal_block_ext(self, header_text): header_dict = dumpe2fsparser.parse_header_text(header_text) - if header_dict.has_key('journal-inode') is not True: + if ("journal-inode" in header_dict) is not True: return -1, -1 - journal_inum = header_dict['journal-inode'] - journal_len = header_dict['journal-length'] + journal_inum = header_dict["journal-inode"] + journal_len = header_dict["journal-length"] - ext_text = ext4dumpextents.dump_extents_of_a_file(self.device_path, - '<{}>'.format(journal_inum)) + ext_text = ext4dumpextents.dump_extents_of_a_file( + self.device_path, f"<{journal_inum}>" + ) table = ext4dumpextents.parse_dump_extents_output(ext_text) - return table[0]['Physical_start'], table[0]['Physical_end'] - - + return table[0]["Physical_start"], table[0]["Physical_end"] diff --git a/wiscsee/wiscsim/host.py b/wiscsee/wiscsim/host.py index e2d8d99..924b8e2 100644 --- a/wiscsee/wiscsim/host.py +++ b/wiscsee/wiscsim/host.py @@ -1,17 +1,17 @@ from commons import * -from ftlsim_commons import * -import hostevent +from .ftlsim_commons import * +from . import hostevent -class Host(object): +class Host: def __init__(self, conf, simpy_env, event_iter): self.conf = conf self.env = simpy_env self.event_iter = event_iter self._ncq = NCQSingleQueue( - ncq_depth = self.conf['SSDFramework']['ncq_depth'], - simpy_env = self.env) + ncq_depth=self.conf["SSDFramework"]["ncq_depth"], simpy_env=self.env + ) def get_ncq(self): return self._ncq @@ -22,12 +22,10 @@ def _process(self): # due to padding, accesing disk head will be negative. continue - if event.action == 'D': + if event.action == "D": self._ncq.queue[event] = "" def run(self): self._process() self._ncq.queue[hostevent.ControlEvent(OP_SHUT_SSD)] = "" - yield simpy.AllOf(self.env, []) - - + yield simpy.AllOf(self.env, []) diff --git a/wiscsee/wiscsim/hostevent.py b/wiscsee/wiscsim/hostevent.py index 5d37ac0..0e9d76d 100644 --- a/wiscsee/wiscsim/hostevent.py +++ b/wiscsee/wiscsim/hostevent.py @@ -1,7 +1,9 @@ -from ftlsim_commons import Extent +from past.utils import old_div +from .ftlsim_commons import Extent from commons import * -class HostEventBase(object): + +class HostEventBase: def get_operation(self): raise NotImplementedError @@ -15,22 +17,31 @@ def __init__(self, operation, arg1=None, arg2=None, arg3=None): self.arg1 = arg1 self.arg2 = arg2 self.arg3 = arg3 - self.action = 'D' # following the format of data event + self.action = "D" # following the format of data event def get_operation(self): return self.operation def get_type(self): - return 'ControlEvent' + return "ControlEvent" def __str__(self): - return "ControlEvent: {}: {}, {}, {}".format(self.operation, - self.arg1, self.arg2, self.arg3) + return f"ControlEvent: {self.operation}: {self.arg1}, {self.arg2}, {self.arg3}" class Event(HostEventBase): - def __init__(self, sector_size, pid, operation, offset, size, - timestamp = None, pre_wait_time = None, sync = True, action = 'D'): + def __init__( + self, + sector_size, + pid, + operation, + offset, + size, + timestamp=None, + pre_wait_time=None, + sync=True, + action="D", + ): self.pid = int(pid) self.operation = operation self.offset = int(offset) @@ -39,64 +50,62 @@ def __init__(self, sector_size, pid, operation, offset, size, self.timestamp = timestamp self.pre_wait_time = pre_wait_time self.action = action - assert action in ('D', 'C'), "action:{}".format(action) + assert action in ("D", "C"), f"action:{action}" - assert self.offset % sector_size == 0,\ - "offset {} is not aligned with sector size {}.".format( - self.offset, sector_size) - self.sector = self.offset / sector_size + assert self.offset % sector_size == 0, ( + f"offset {self.offset} is not aligned with sector size {sector_size}." + ) + self.sector = old_div(self.offset, sector_size) - assert self.size % sector_size == 0, \ - "size {} is not multiple of sector size {}".format( - self.size, sector_size) + assert self.size % sector_size == 0, ( + f"size {self.size} is not multiple of sector size {sector_size}" + ) - self.sector_count = self.size / sector_size + self.sector_count = old_div(self.size, sector_size) def get_operation(self): return self.operation def get_type(self): - return 'Event' + return "Event" def get_lpn_extent(self, conf): lpn_start, lpn_count = conf.off_size_to_page_range( - self.offset, self.size, force_alignment=False) - return Extent(lpn_start = lpn_start, lpn_count = lpn_count) + self.offset, self.size, force_alignment=False + ) + return Extent(lpn_start=lpn_start, lpn_count=lpn_count) def __str__(self): - return "Event pid:{pid}, operation:{operation}, offset:{offset}, "\ - "size:{size}, sector:{sector}, sector_count:{sector_count}, "\ - "sync:{sync}, timestamp:{timestamp}, action:{action}"\ - .format(pid = self.pid, operation = self.operation, - offset = self.offset, size = self.size, - sector = self.sector, sector_count = self.sector_count, - sync = self.sync, timestamp = self.timestamp, - action = self.action) + return ( + f"Event pid:{self.pid}, operation:{self.operation}, offset:{self.offset}, " + f"size:{self.size}, sector:{self.sector}, sector_count:{self.sector_count}, " + f"sync:{self.sync}, timestamp:{self.timestamp}, action:{self.action}" + ) -class FileLineIterator(object): +class FileLineIterator: def __init__(self, file_path): self.file_path = file_path def __iter__(self): - with open(self.file_path, 'r') as f: + with open(self.file_path) as f: for line in f: line = line.strip() yield line -class EventIterator(object): +class EventIterator: """ Convert string line to event, and iter """ + def __init__(self, conf, filelineiter): self.conf = conf - self.sector_size = self.conf['sector_size'] + self.sector_size = self.conf["sector_size"] self.filelineiter = filelineiter - self.event_file_column_names = self.conf['event_file_column_names'] + self.event_file_column_names = self.conf["event_file_column_names"] - self._translation = {'read': OP_READ, 'write': OP_WRITE, - 'discard':OP_DISCARD} + self._translation = {"read": OP_READ, "write": OP_WRITE, "discard": OP_DISCARD} def _convert(self, op_in_file): return self._translation[op_in_file] @@ -104,19 +113,18 @@ def _convert(self, op_in_file): def str_to_event(self, line): items = line.split() if len(self.event_file_column_names) != len(items): - raise RuntimeError("Lengths not equal: {} {}".format( - self.event_file_column_names, items)) - dic = dict(zip(self.event_file_column_names, items)) - dic['sector_size'] = self.sector_size - if dic['pre_wait_time'] != 'NA': - dic['pre_wait_time'] = float(dic['pre_wait_time']) + raise RuntimeError( + f"Lengths not equal: {self.event_file_column_names} {items}" + ) + dic = dict(list(zip(self.event_file_column_names, items))) + dic["sector_size"] = self.sector_size + if dic["pre_wait_time"] != "NA": + dic["pre_wait_time"] = float(dic["pre_wait_time"]) - dic['operation'] = self._convert(dic['operation']) + dic["operation"] = self._convert(dic["operation"]) return Event(**dic) def __iter__(self): for line in self.filelineiter: yield self.str_to_event(line) - - diff --git a/wiscsee/wiscsim/learnedftl.py b/wiscsee/wiscsim/learnedftl.py index db03ad7..40b5b8f 100644 --- a/wiscsee/wiscsim/learnedftl.py +++ b/wiscsee/wiscsim/learnedftl.py @@ -1,18 +1,22 @@ -from asyncore import write +from past.utils import old_div + +# from asyncore import write import bidict import sys import copy from collections import deque, OrderedDict, defaultdict import datetime import time -import Queue +import queue import itertools import math import objgraph + # from pympler import asizeof import pickle from bisect import bisect_left, insort_left import numpy as np + # from wiscsim.lsm_tree.bloom_filter import BloomFilter import bitarray import bitarray.util @@ -20,15 +24,15 @@ from wiscsim.sftl import SFTLPage, DFTLPage import config -import ftlbuilder -from datacache import * -import recorder +from . import ftlbuilder +from .datacache import * +from . import recorder from utilities import utils from .bitmap import FlashBitmap2 from wiscsim.devblockpool import * -from ftlsim_commons import * +from .ftlsim_commons import * from commons import * -import dftldes +from . import dftldes LPN_TO_DEBUG = -1 @@ -49,10 +53,10 @@ # FTL components -class Ftl(ftlbuilder.FtlBuilder): +class Ftl(ftlbuilder.FtlBuilder): def __init__(self, confobj, recorderobj, flashobj, simpy_env, des_flash, ncq): - super(Ftl, self).__init__(confobj, recorderobj, flashobj) + super().__init__(confobj, recorderobj, flashobj) self.des_flash = des_flash self.env = simpy_env @@ -75,20 +79,20 @@ def __init__(self, confobj, recorderobj, flashobj, simpy_env, des_flash, ncq): self.promotion_interval = 500 * MB self.gc_interval = 1000 * MB self.rw_events = 0 - #self.buffer = WriteBuffer(16*self.conf.n_pages_per_block, 16*self.conf.n_pages_per_block, filtering=7.0/8.0) - #self.buffer = SimpleWriteBuffer(self.conf.n_pages_per_block) - #self.datacache = DataCache(self.conf['cache_size'], self.conf.page_size, method="LRU", priority=False) + # self.buffer = WriteBuffer(16*self.conf.n_pages_per_block, 16*self.conf.n_pages_per_block, filtering=7.0/8.0) + # self.buffer = SimpleWriteBuffer(self.conf.n_pages_per_block) + # self.datacache = DataCache(self.conf['cache_size'], self.conf.page_size, method="LRU", priority=False) # print(self.conf.page_size) - self.rw_cache = RWCache(self.conf['cache_size'], self.conf.page_size, 8*MB, 7.0/8.0) + self.rw_cache = RWCache( + self.conf["cache_size"], self.conf.page_size, 8 * MB, 7.0 / 8.0 + ) self.hist = defaultdict(int) self.read_latencies = [] self.write_latencies = [] - self.waf = {"request" : 0, "actual" : 0} - self.raf = {"request" : 0, "actual" : 0} + self.waf = {"request": 0, "actual": 0} + self.raf = {"request": 0, "actual": 0} self.enable_recording = False - - def recorder_enabled(self, enable=True): self.enable_recording = enable @@ -97,50 +101,91 @@ def lpn_to_ppn(self, lpn): return self.metadata.lpn_to_ppn(lpn) def end_ssd(self): - self.metadata.mapping_table.compact(promote=True) - log_msg("End-to-end overall response time per page: %.2fus; Num of requests %d" % ((np.sum(self.write_latencies) + np.sum(self.read_latencies)) / (self.waf["request"] + self.raf['request']), self.waf["request"] + self.raf['request'])) + log_msg( + f"End-to-end overall response time per page: {old_div((np.sum(self.write_latencies) + np.sum(self.read_latencies)), (self.waf['request'] + self.raf['request'])):.2f}us; Num of requests {self.waf['request'] + self.raf['request']}" + ) if len(self.read_latencies) > 0: - log_msg("End-to-end read response time per page: %.2fus; Num of reads %d" % (np.sum(self.read_latencies) / self.raf['request'], self.raf['request'])) + log_msg( + f"End-to-end read response time per page: {old_div(np.sum(self.read_latencies), self.raf['request']):.2f}us; Num of reads {self.raf['request']}" + ) if len(self.write_latencies) > 0: - log_msg("End-to-end write response time per page: %.2fus; Num of writes %d" % (np.sum(self.write_latencies) / self.waf["request"], self.waf['request'])) - - if self.waf['request'] > 0: - log_msg("Write Amplification Factor: %.2f; Actual: %d; Request: %d" % (self.waf['actual'] / float(self.waf['request']), self.waf['actual'], self.waf['request'])) - - if self.raf['request'] > 0: - log_msg("Read Amplification Factor: %.2f; Actual: %d; Request: %d" % (self.raf['actual'] / float(self.raf['request']), self.raf['actual'], self.raf['request'])) - - if self.counter['mapping_table_write_miss'] + self.counter['mapping_table_write_hit'] > 0: - log_msg("Mapping Table Write Miss Ratio: %.2f" % (self.counter['mapping_table_write_miss'] / float(self.counter['mapping_table_write_miss'] + self.counter['mapping_table_write_hit']))) - - if self.counter['mapping_table_read_miss'] + self.counter['mapping_table_read_hit'] > 0: - log_msg("Mapping Table Read Miss Ratio: %.2f" % (self.counter['mapping_table_read_miss'] / float(self.counter['mapping_table_read_miss'] + self.counter['mapping_table_read_hit']))) + log_msg( + f"End-to-end write response time per page: {old_div(np.sum(self.write_latencies), self.waf['request']):.2f}us; Num of writes {self.waf['request']}" + ) + + if self.waf["request"] > 0: + log_msg( + f"Write Amplification Factor: {self.waf['actual'] / float(self.waf['request']):.2f}; Actual: {self.waf['actual']}; Request: {self.waf['request']}" + ) + + if self.raf["request"] > 0: + log_msg( + f"Read Amplification Factor: {self.raf['actual'] / float(self.raf['request']):.2f}; Actual: {self.raf['actual']}; Request: {self.raf['request']}" + ) + + if ( + self.counter["mapping_table_write_miss"] + + self.counter["mapping_table_write_hit"] + > 0 + ): + log_msg( + "Mapping Table Write Miss Ratio: %.2f" + % ( + self.counter["mapping_table_write_miss"] + / float( + self.counter["mapping_table_write_miss"] + + self.counter["mapping_table_write_hit"] + ) + ) + ) + + if ( + self.counter["mapping_table_read_miss"] + + self.counter["mapping_table_read_hit"] + > 0 + ): + log_msg( + "Mapping Table Read Miss Ratio: %.2f" + % ( + self.counter["mapping_table_read_miss"] + / float( + self.counter["mapping_table_read_miss"] + + self.counter["mapping_table_read_hit"] + ) + ) + ) log_msg(self.counter) if sum(self.metadata.levels.values()) > 0: - log_msg("Avg lookup", sum(self.metadata.levels.values()), sum(int(k)*int(v) for k, v in self.metadata.levels.items()) / float(sum(self.metadata.levels.values()))) + log_msg( + "Avg lookup", + sum(self.metadata.levels.values()), + sum(int(k) * int(v) for k, v in list(self.metadata.levels.items())) + / float(sum(self.metadata.levels.values())), + ) - self.recorder.append_to_value_list('distribution of lookups', - self.metadata.levels) + self.recorder.append_to_value_list( + "distribution of lookups", self.metadata.levels + ) - crb_distribution = [] - for frame in self.metadata.mapping_table.frames.values(): + for frame in list(self.metadata.mapping_table.frames.values()): crb_size = 0 for segment in frame.segments: if segment.filter: crb_size += len([e for e in segment.filter if e]) crb_distribution.append(crb_size) - self.recorder.append_to_value_list('distribution of CRB size', - crb_distribution) + self.recorder.append_to_value_list("distribution of CRB size", crb_distribution) if len(crb_distribution) > 0: - log_msg("CRB avg size: %.2f, CRB 99 size: %.2f, CRB variation: %.2f" % (np.average(crb_distribution), np.percentile(crb_distribution, 99), np.std(crb_distribution))) + log_msg( + f"CRB avg size: {np.average(crb_distribution):.2f}, CRB 99 size: {np.percentile(crb_distribution, 99):.2f}, CRB variation: {np.std(crb_distribution):.2f}" + ) def display_msg(self, mode): # log_msg('Event', self.rw_events, 'Read (MB)', self.read_bytes / MB, 'reading', round(float(req_size) / MB, 2)) @@ -152,8 +197,31 @@ def display_msg(self, mode): avg_lookup = 0 if float(sum(self.metadata.levels.values())) != 0: - avg_lookup = sum(int(k)*int(v) for k, v in self.metadata.levels.items()) / float(sum(self.metadata.levels.values())) - log_msg('Event', self.rw_events, '%s (MB)' % mode, display_bytes / MB, "Mapping Table", self.metadata.mapping_table.memory, "Reference Mapping Table", self.metadata.reference_mapping_table.memory, "Distribution of lookups", self.metadata.levels[1], sum(self.metadata.levels.values()), avg_lookup, "Misprediction", self.hist, "Latency per page: %.2fus" % ((np.sum(self.write_latencies) + np.sum(self.read_latencies)) / (self.waf["request"] + self.raf['request']))) + avg_lookup = sum( + int(k) * int(v) for k, v in list(self.metadata.levels.items()) + ) / float(sum(self.metadata.levels.values())) + log_msg( + "Event", + self.rw_events, + f"{mode} (MB)", + old_div(display_bytes, MB), + "Mapping Table", + self.metadata.mapping_table.memory, + "Reference Mapping Table", + self.metadata.reference_mapping_table.memory, + "Distribution of lookups", + self.metadata.levels[1], + sum(self.metadata.levels.values()), + avg_lookup, + "Misprediction", + self.hist, + "Latency per page: {:.2f}us".format( + old_div( + (np.sum(self.write_latencies) + np.sum(self.read_latencies)), + (self.waf["request"] + self.raf["request"]), + ) + ), + ) sys.stdout.flush() # if float(self.waf['request']) > 0: @@ -171,7 +239,7 @@ def display_msg(self, mode): def read_ext(self, extent): should_print = False req_size = extent.lpn_count * self.conf.page_size - self.recorder.add_to_general_accumulater('traffic', 'read', req_size) + self.recorder.add_to_general_accumulater("traffic", "read", req_size) self.read_bytes += req_size self.rw_events += 1 if self.read_bytes > self.pre_read_bytes + self.display_interval: @@ -179,8 +247,8 @@ def read_ext(self, extent): self.pre_read_bytes = self.read_bytes extents = split_ext(extent) - start_time = self.env.now # <----- start - + start_time = self.env.now # <----- start + op_id = self.recorder.get_unique_num() # if op_id == 279353: # should_print = True @@ -193,19 +261,18 @@ def read_ext(self, extent): log_msg(start_time) for ext in extents: - # nothing to lookup if no lpn is written + # nothing to lookup if no lpn is written if not self.metadata.reference_mapping_table.get(ext.lpn_start): - self.counter['RAW'] += 1 + self.counter["RAW"] += 1 continue - assert(ext.lpn_count == 1) + assert ext.lpn_count == 1 requested_read += 1 - #cachehit, writeback, lpn_previous = self.datacache.read(ext.lpn_start) - + # cachehit, writeback, lpn_previous = self.datacache.read(ext.lpn_start) + # cachehit = self.datacache.read(ext.lpn_start) # or self.buffer.read(ext.lpn_start) cachehit, writeback, ppn = self.rw_cache.read(ext.lpn_start) - if not cachehit: procs += [self.env.process(self.read_logical_block(ext, should_print))] else: @@ -213,10 +280,10 @@ def read_ext(self, extent): yield self.env.timeout(CACHE_HIT) # yield self.env.timeout(delay=100) # if writeback: - # additional_lpns_to_evict = self.datacache.evict_extra(0) - # yield self.env.process(self.write_ext(Extent(lpn_previous, 1), cached=False)) - # yield extra writes - + # additional_lpns_to_evict = self.datacache.evict_extra(0) + # yield self.env.process(self.write_ext(Extent(lpn_previous, 1), cached=False)) + # yield extra writes + if writeback: yield self.env.process(self._write_ppns([ppn])) total_pages_to_write.append(ppn) @@ -228,33 +295,45 @@ def read_ext(self, extent): ret = yield simpy.AllOf(self.env, procs) - total_pages_to_read += [page for pages in ret.values() for page in pages[1]] - total_pages_to_write += [page for pages in ret.values() for page in pages[0]] + total_pages_to_read += [ + page for pages in list(ret.values()) for page in pages[1] + ] + total_pages_to_write += [ + page for pages in list(ret.values()) for page in pages[0] + ] lpns_to_read = float(len(total_pages_to_read)) - + end_time = self.env.now if self.enable_recording: if requested_read > 0: - self.read_latencies += [(end_time - start_time) / 1000.0] # [(end_time - start_time)/(1000.0*requested_read)]*int(requested_read) + self.read_latencies += [ + (end_time - start_time) / 1000.0 + ] # [(end_time - start_time)/(1000.0*requested_read)]*int(requested_read) self.raf["request"] += requested_read self.raf["actual"] += lpns_to_read self.waf["actual"] += len(total_pages_to_write) - write_timeline(self.conf, self.recorder, - op_id = op_id, op = 'read_ext', arg = extent.lpn_count, - start_time = start_time, end_time = end_time) + write_timeline( + self.conf, + self.recorder, + op_id=op_id, + op="read_ext", + arg=extent.lpn_count, + start_time=start_time, + end_time=end_time, + ) # if lpns_to_read > len(extents) and len(extents) >= 6: # log_msg(lpns_to_read, len(extents), requested_read, len(pages_to_write), ((end_time - start_time)/(1000.0*requested_read)), end_time - start_time) # log_msg([page // self.conf.n_pages_per_channel for page in pages_to_read]) - #self.env.exit(lpns_to_read) + # self.env.exit(lpns_to_read) def read_logical_block(self, extent, should_print=False): - assert(extent.lpn_count == 1) + assert extent.lpn_count == 1 # replace the following lines with a nice interface lpn = extent.lpn_start @@ -277,11 +356,10 @@ def read_logical_block(self, extent, should_print=False): # if len(read_ppns) >= 2: # self.datacache.set_priority(extent.lpn_start, priority=True) # read_ppns = read_ppns[-1:] - block_id = ppn / self.conf.n_pages_per_block - + block_id = old_div(ppn, self.conf.n_pages_per_block) - #content = self.flash.page_read(ppn, cat = TAG_FORGROUND) - #contents.append(content) + # content = self.flash.page_read(ppn, cat = TAG_FORGROUND) + # contents.append(content) procs = [] for read_ppn in pages_to_read: if should_print: @@ -301,20 +379,21 @@ def read_logical_block(self, extent, should_print=False): # # yield self.env.timeout(delay=self.conf['flash_config']['t_R']) # yield self.env.process( # self.des_flash.rw_ppns([0], 'read', tag = "Unknown")) - self.env.exit((pages_to_write, pages_to_read)) + return (pages_to_write, pages_to_read) def lba_write(self, lpn, data=None): yield self.env.process( - self.write_ext(Extent(lpn_start=lpn, lpn_count=1), [data])) + self.write_ext(Extent(lpn_start=lpn, lpn_count=1), [data]) + ) # yield self.env.process(self.garbage_collector.clean()) def write_ext(self, extent, data=None): req_size = extent.lpn_count * self.conf.page_size - self.recorder.add_to_general_accumulater('traffic', 'write', req_size) + self.recorder.add_to_general_accumulater("traffic", "write", req_size) self.written_bytes += req_size self.rw_events += 1 - + if self.written_bytes > self.pre_written_bytes + self.display_interval: self.metadata.mapping_table.compact() self.metadata.mapping_table.promote() @@ -323,8 +402,8 @@ def write_ext(self, extent, data=None): yield self.env.timeout(COMPACTION_DELAY) # roots = objgraph.get_leaking_objects() - # objgraph.show_most_common_types() - # objgraph.show_growth() + # objgraph.show_most_common_types() + # objgraph.show_growth() # print(sys.getsizeof(self.metadata)) # print(sys.getsizeof(self.metadata.mapping_table)) @@ -349,23 +428,21 @@ def write_ext(self, extent, data=None): # print(asizeof.asizeof(self.metadata.mapping_table.memory_counter)) # print(asizeof.asizeof(self.metadata.mapping_table.frame_on_flash)) # print(asizeof.asizeof(self.metadata.mapping_table.dirty)) - - # print(asizeof.asizeof(self.metadata.mapping_table.frames.values()[0].runs)) # print(asizeof.asizeof(self.metadata.mapping_table.frames.values()[0].segments)) # log_msg('Mapping Table Compaction Begins') - - # if self.written_bytes % self.compaction_interval == 0: + + # if self.written_bytes % self.compaction_interval == 0: self.metadata.mapping_table.compact() - # if self.written_bytes % self.promotion_interval == 0: + # if self.written_bytes % self.promotion_interval == 0: self.metadata.mapping_table.promote() # log_msg('Mapping Table Compaction Ends') extents = split_ext(extent) - start_time = self.env.now # <----- start + start_time = self.env.now # <----- start op_id = self.recorder.get_unique_num() write_procs = [] total_pages_to_write = [] @@ -378,11 +455,11 @@ def write_ext(self, extent, data=None): # yield self.env.process(self.write_ext(Extent(lpn_previous, 1), cached=False)) # yield extra writes # else: - #self.datacache.resize(4*MB - self.metadata.mapping_table.memory) - #self.datacache.resize(4*MB - self.metadata.reference_mapping_table.memory) - # for lpn in exts: - # p = self.env.process(self.write_single_page(mappings[lpn])) - # write_procs.append(p) + # self.datacache.resize(4*MB - self.metadata.mapping_table.memory) + # self.datacache.resize(4*MB - self.metadata.reference_mapping_table.memory) + # for lpn in exts: + # p = self.env.process(self.write_single_page(mappings[lpn])) + # write_procs.append(p) # print(ext.lpn_start) # if self.rw_events > 10: # exit() @@ -396,7 +473,6 @@ def write_ext(self, extent, data=None): write_procs.append(p) total_pages_to_write.append(ppn) - # self.datacache.invalidate(ext) if self.rw_cache.should_assign_page(): exts = self.rw_cache.flush_unassigned() @@ -406,12 +482,12 @@ def write_ext(self, extent, data=None): # if len(pages_to_read) != 0 or len(pages_to_write) != 0: # print(pages_to_read, pages_to_write) - # self.buffer.write(ext.lpn_start) - # self.datacache.invalidate(ext) - # if self.buffer.should_flush(): - # exts = self.buffer.flush() - # self.counter += len(exts) - # mappings = self.metadata.update(exts) + # self.buffer.write(ext.lpn_start) + # self.datacache.invalidate(ext) + # if self.buffer.should_flush(): + # exts = self.buffer.flush() + # self.counter += len(exts) + # mappings = self.metadata.update(exts) for ppn in pages_to_write: p = self.env.process(self._write_ppns([ppn])) write_procs.append(p) @@ -425,36 +501,41 @@ def write_ext(self, extent, data=None): yield simpy.AllOf(self.env, write_procs) - end_time = self.env.now # <----- end + end_time = self.env.now # <----- end if self.enable_recording: - self.write_latencies.append((end_time - start_time)/1000.0) + self.write_latencies.append((end_time - start_time) / 1000.0) self.waf["request"] += extent.lpn_count self.waf["actual"] += len(total_pages_to_write) - - write_timeline(self.conf, self.recorder, - op_id = op_id, op = 'write_ext', arg = extent.lpn_count, - start_time = start_time, end_time = end_time) + write_timeline( + self.conf, + self.recorder, + op_id=op_id, + op="write_ext", + arg=extent.lpn_count, + start_time=start_time, + end_time=end_time, + ) def _block_iter_of_extent(self, extent): block_start, _ = self.conf.page_to_block_off(extent.lpn_start) block_last, _ = self.conf.page_to_block_off(extent.last_lpn()) - return range(block_start, block_last + 1) + return list(range(block_start, block_last + 1)) def print_mappings(self, mappings): block = 1136 - for lpn, ppn in mappings.items(): + for lpn, ppn in list(mappings.items()): blk, _ = self.conf.page_to_block_off(lpn) # blk, _ = self.conf.page_to_block_off(ppn) if blk == block: - print(lpn, '->', ppn) + print((lpn, "->", ppn)) def _sub_ext_data(self, data, extent, sub_ext): start = sub_ext.lpn_start - extent.lpn_start count = sub_ext.lpn_count - sub_data = data[start:(start + count)] + sub_data = data[start : (start + count)] return sub_data def _write_ppns(self, ppns): @@ -462,30 +543,26 @@ def _write_ppns(self, ppns): The ppns in mappings is obtained from loggroup.next_ppns() """ # flash controller - yield self.env.process( - self.des_flash.rw_ppns(ppns, 'write', - tag="Unknown")) + yield self.env.process(self.des_flash.rw_ppns(ppns, "write", tag="Unknown")) - self.env.exit((0, 0)) + return (0, 0) def _read_ppns(self, ppns): """ The ppns in mappings is obtained from loggroup.next_ppns() """ # flash controller - yield self.env.process( - self.des_flash.rw_ppns(ppns, 'read', - tag="Unknown")) + yield self.env.process(self.des_flash.rw_ppns(ppns, "read", tag="Unknown")) def _update_log_mappings(self, mappings): """ The ppns in mappings must have been get by loggroup.next_ppns() """ - for lpn, ppn in mappings.items(): + for lpn, ppn in list(mappings.items()): self.log_mapping_table.add_mapping(lpn, ppn) def _remap_oob(self, new_mappings): - for lpn, new_ppn in new_mappings.items(): + for lpn, new_ppn in list(new_mappings.items()): found, old_ppn, loc = self.translator.lpn_to_ppn(lpn) if found == False: old_ppn = None @@ -496,16 +573,21 @@ def lba_discard(self, lpn): def discard_ext(self, extent): req_size = extent.lpn_count * self.conf.page_size - self.recorder.add_to_general_accumulater( - 'traffic', 'discard', req_size) + self.recorder.add_to_general_accumulater("traffic", "discard", req_size) self.discarded_bytes += req_size if self.discarded_bytes > self.pre_discarded_bytes + self.display_interval: - log_msg('Discarded (MB)', self.pre_discarded_bytes / MB, 'discarding', round(float(req_size) / MB, 2)) + log_msg( + "Discarded (MB)", + old_div(self.pre_discarded_bytes, MB), + "discarding", + round(float(req_size) / MB, 2), + ) sys.stdout.flush() self.pre_discarded_bytes = self.discarded_bytes - self.recorder.add_to_general_accumulater('traffic', 'discard', - extent.lpn_count*self.conf.page_size) + self.recorder.add_to_general_accumulater( + "traffic", "discard", extent.lpn_count * self.conf.page_size + ) extents = split_ext(self.conf.n_pages_per_block, extent) for logical_block_ext in extents: @@ -529,17 +611,24 @@ def post_processing(self): pass def clean(self, forced=False, merge=True): - self.pre_written_bytes_gc = self.written_bytes erased_pbns = [] validate_pages = [] - - num_valid = sum(self.metadata.bvc.counter[block] for block in self.metadata.bvc.counter) - num_all = float(sum(self.conf.n_pages_per_block for block in self.metadata.bvc.counter if self.metadata.bvc.counter[block] > 0)) + + num_valid = sum( + self.metadata.bvc.counter[block] for block in self.metadata.bvc.counter + ) + num_all = float( + sum( + self.conf.n_pages_per_block + for block in self.metadata.bvc.counter + if self.metadata.bvc.counter[block] > 0 + ) + ) # print(num_valid / num_all) if num_all == 0: return - if num_valid / num_all > 0.9: + if old_div(num_valid, num_all) > 0.9: return for block in self.metadata.bvc.counter: @@ -567,23 +656,35 @@ def clean(self, forced=False, merge=True): erase_procs = [] for erased_pbn in erased_pbns: - erase_procs += [self.env.process(self.des_flash.erase_pbn_extent(pbn_start = erased_pbn, pbn_count = 1, tag = None))] + erase_procs += [ + self.env.process( + self.des_flash.erase_pbn_extent( + pbn_start=erased_pbn, pbn_count=1, tag=None + ) + ) + ] write_procs = [] for ppn in all_ppns_to_write: p = self.env.process(self._write_ppns([ppn])) write_procs.append(p) - + start = self.env.now yield simpy.AllOf(self.env, erase_procs) erase_finished = self.env.now yield simpy.AllOf(self.env, write_procs) write_finished = self.env.now - print(len(validate_pages), erase_finished - start, write_finished - erase_finished) + print( + ( + len(validate_pages), + erase_finished - start, + write_finished - erase_finished, + ) + ) def is_wear_leveling_needed(self): factor, diff = self.block_pool.get_wear_status() - self.recorder.append_to_value_list('wear_diff', diff) + self.recorder.append_to_value_list("wear_diff", diff) # print 'ddddddddddddddddddddiiiiiiiiiiifffffffffff', diff return self.block_pool.need_wear_leveling() @@ -600,28 +701,30 @@ def snapshot_erasure_count_dist(self): dist = self.block_pool.get_erasure_count_dist() # print self.env.now # print dist - self.recorder.append_to_value_list('ftl_func_erasure_count_dist', - dist) + self.recorder.append_to_value_list("ftl_func_erasure_count_dist", dist) def snapshot_user_traffic(self): return - self.recorder.append_to_value_list('ftl_func_user_traffic', - {'timestamp': self.env.now/float(SEC), - 'write_traffic_size': self.written_bytes, - 'read_traffic_size': self.read_bytes, - 'discard_traffic_size': self.discarded_bytes, - }, - ) - -class PageValidityBitmap(object): + self.recorder.append_to_value_list( + "ftl_func_user_traffic", + { + "timestamp": self.env.now / float(SEC), + "write_traffic_size": self.written_bytes, + "read_traffic_size": self.read_bytes, + "discard_traffic_size": self.discarded_bytes, + }, + ) + + +class PageValidityBitmap: "Using one bit to represent state of a page" + "Erased state is recorded by BVC" VALID, INVALID = (1, 0) def __init__(self, conf, bvc): if not isinstance(conf, config.Config): - raise TypeError("conf is not conf.Config. it is {}". - format(type(conf).__name__)) + raise TypeError(f"conf is not conf.Config. it is {type(conf).__name__}") self.conf = conf self.bitmap = bitarray.bitarray(conf.total_num_pages()) @@ -658,7 +761,6 @@ def get_valid_pages(self, blocknum): valid_pages.append(pg) return valid_pages - def is_page_valid(self, pagenum): return self.bitmap[pagenum] == self.VALID @@ -670,19 +772,19 @@ def memory(self): return round(len(self.bitmap) // 8) -class BlockValidityCounter(object): +class BlockValidityCounter: """ - Timestamp table PPN -> timestamp - Here are the rules: - 1. only programming a PPN updates the timestamp of PPN - if the content is new from FS, timestamp is the timestamp of the - LPN - if the content is copied from other flash block, timestamp is the - same as the previous ppn - 2. discarding, and reading a ppn does not change it. - 3. erasing a block will remove all the timestamps of the block - 4. so cur_timestamp can only be advanced by LBA operations - Table PPN -> valid pages + Timestamp table PPN -> timestamp + Here are the rules: + 1. only programming a PPN updates the timestamp of PPN + if the content is new from FS, timestamp is the timestamp of the + LPN + if the content is copied from other flash block, timestamp is the + same as the previous ppn + 2. discarding, and reading a ppn does not change it. + 3. erasing a block will remove all the timestamps of the block + 4. so cur_timestamp can only be advanced by LBA operations + Table PPN -> valid pages """ def __init__(self, conf): @@ -690,10 +792,12 @@ def __init__(self, conf): self.last_inv_time_of_block = {} self.timestamp_table = {} self.cur_timestamp = 0 - self.counter = defaultdict(lambda:0) - self.free_block_list = [self.conf.n_blocks_per_channel * channel + block - for block in range(self.conf.n_blocks_per_channel) - for channel in range(self.conf.n_channels_per_dev)] + self.counter = defaultdict(lambda: 0) + self.free_block_list = [ + self.conf.n_blocks_per_channel * channel + block + for block in range(self.conf.n_blocks_per_channel) + for channel in range(self.conf.n_channels_per_dev) + ] def _incr_timestamp(self): """ @@ -731,7 +835,7 @@ def memory(self): return -class OutOfBandAreas(object): +class OutOfBandAreas: """ Jinghan: We use OOB to store the p2l mapping for each page and all PPNs within the same segment. Since we do not update the segments in-place, we also do not have to update the OOB data. OOB impl: Dict> @@ -741,18 +845,18 @@ class OutOfBandAreas(object): def __init__(self, conf, gamma): self.oob_data = defaultdict(None) self.gamma = gamma - self.per_page_size = conf['flash_config']["oob_size_per_page"] + self.per_page_size = conf["flash_config"]["oob_size_per_page"] self.p2l_entry_size = 4 - self.num_p2l_entries = min(2*self.gamma, self.per_page_size / 4) + self.num_p2l_entries = min(2 * self.gamma, old_div(self.per_page_size, 4)) # entries: List> def set_oob(self, source_page, entries): - assert(entries == None or isinstance(entries, list)) + assert entries == None or isinstance(entries, list) if entries == None: self.oob_data[source_page] = entries else: - rev_map = dict() #bidict.bidict() - for (lpn, ppn) in entries: + rev_map = dict() # bidict.bidict() + for lpn, ppn in entries: rev_map[lpn] = ppn self.oob_data[source_page] = rev_map @@ -760,20 +864,18 @@ def ppn_to_lpn(self, ppn, source_page=None): raise NotImplementedError if not source_page: source_page = ppn - assert( - source_page in self.oob_data and ppn in self.oob_data[source_page]) + assert source_page in self.oob_data and ppn in self.oob_data[source_page] return self.oob_data[source_page][ppn] def lpn_to_ppn(self, lpn, source_page): if source_page in self.oob_data: rev_map = self.oob_data[source_page] if rev_map: - return rev_map.get(lpn) #.inv.get(lpn) + return rev_map.get(lpn) # .inv.get(lpn) return None - -class OutOfBandAreasMemOpt(object): +class OutOfBandAreasMemOpt: """ Memory optimized version Jinghan: We use OOB to store the p2l mapping for each page and all PPNs within the same segment. Since we do not update the segments in-place, we also do not have to update the OOB data. @@ -783,7 +885,9 @@ class OutOfBandAreasMemOpt(object): def __init__(self, conf, gamma, reference_mapping_table): self.gamma = gamma - self.num_p2l_entries = min(2*self.gamma, conf['flash_config']["oob_size_per_page"] / 4) + self.num_p2l_entries = min( + 2 * self.gamma, old_div(conf["flash_config"]["oob_size_per_page"], 4) + ) self.reference_mapping_table = reference_mapping_table # entries: List> @@ -800,8 +904,7 @@ def lpn_to_ppn(self, lpn, source_page): return None - -class FlashMetadata(object): +class FlashMetadata: def __init__(self, confobj, counter): self.conf = confobj self.counter = counter @@ -811,8 +914,8 @@ def __init__(self, confobj, counter): self.total_pages = self.flash_num_blocks * self.flash_npage_per_block # mapping table - self.gamma = self.conf['gamma'] - #self.mapping_table = LogPLR(frame_no=0, gamma=self.gamma) + self.gamma = self.conf["gamma"] + # self.mapping_table = LogPLR(frame_no=0, gamma=self.gamma) self.mapping_table = FrameLogPLR(confobj, self, counter, gamma=self.gamma) Segment.PAGE_PER_BLOCK = self.flash_npage_per_block @@ -823,7 +926,9 @@ def __init__(self, confobj, counter): self.pvb = PageValidityBitmap(confobj, self.bvc) # ppn -> lpn mapping stored in OOB # self.oob = OutOfBandAreas(confobj, gamma=self.gamma) - self.oob = OutOfBandAreasMemOpt(confobj, self.gamma, self.reference_mapping_table) + self.oob = OutOfBandAreasMemOpt( + confobj, self.gamma, self.reference_mapping_table + ) self.last_oob_page = [] self.next_free_ppn = 0 # WAF @@ -832,7 +937,6 @@ def __init__(self, confobj, counter): # counters self.levels = defaultdict(int) - ############# Flash read related ############ def ppn_to_lpn(self, ppn, source_page=None): @@ -842,7 +946,9 @@ def ppn_to_lpn(self, ppn, source_page=None): def lpn_to_ppn(self, lpn): real_ppn = None - results, num_lookup, pages_to_write, pages_to_read = self.mapping_table.lookup(lpn, first=True) + results, num_lookup, pages_to_write, pages_to_read = self.mapping_table.lookup( + lpn, first=True + ) self.levels[num_lookup] += 1 # if len(results) == 0: # return None, None @@ -851,7 +957,7 @@ def lpn_to_ppn(self, lpn): if accurate: # pages_to_read += [ppn] real_ppn = ppn - + else: actual = self.oob.lpn_to_ppn(lpn, source_page=ppn) if actual: @@ -874,13 +980,23 @@ def lpn_to_ppn(self, lpn): # entry not exists; continue to search neighbor block (ppn is predicted to the wrong block) else: pages_to_read += [ppn] - if ppn % self.conf.n_pages_per_block < self.conf.n_pages_per_block / 2.0: - ppn = int(ppn / self.conf.n_pages_per_block) * self.conf.n_pages_per_block - 1 + if ( + ppn % self.conf.n_pages_per_block + < self.conf.n_pages_per_block / 2.0 + ): + ppn = ( + int(old_div(ppn, self.conf.n_pages_per_block)) + * self.conf.n_pages_per_block + - 1 + ) else: - ppn = int(ppn / self.conf.n_pages_per_block + 1) * self.conf.n_pages_per_block + ppn = ( + int(old_div(ppn, self.conf.n_pages_per_block) + 1) + * self.conf.n_pages_per_block + ) actual = self.oob.lpn_to_ppn(lpn, source_page=ppn) try: - assert(actual) + assert actual except: # if this assert fails, it is possible that prediction is out of oob range, but still in the same block self.validation(lpn, None) @@ -894,28 +1010,30 @@ def lpn_to_ppn(self, lpn): del results return real_ppn, pages_to_write, pages_to_read - - ''' + """ @return Dict - ''' + """ + def update(self, extents): mappings = dict() pages_to_read = [] pages_to_write = [] for i in range(0, len(extents), self.conf.n_pages_per_block): - submap, subpages_to_read, subpages_to_write = self.update_block(extents[i:i+self.conf.n_pages_per_block]) + submap, subpages_to_read, subpages_to_write = self.update_block( + extents[i : i + self.conf.n_pages_per_block] + ) mappings.update(submap) pages_to_read.extend(subpages_to_read) pages_to_write.extend(subpages_to_write) - return mappings, list(set(pages_to_read)), list(set(pages_to_write)) - ''' + """ @return Dict - ''' + """ + def update_block(self, extents): - assert(len(extents) <= self.conf.n_pages_per_block) + assert len(extents) <= self.conf.n_pages_per_block entries = [] pages_to_read = [] pages_to_write = [] @@ -930,7 +1048,7 @@ def update_block(self, extents): entry = (lpn, next_free_ppn + i) entries.append(entry) - #TODO: additional flash reads; make this async; write to bitmap + # TODO: additional flash reads; make this async; write to bitmap self.pvb.validate_block(next_free_block) # if self.conf["dry_run"]: # for (lpn, ppn) in entries: @@ -952,12 +1070,14 @@ def update_block(self, extents): # self.mapping_table.gc(next_free_block) ## update mapping table - mapping_pages_to_write, mapping_pages_to_read = self.mapping_table.update(entries, next_free_block) + mapping_pages_to_write, mapping_pages_to_read = self.mapping_table.update( + entries, next_free_block + ) pages_to_write += mapping_pages_to_write pages_to_read += mapping_pages_to_read # update reference mapping table - for (lpn, ppn) in entries: + for lpn, ppn in entries: self.reference_mapping_table.set(lpn, ppn) # update oob @@ -967,11 +1087,10 @@ def update_block(self, extents): # print(max(0, i - self.gamma - 1), i + self.gamma + 2) upper = int(i + max_gamma + 2) lower = int(i - max_gamma - 1) - self.oob.set_oob(ppn, entries[max(0,lower):upper]) + self.oob.set_oob(ppn, entries[max(0, lower) : upper]) return dict(entries), pages_to_read, pages_to_write - ############# GC related ############ def erase_block(self, flash_block): @@ -1017,24 +1136,27 @@ def data_page_move(self, lpn, old_ppn, new_ppn): def validation(self, lpn, ppn): try: - assert(ppn == self.reference_mapping_table.get(lpn)) + assert ppn == self.reference_mapping_table.get(lpn) except: # print(self.mapping_table.runs[0]) - results, num_lookup, pages_to_write, pages_to_read = self.mapping_table.lookup(lpn, first=False) - print("lpn:", lpn) - print("reference ppn:", - self.reference_mapping_table.get_all(lpn)) - print("learned ppn:", ppn) - print("all ppns in the tree:", results) + results, num_lookup, pages_to_write, pages_to_read = ( + self.mapping_table.lookup(lpn, first=False) + ) + print(("lpn:", lpn)) + print(("reference ppn:", self.reference_mapping_table.get_all(lpn))) + print(("learned ppn:", ppn)) + print(("all ppns in the tree:", results)) for ppn, accurate, seg in results: if seg: - print("learned segment:", seg.full_str()) - print("oob data:", str(self.oob.oob_data[ppn])) + print(("learned segment:", seg.full_str())) + print(("oob data:", str(self.oob.oob_data[ppn]))) exit(0) + # Learning-related components -class SimpleSegment(): + +class SimpleSegment: def __init__(self, k, b, x1, x2): self.b = b self.k = k @@ -1042,19 +1164,21 @@ def __init__(self, k, b, x1, x2): self.x2 = x2 def __str__(self): - return "(%d, %.2f, %d, %d)" % (self.b, self.k, self.x1, self.x2) + return f"({self.b}, {self.k:.2f}, {self.x1}, {self.x2})" def __repr__(self): return str(self) def get_y(self, x): - predict = int(round(x*self.k + self.b)) + predict = int(round(x * self.k + self.b)) return predict @staticmethod def intersection(s1, s2): - p = (float(s2.b - s1.b) / (s1.k - s2.k), - float(s1.k * s2.b - s2.k * s1.b) / (s1.k - s2.k)) + p = ( + float(s2.b - s1.b) / (s1.k - s2.k), + float(s1.k * s2.b - s2.k * s1.b) / (s1.k - s2.k), + ) return p @staticmethod @@ -1080,7 +1204,7 @@ def frompoints(p1, p2): return SimpleSegment(k, b, p1[0], p2[0]) -class Segment(): +class Segment: FPR = 0.01 PAGE_PER_BLOCK = 256 BITMAP = True @@ -1099,7 +1223,7 @@ def __init__(self, k, b, x1, x2, points=None): if not consecutive: if Segment.BITMAP: - self.filter = bitarray.bitarray(self.x2 - self.x1 + 1) + self.filter = bitarray.bitarray(self.x2 - self.x1 + 1) self.filter.setall(0) for pt in points: self.filter[pt[0] - self.x1] = 1 @@ -1107,20 +1231,22 @@ def __init__(self, k, b, x1, x2, points=None): self.filter = BloomFilter(len(points), Segment.FPR) for pt in points: self.filter.add(pt[0]) - - if LPN_TO_DEBUG in zip(*points)[0]: + + if LPN_TO_DEBUG in list(zip(*points))[0]: log_msg("new seg", self) def __str__(self): - return "%.4f, %d, [%d, %d], memory: %dB, accuracy: %s, bitmap: %s" \ - % (self.k, self.b, self.x1, self.x2, self.memory, self.accurate, self.filter) + return f"{self.k:.4f}, {self.b}, [{self.x1}, {self.x2}], memory: {self.memory}B, accuracy: {self.accurate}, bitmap: {self.filter}" def __repr__(self): return str(self) - return "(%d, %.4f, %d, %d, %s)" % (self.b, self.k, self.x1, self.x2, self.accurate) + return f"({self.b}, {self.k:.4f}, {self.x1}, {self.x2}, {self.accurate})" def full_str(self): - return "(%d, %.4f, %d, %d, %s) " % (self.b, self.k, self.x1, self.x2, self.accurate) + str(self._points) + return ( + f"({self.b}, {self.k:.4f}, {self.x1}, {self.x2}, {self.accurate}) " + + str(self._points) + ) def is_valid(self, x): if not (self.x1 <= x and x <= self.x2): @@ -1135,7 +1261,7 @@ def is_valid(self, x): def get_y(self, x, check=True): if not check or self.is_valid(x): - predict = int(round(x*self.k + self.b)) + predict = int(round(x * self.k + self.b)) # lowbound = self.blocknum * Segment.PAGE_PER_BLOCK # upbound = (self.blocknum + 1) * Segment.PAGE_PER_BLOCK - 1 # return max(min(predict, upbound), lowbound) @@ -1150,7 +1276,7 @@ def check_properties(self, points): is_accruate = False # if abs(self.get_y(pt[0], check=False) - pt[1]) > 5: # print(self, self.get_y(pt[0], check=False), pt[1]) - if len(np.unique(np.diff(zip(*points)[0]))) > 1: + if len(np.unique(np.diff(list(zip(*points))[0]))) > 1: is_consecutive = False return is_accruate, is_consecutive @@ -1161,7 +1287,6 @@ def overlaps(self, other): def overlaps_with_range(self, x1, x2): return min(self.x2, x2) - max(self.x1, x1) >= 0 - # check whether two segments can be put into the same level # if they can be put in the same level, return False # (here we assume other is older than self) @@ -1207,44 +1332,43 @@ def merge(new, old): else: return new, old, False - @staticmethod def bitwise_merge(new, old): lo, hi = min(old.x1, new.x1), max(old.x2, new.x2) - new_bm = bitarray.bitarray(hi-lo+1) - old_bm = bitarray.bitarray(hi-lo+1) + new_bm = bitarray.bitarray(hi - lo + 1) + old_bm = bitarray.bitarray(hi - lo + 1) new_bm.setall(0) old_bm.setall(0) if new.consecutive: - new_bm[new.x1-lo : new.x2-lo+1 : new.rec_k] = 1 + new_bm[new.x1 - lo : new.x2 - lo + 1 : new.rec_k] = 1 elif Segment.BITMAP: - new_bm[new.x1-lo : new.x2-lo+1] = new.filter - + new_bm[new.x1 - lo : new.x2 - lo + 1] = new.filter + if old.consecutive: - old_bm[old.x1-lo : old.x2-lo+1 : old.rec_k] = 1 + old_bm[old.x1 - lo : old.x2 - lo + 1 : old.rec_k] = 1 elif Segment.BITMAP: - old_bm[old.x1-lo : old.x2-lo+1] = old.filter - + old_bm[old.x1 - lo : old.x2 - lo + 1] = old.filter + try: old_bm = old_bm & (~new_bm) except: - print(lo, hi) - print(old, new) - print(old._points, new._points) - print(old_bm, new_bm) + print((lo, hi)) + print((old, new)) + print((old._points, new._points)) + print((old_bm, new_bm)) exit(0) first_valid = old_bm.find(1) if first_valid == -1: return new, None last_valid = bitarray.util.rindex(old_bm, 1) - old.x1 = first_valid + lo + old.x1 = first_valid + lo old.x2 = last_valid + lo if not old.consecutive and Segment.BITMAP: - old.filter = old_bm[first_valid : last_valid+1] - assert(old.filter != None) + old.filter = old_bm[first_valid : last_valid + 1] + assert old.filter != None # TODO: re-check accuracy and consecutive @@ -1260,39 +1384,61 @@ def mergable(self): @property def length(self): - return (self.x2-self.x1) // self.rec_k + 1 + return (self.x2 - self.x1) // self.rec_k + 1 @property def memory(self): if self.x1 == self.x2: - return SUBLPN_BYTES + PPN_BYTES # + FLOAT16_BYTES + LENGTH_BYTES + return SUBLPN_BYTES + PPN_BYTES # + FLOAT16_BYTES + LENGTH_BYTES else: if self.consecutive: - return SUBLPN_BYTES + PPN_BYTES + FLOAT16_BYTES + LENGTH_BYTES # 4+4+2+1 + return ( + SUBLPN_BYTES + PPN_BYTES + FLOAT16_BYTES + LENGTH_BYTES + ) # 4+4+2+1 else: if Segment.BITMAP: # filter_size = len(self.filter) / 8.0 ones = len([e for e in self.filter if e]) - non_consec_ones = len([i for i in range(len(self.filter)) if i > 0 and i < len(self.filter)-1 and self.filter[i] is not self.filter[i-1]]) + non_consec_ones = len( + [ + i + for i in range(len(self.filter)) + if i > 0 + and i < len(self.filter) - 1 + and self.filter[i] is not self.filter[i - 1] + ] + ) # print(self.filter, [i for i in range(len(self.filter)) if i > 0 and i < len(self.filter)-1 and self.filter[i] is not self.filter[i-1]]) # zeros = len(self.filter) - ones # sparse_encoding_size = min(ones, zeros) * 1 + LENGTH_BYTES - return SUBLPN_BYTES + PPN_BYTES + FLOAT16_BYTES + non_consec_ones * 1 + LENGTH_BYTES - #return SUBLPN_BYTES + PPN_BYTES + FLOAT16_BYTES + min(filter_size, sparse_encoding_size) + return ( + SUBLPN_BYTES + + PPN_BYTES + + FLOAT16_BYTES + + non_consec_ones * 1 + + LENGTH_BYTES + ) + # return SUBLPN_BYTES + PPN_BYTES + FLOAT16_BYTES + min(filter_size, sparse_encoding_size) else: - return SUBLPN_BYTES + PPN_BYTES + FLOAT16_BYTES + round(self.filter.bit_array_size / 8.0) - + return ( + SUBLPN_BYTES + + PPN_BYTES + + FLOAT16_BYTES + + round(self.filter.bit_array_size / 8.0) + ) + @property def rec_k(self): - return int(round(1.0/self.k)) + return int(round(1.0 / self.k)) + @property def blocknum(self): mid = (self.x2 + self.x1) / 2.0 predict = int(round(mid * self.k + self.b)) - return int(predict / Segment.PAGE_PER_BLOCK) + return int(old_div(predict, Segment.PAGE_PER_BLOCK)) -class PLR(): +class PLR: FIRST = "first" SECOND = "second" READY = "ready" @@ -1324,7 +1470,7 @@ def learn(self, points): self.segments.append(seg) if rej != None: rejs.append(rej) - + seg = self.build_segment() if seg != None: self.segments.append(seg) @@ -1345,16 +1491,18 @@ def build_segment(self): if self.state == PLR.FIRST: seg = None elif self.state == PLR.SECOND: - seg = Segment(1, self.s0[1] - self.s0[0], self.s0[0], self.s0[0], - points=self.points) + seg = Segment( + 1, self.s0[1] - self.s0[0], self.s0[0], self.s0[0], points=self.points + ) elif self.state == PLR.READY: avg_slope = np.float16((self.rho_lower.k + self.rho_upper.k) / 2.0) # avg_slope = (self.rho_lower.k + self.rho_upper.k) / 2.0 # rec_k = round(1.0/avg_slope) intercept = -self.sint[0] * avg_slope + self.sint[1] - seg = Segment(avg_slope, intercept, self.s0[0], self.s1[0], - points=self.points) - + seg = Segment( + avg_slope, intercept, self.s0[0], self.s1[0], points=self.points + ) + return seg def process(self, point): @@ -1373,16 +1521,23 @@ def process(self, point): else: self.s1 = point self.state = PLR.READY - self.rho_lower = SimpleSegment.frompoints(SimpleSegment.get_upper_bound(self.s0, self.gamma), - SimpleSegment.get_lower_bound(self.s1, self.gamma)) - self.rho_upper = SimpleSegment.frompoints(SimpleSegment.get_lower_bound(self.s0, self.gamma), - SimpleSegment.get_upper_bound(self.s1, self.gamma)) - self.sint = SimpleSegment.intersection( - self.rho_upper, self.rho_lower) + self.rho_lower = SimpleSegment.frompoints( + SimpleSegment.get_upper_bound(self.s0, self.gamma), + SimpleSegment.get_lower_bound(self.s1, self.gamma), + ) + self.rho_upper = SimpleSegment.frompoints( + SimpleSegment.get_lower_bound(self.s0, self.gamma), + SimpleSegment.get_upper_bound(self.s1, self.gamma), + ) + self.sint = SimpleSegment.intersection(self.rho_upper, self.rho_lower) self.state = PLR.READY elif self.state == PLR.READY: - if not SimpleSegment.is_above(point, self.rho_lower) or not SimpleSegment.is_below(point, self.rho_upper) or self.should_stop(point): + if ( + not SimpleSegment.is_above(point, self.rho_lower) + or not SimpleSegment.is_below(point, self.rho_upper) + or self.should_stop(point) + ): prev_segment = self.build_segment() self.s0 = point self.state = PLR.SECOND @@ -1400,7 +1555,6 @@ def process(self, point): # self.sint = SimpleSegment.intersection( # self.rho_upper, self.rho_lower) - self.points.append(point) return (prev_segment, None) @@ -1419,7 +1573,7 @@ def __len__(self): return len(self.it) -class LogPLR(): +class LogPLR: def __init__(self, gamma, frame_no): self.plr = PLR(gamma) # one run is one level of segments with non-overlapping intervals @@ -1427,7 +1581,7 @@ def __init__(self, gamma, frame_no): self.frame_no = frame_no # mapping from block to segments # self.block_map = defaultdict(list) - + def update(self, entries, blocknum): # make sure no same LPNs exist in the entries sorted_entries = sorted(entries) @@ -1441,7 +1595,7 @@ def update(self, entries, blocknum): return [], [] def merge(self, old_plr): - assert(self.frame_no == old_plr.frame_no) + assert self.frame_no == old_plr.frame_no self.runs.extend(old_plr.runs) # bottleneck @@ -1472,7 +1626,7 @@ def lookup(self, LPA, first=True): for level in sorted(empty_levels, reverse=True): del self.runs[level] - + return results, lookup, [], [] def lookup_range(self, start, end): @@ -1495,8 +1649,6 @@ def lookup_range(self, start, end): break return results - - # recursively add segments to each level # bottleneck @@ -1508,20 +1660,18 @@ def add_segments(self, level, segments, recursive=True): conflicts = [] for new_seg in segments: if new_seg.get_y(LPN_TO_DEBUG): - log_msg("%s added to run %d" % (new_seg, level)) + log_msg(f"{new_seg} added to run {level}") if len(run) == 0: run.append(new_seg) continue - # run[index].x1 >= new_seg.x1 - index = bisect_left(KeyWrapper( - run, key=lambda seg: seg.x1), new_seg.x1) + index = bisect_left(KeyWrapper(run, key=lambda seg: seg.x1), new_seg.x1) run.insert(index, new_seg) overlaps = [] if index != 0: - overlaps.append((index-1, run[index-1])) - for i in range(index+1, len(run)): + overlaps.append((index - 1, run[index - 1])) + for i in range(index + 1, len(run)): if run[i].x1 > new_seg.x2: break overlaps.append((i, run[i])) @@ -1530,18 +1680,18 @@ def add_segments(self, level, segments, recursive=True): for index, old_seg in overlaps: to_print = old_seg.get_y(LPN_TO_DEBUG) if to_print: - log_msg("%s tries to merge with %s" % (new_seg, old_seg)) + log_msg(f"{new_seg} tries to merge with {old_seg}") new_seg, old_seg, same_level = Segment.merge(new_seg, old_seg) if not old_seg: indices_to_delete.append(index) if to_print: - log_msg("%s removed old seg" % (new_seg)) + log_msg(f"{new_seg} removed old seg") elif not same_level: conflicts.append(old_seg) indices_to_delete.append(index) if to_print: - log_msg("%s -> %s" % (new_seg, old_seg)) + log_msg(f"{new_seg} -> {old_seg}") for index in sorted(indices_to_delete, reverse=True): if run[index].get_y(LPN_TO_DEBUG): log_msg("removed old seg", (run[index])) @@ -1549,18 +1699,15 @@ def add_segments(self, level, segments, recursive=True): if recursive: if len(conflicts) > 0: - self.add_segments(level+1, conflicts) + self.add_segments(level + 1, conflicts) else: if len(conflicts) > 0: - self.runs.insert(level+1, conflicts) - - - + self.runs.insert(level + 1, conflicts) def __str__(self): repr = "" for level in range(len(self.runs)): - repr += "== level %d ==\n %s \n" % (level, str(self.runs[level])) + repr += f"== level {level} ==\n {str(self.runs[level])} \n" return repr @property @@ -1569,7 +1716,10 @@ def segments(self): @property def memory(self): - return sum([seg.memory for i, run in enumerate(self.runs) for seg in run]) + LPN_BYTES + return ( + sum([seg.memory for i, run in enumerate(self.runs) for seg in run]) + + LPN_BYTES + ) @property def levels(self): @@ -1577,7 +1727,7 @@ def levels(self): # now we only use compact def gc(self, blocknum): - return + return for seg in self.block_map[blocknum]: for run in reversed(self.runs): index = bisect_left(KeyWrapper(run, key=lambda _: _.x1), seg.x1) @@ -1586,7 +1736,7 @@ def gc(self, blocknum): log_msg("gc removed old seg", (seg)) run.remove(seg) break - + self.block_map[blocknum] = [] def promote(self): @@ -1597,16 +1747,18 @@ def promote(self): return layers = self.runs[:] - for i in range(1,len(layers)): + for i in range(1, len(layers)): lower_layer = layers[i] for old_seg in reversed(lower_layer): promoted_layer = i promoted_index = None - for j in reversed(range(0,i)): + for j in reversed(list(range(0, i))): upper_layer = layers[j] - index = bisect_left(KeyWrapper(upper_layer, key=lambda seg: seg.x1), old_seg.x1) + index = bisect_left( + KeyWrapper(upper_layer, key=lambda seg: seg.x1), old_seg.x1 + ) overlaps = False - for k in range(max(0, index-1), len(upper_layer)): + for k in range(max(0, index - 1), len(upper_layer)): if upper_layer[k].x1 > old_seg.x2: break if upper_layer[k].overlaps(old_seg): @@ -1617,15 +1769,15 @@ def promote(self): else: promoted_layer = j promoted_index = index - + if promoted_layer < i: # if self.frame_no == 3075: # log_msg("Promote %s to level %d" % (old_seg, promoted_layer)) layers[promoted_layer].insert(promoted_index, old_seg) lower_layer.remove(old_seg) - + self.runs = [run for run in self.runs if len(run) != 0] - + def compact(self, promote=False): if len(self.runs) == 0: return @@ -1641,20 +1793,22 @@ def compact(self, promote=False): if promote: self.promote() - return#len(self.segments), relearn_dict + return # len(self.segments), relearn_dict def compact_range(self, start, end): results = self.lookup_range(start, end) # relearn = dict() - for upper_layer, new_segs in results.items(): - for lower_layer, old_segs in results.items(): + for upper_layer, new_segs in list(results.items()): + for lower_layer, old_segs in list(results.items()): if upper_layer < lower_layer: for new_seg in new_segs: for old_seg in old_segs: # if old_seg not in relearn and old_seg.x1 < new_seg.x1 and new_seg.x2 < old_seg.x2: # if not old_seg.consecutive: # relearn[old_seg] = sum(old_seg.filter) - 2 - new_seg, updated_old_seg, same_level = Segment.merge(new_seg, old_seg) + new_seg, updated_old_seg, same_level = Segment.merge( + new_seg, old_seg + ) if not updated_old_seg: self.runs[lower_layer].remove(old_seg) results[lower_layer].remove(old_seg) @@ -1665,7 +1819,10 @@ def compact_range(self, start, end): # Each LogPLR is responsible for one range class FrameLogPLR: ON_FLASH, CLEAN, DIRTY = "ON_FLASH", "CLEAN", "DIRTY" - def __init__(self, conf, metadata, counter, gamma, max_size=1*1024**2, frame_length=256): + + def __init__( + self, conf, metadata, counter, gamma, max_size=1 * 1024**2, frame_length=256 + ): global SUBLPN_BYTES SUBLPN_BYTES = 1 self.conf = conf @@ -1674,14 +1831,14 @@ def __init__(self, conf, metadata, counter, gamma, max_size=1*1024**2, frame_len self.gamma = gamma self.frame_length = frame_length self.frames = LRUCache() - self.max_size = self.conf['mapping_cache_bytes'] + self.max_size = self.conf["mapping_cache_bytes"] # assert(self.max_size >= self.conf.page_size) # internal_type = "sftl" - if self.conf['internal_ftl_type'] == "sftl": + if self.conf["internal_ftl_type"] == "sftl": self.type = "sftl" self.frame_length = 1024 - elif self.conf['internal_ftl_type'] == "dftldes": + elif self.conf["internal_ftl_type"] == "dftldes": self.type = "dftldes" self.frame_length = 1024 else: @@ -1701,13 +1858,13 @@ def __init__(self, conf, metadata, counter, gamma, max_size=1*1024**2, frame_len def create_frame(self, frame_no): if self.type == "learnedftl": return LogPLR(self.gamma, frame_no) - + elif self.type == "sftl": return SFTLPage(frame_no, self.frame_length) elif self.type == "dftldes": return DFTLPage(frame_no) - + else: raise NotImplementedError @@ -1722,7 +1879,7 @@ def update(self, entries, blocknum): pages_to_write, pages_to_read = [], [] split_entries = FrameLogPLR.split_into_frame(self.frame_length, entries) frame_nos = [] - for frame_no, entries in split_entries.items(): + for frame_no, entries in list(split_entries.items()): frame_nos += [frame_no] if frame_no not in self.frames: self.frames[frame_no] = self.create_frame(frame_no) @@ -1737,7 +1894,7 @@ def update(self, entries, blocknum): self.dirty[frame_no] = True self.change_size_of_frame(frame_no, self.frames[frame_no].memory) - + if self.should_flush(): pages_to_write, pages_to_read = self.flush() @@ -1755,7 +1912,7 @@ def lookup(self, lpn, first=True): self.frames.move_to_head(frame_no, frame) # log_msg("Move to head", frame_no) results, lookup, _, _ = frame.lookup(lpn, first) - + if len(results) != 0: self.counter["mapping_table_read_hit"] += 1 else: @@ -1775,7 +1932,7 @@ def lookup(self, lpn, first=True): self.dirty[frame_no] = False self.frames[frame_no] = frame del self.frame_on_flash[frame_no] - + self.frames.move_to_head(frame_no, self.frames[frame_no]) # log_msg("Move to head", frame_no) # log_msg(self.frames.keys()) @@ -1794,7 +1951,7 @@ def lookup(self, lpn, first=True): # if should_print: # print("after",self.memory) - + return results, lookup, pages_to_write, pages_to_read # gc is currently replaced with compaction @@ -1805,17 +1962,17 @@ def compact(self, promote=False, frame_nos=None): # relearn_dict = dict() # total_segments = 0 if not frame_nos: - for frame_no, frame in self.frames.items(): + for frame_no, frame in list(self.frames.items()): frame.compact(promote=promote) # total_segments += total_segs # relearn_dict.update(relearn) - self.change_size_of_frame(frame_no, frame.memory) + self.change_size_of_frame(frame_no, frame.memory) else: for frame_no in frame_nos: frame = self.frames[frame_no] frame.compact(promote=promote) - self.change_size_of_frame(frame_no, frame.memory) - + self.change_size_of_frame(frame_no, frame.memory) + # appox_segments = 0 # appox_segments_length = [] # for seg in self.segments: @@ -1826,10 +1983,9 @@ def compact(self, promote=False, frame_nos=None): # print(total_segments, len(relearn_dict), sum(relearn_dict.values())) def promote(self): - for frame in self.frames.values(): + for frame in list(self.frames.values()): frame.promote() - def should_flush(self): if self.memory > self.max_size: # n_pages = (self.memory - self.max_size) /self.conf.page_size @@ -1842,27 +1998,29 @@ def allocate_ppn_for_frame(self, frame_no): self.current_trans_block = self.metadata.bvc.next_free_block() self.metadata.pvb.validate_block(self.current_trans_block) - next_free_ppn = self.conf.n_pages_per_block * self.current_trans_block + self.current_trans_page_offset + next_free_ppn = ( + self.conf.n_pages_per_block * self.current_trans_block + + self.current_trans_page_offset + ) self.current_trans_page_offset += 1 if frame_no not in self.GTD: old_ppn = None self.GTD[frame_no] = next_free_ppn new_ppn = self.GTD[frame_no] - + else: old_ppn = self.GTD[frame_no] self.GTD[frame_no] = next_free_ppn new_ppn = self.GTD[frame_no] return new_ppn, old_ppn - + def flush(self): evicted_frames = [] pages_to_read = [] pages_to_write = [] - original_memory = self.memory # assert(self.memory == sum([frame.memory for frame in self.frames.values()])) while original_memory > self.max_size: @@ -1870,7 +2028,7 @@ def flush(self): # log_msg(frame_no, "evicted") freed_mem = self.memory_counter[frame_no] original_memory -= freed_mem - self.change_size_of_frame(frame_no, 0) + self.change_size_of_frame(frame_no, 0) evicted_frames.append(frame_no) new_ppn, old_ppn = self.allocate_ppn_for_frame(frame_no) @@ -1886,7 +2044,6 @@ def flush(self): evict_frame.merge(old_frame) self.frame_on_flash[frame_no] = evict_frame - # log_msg("%.2f miss ratio, %s evicted, %d memory, %d in cache, %d on flash" % (self.misses / float(self.misses + self.hits), evicted_frames, self.memory, len(self.frames), len(self.frame_on_flash))) # log_msg("%d miss, %d memory flushed, %s evicted, %d memory, %d in cache, %d on flash" % (self.misses, original_memory - self.memory, evicted_frames, self.memory, len(self.frames), len(self.frame_on_flash))) @@ -1897,8 +2054,8 @@ def change_size_of_frame(self, frame_no, new_mem): if frame_no in self.memory_counter: old_mem = self.memory_counter[frame_no] self.memory_counter[frame_no] = new_mem - self.total_memory += (new_mem - old_mem) - + self.total_memory += new_mem - old_mem + # TODO: bottleneck @property def memory(self): @@ -1908,30 +2065,32 @@ def memory(self): @property def segments(self): - return [seg for frame in self.frames.values() for seg in frame.segments] + return [seg for frame in list(self.frames.values()) for seg in frame.segments] @property def levels(self): if len(self.frames) == 0: return 0 - return max([frame.levels for frame in self.frames.values()]) + return max([frame.levels for frame in list(self.frames.values())]) @property def dist_levels(self): if len(self.frames) == 0: return 0, 0 - dist = [frame.levels for frame in self.frames.values() if frame.levels != 0] + dist = [ + frame.levels for frame in list(self.frames.values()) if frame.levels != 0 + ] return dist - -class PFTL(object): + +class PFTL: def __init__(self): # store update history for verification purpose self.mapping_table = defaultdict(list) def set(self, lpn, ppn): self.mapping_table[lpn].append(ppn) - + def get(self, lpn): # force check; since we are using defaultdict we don't want to create empty entry if lpn not in self.mapping_table: @@ -1950,7 +2109,6 @@ def memory(self): return len(self.mapping_table) * (PPN_BYTES + LPN_BYTES) - def split_ext(extent): if extent.lpn_count == 0: return None @@ -1962,9 +2120,15 @@ def split_ext(extent): return exts + # no need to dump the entire timeline if storage space is limited def write_timeline(conf, recorder, op_id, op, arg, start_time, end_time): return - recorder.write_file('timeline.txt', - op_id = op_id, op = op, arg = arg, - start_time = start_time, end_time = end_time) \ No newline at end of file + recorder.write_file( + "timeline.txt", + op_id=op_id, + op=op, + arg=arg, + start_time=start_time, + end_time=end_time, + ) diff --git a/wiscsee/wiscsim/lrulist.py b/wiscsee/wiscsim/lrulist.py index b2cdaf2..d020c55 100644 --- a/wiscsee/wiscsim/lrulist.py +++ b/wiscsee/wiscsim/lrulist.py @@ -1,29 +1,35 @@ import collections -class Node(object): - def __init__(self, key = None, value = None, empty = False): + +class Node: + def __init__(self, key=None, value=None, empty=False): self.empty = empty if not empty: self.key = key self.value = value # next # prev + def visual(self): - return "key:{key}\t value:{value}\t prev:{prev}\t next:{next}\t "\ + return ( + "key:{key}\t value:{value}\t prev:{prev}\t next:{next}\t " "empty:{empty}".format( - key = self.__dict__.get('key', None), - value = self.__dict__.get('value', None), - prev = self.__dict__.get('prev', None), - next = self.__dict__.get('next', None), - empty = self.__dict__.get('empty', None)) + key=self.__dict__.get("key", None), + value=self.__dict__.get("value", None), + prev=self.__dict__.get("prev", None), + next=self.__dict__.get("next", None), + empty=self.__dict__.get("empty", None), + ) + ) -class LinkedList(object): + +class LinkedList: """ Requirement for the node: Any vairiable that can set 'prev', and 'next' attributes can be used. """ - class _EmptyNode(object): + class _EmptyNode: pass def __init__(self): @@ -95,17 +101,17 @@ def move_toward_head_by_one(self, node): # node is not head prev_node = node.prev self.delete(node) - self.add_before2(new_node = node, node = prev_node) + self.add_before2(new_node=node, node=prev_node) else: return False def move_toward_tail_by_one(self, node): "Boolean is returned to indicate status" - if node.next is not self._end_guard: + if node.__next__ is not self._end_guard: # node is not the tail - next_next_node = node.next.next - self.delete(node) # _head is handled properly here - self.add_before2(new_node = node, node = next_next_node) + next_next_node = node.next.__next__ + self.delete(node) # _head is handled properly here + self.add_before2(new_node=node, node=next_next_node) else: return False @@ -121,12 +127,12 @@ def delete(self, node): # special case, node is the head if node is self._head: - self._head = node.next + self._head = node.__next__ prev_node = node.prev - next_node = node.next + next_node = node.__next__ - prev_node.next = node.next + prev_node.next = node.__next__ next_node.prev = node.prev self.size -= 1 @@ -153,7 +159,7 @@ def __iter__(self): node = self._head while node is not self._end_guard: yield node - node = node.next + node = node.__next__ def __reversed__(self): node = self._end_guard.prev @@ -169,17 +175,19 @@ def __str__(self): node = self._head while node is not self._end_guard: listview.append(str(node)) - node = node.next - listview = '->'.join(listview) + node = node.__next__ + listview = "->".join(listview) return listview -class LruCache(collections.MutableMapping): + +class LruCache(collections.abc.MutableMapping): """ Geting and setting (recent use) a value will move it to the head of the list. It provides mapping interfaces like dict. """ - def __init__(self, data = None, **kwargs): + + def __init__(self, data=None, **kwargs): self.table = {} # node must be type 'class Node' self.linked_list = LinkedList() @@ -189,13 +197,13 @@ def __init__(self, data = None, **kwargs): self.update(data, **kwargs) def has_key(self, key): - return self.table.has_key(key) + return key in self.table def keys(self): - return self.table.keys() + return list(self.table.keys()) - def get(self, key, default = None): - if self.table.has_key(key): + def get(self, key, default=None): + if key in self.table: # will affect list order return self.__getitem__(key) else: @@ -214,21 +222,21 @@ def __delitem__(self, key): self.linked_list.delete(node) def __setitem__(self, key, value): - if self.table.has_key(key): + if key in self.table: # update node = self.table[key] node.value = value self.linked_list.move_to_head(node) else: # create new - node = Node(key = key, value = value) + node = Node(key=key, value=value) self.linked_list.add_to_head(node) self.table[key] = node # FIXME: bottleneck def add_as_least_used(self, key, value): - assert not self.table.has_key(key) - node = Node(key = key, value = value) + assert key not in self.table + node = Node(key=key, value=value) self.linked_list.add_to_tail(node) self.table[key] = node @@ -244,7 +252,6 @@ def __reversed__(self): def items(self): return self.least_to_most_items() - def __len__(self): return len(self.linked_list) @@ -276,7 +283,6 @@ def __repr__(self): return str(t) - """ Segmented LRU (SLRU) @@ -306,22 +312,24 @@ class LinkedListVisNode(LinkedList): in __repr__(). This requires the node in the list to have visual() method. """ + def __repr__(self): listview = [] node = self._head while node is not self._end_guard: listview.append(node.visual()) - node = node.next - listview = '->\n'.join(listview) + node = node.__next__ + listview = "->\n".join(listview) - return 'LinkedList:\n' + listview + return "LinkedList:\n" + listview -class SegmentedLruCache(object): +class SegmentedLruCache: """ nodes should have a type attribute: PROTECTED, PROBATIONARY It has key-value interface. Node is used internally. """ + def __init__(self, max_entries, max_protected_ratio): self.protected_list = LinkedListVisNode() self.probationary_list = LinkedListVisNode() @@ -332,10 +340,10 @@ def __init__(self, max_entries, max_protected_ratio): self.table = {} def has_key(self, key): - return self.table.has_key(key) + return key in self.table def keys(self): - return self.table.keys() + return list(self.table.keys()) def _add_new_node(self, node): """ @@ -367,13 +375,13 @@ def _move_from_prob_to_prot(self, node): self._move_from_prot_to_prob(victim_node) self.probationary_list.delete(node) - self.protected_list.add_to_head(node) # MRU side + self.protected_list.add_to_head(node) # MRU side node.owner_list = self.protected_list def _move_from_prot_to_prob(self, node): "move from probationary to MRU side of protected" self.protected_list.delete(node) - self.probationary_list.add_to_head(node) # MRU side + self.probationary_list.add_to_head(node) # MRU side node.owner_list = self.probationary_list def hit(self, node): @@ -386,7 +394,7 @@ def hit(self, node): ############### APIs ################ def items(self): - for key in self.keys(): + for key in list(self.keys()): yield key, self.table[key].value def __getitem__(self, key): @@ -394,8 +402,8 @@ def __getitem__(self, key): self.hit(node) return node.value - def get(self, key, default = None): - if self.table.has_key(key): + def get(self, key, default=None): + if key in self.table: # will affect list order return self.__getitem__(key) else: @@ -410,14 +418,14 @@ def __setitem__(self, key, value): """ This should be the only API to add key-value to the cache! """ - if self.table.has_key(key): + if key in self.table: # update node = self.table[key] node.value = value self.hit(node) else: # create new - node = Node(key = key, value = value) + node = Node(key=key, value=value) self.table[key] = node self._add_new_node(node) @@ -442,17 +450,22 @@ def __delitem__(self, key): self._remove_item(key) def __iter__(self): - return self.table.keys() + return list(self.table.keys()) def __len__(self): return len(self.table) def __repr__(self): - return 'Protected List:' + repr(self.protected_list) + '\n' + \ - 'Probationary List:' + repr(self.probationary_list) + return ( + "Protected List:" + + repr(self.protected_list) + + "\n" + + "Probationary List:" + + repr(self.probationary_list) + ) -class LruDict(collections.MutableMapping): +class LruDict(collections.abc.MutableMapping): # __getitem__, __setitem__, __delitem__, __iter__, __len__ """ All [] operations will change order of the key @@ -460,6 +473,7 @@ class LruDict(collections.MutableMapping): WARNING: If used with simpy, OrderedDict.items() and related functions are very slow (5 secs for a few thousands of items). """ + def __init__(self, data=None, **kwargs): """ This is a regular constructor of dict. data can be mapping or an @@ -493,8 +507,7 @@ def least_to_most_iter(self): return self.__iter__() def __iter__(self): - for k in self._store: - yield k + yield from self._store def most_to_least_iter(self): return self.__reversed__() @@ -506,27 +519,24 @@ def __len__(self): return len(self._store) def has_key(self, key): - return self._store.has_key(key) + return key in self._store def items(self): - for k, v in self._store.items(): - yield k, v + yield from list(self._store.items()) def least_to_most_items(self): - return self._store.items() + return list(self._store.items()) def victim_key(self): return self.least_recent() def most_recent(self): it = self.most_to_least_iter() - return it.next() + return next(it) def least_recent(self): it = self.least_to_most_iter() - return it.next() + return next(it) def peek(self, key): return self._store[key] - - diff --git a/wiscsee/wiscsim/nkftl2.py b/wiscsee/wiscsim/nkftl2.py index 9304ced..a825ec2 100644 --- a/wiscsee/wiscsim/nkftl2.py +++ b/wiscsee/wiscsim/nkftl2.py @@ -1,20 +1,22 @@ +from past.builtins import cmp +from past.utils import old_div import bidict import sys import copy from collections import deque, OrderedDict import datetime -import Queue +import queue import itertools import config -import ftlbuilder -import recorder +from . import ftlbuilder +from . import recorder from utilities import utils from .bitmap import FlashBitmap2 from wiscsim.devblockpool import * -from ftlsim_commons import * +from .ftlsim_commons import * from commons import * -from datacache import * +from .datacache import * """ ############## Checklist ############### @@ -31,20 +33,19 @@ """ -ERR_NEED_NEW_BLOCK, ERR_NEED_MERGING = ( - 'ERR_NEED_NEW_BLOCK', 'ERR_NEED_MERGING') +ERR_NEED_NEW_BLOCK, ERR_NEED_MERGING = ("ERR_NEED_NEW_BLOCK", "ERR_NEED_MERGING") DATA_USER = "data.user" IN_LOG_BLOCK = "IN_LOG_BLOCK" IN_DATA_BLOCK = "IN_DATA_BLOCK" -TYPE_LOG_BLOCK, TYPE_DATA_BLOCK = ('TYPE_LOG_BLOCK', 'TYPE_DATA_BLOCK') +TYPE_LOG_BLOCK, TYPE_DATA_BLOCK = ("TYPE_LOG_BLOCK", "TYPE_DATA_BLOCK") -TAG_PARTIAL_MERGE = 'PARTIAL.MERGE' -TAG_SWITCH_MERGE = 'SWITCH.MERGE' -TAG_FULL_MERGE = 'FULL.MERGE' -TAG_FORGROUND = 'FORGROUND' -TAG_WRITE_DRIVEN = 'WRITE.DRIVEN.DIRECT.ERASE' -TAG_THRESHOLD_GC = 'THRESHOLD.GC.DIRECT.ERASE' -TAG_SIMPLE_ERASE = 'SIMPLE.ERASE' +TAG_PARTIAL_MERGE = "PARTIAL.MERGE" +TAG_SWITCH_MERGE = "SWITCH.MERGE" +TAG_FULL_MERGE = "FULL.MERGE" +TAG_FORGROUND = "FORGROUND" +TAG_WRITE_DRIVEN = "WRITE.DRIVEN.DIRECT.ERASE" +TAG_THRESHOLD_GC = "THRESHOLD.GC.DIRECT.ERASE" +TAG_SIMPLE_ERASE = "SIMPLE.ERASE" class OutOfSpaceError(RuntimeError): @@ -53,17 +54,15 @@ class OutOfSpaceError(RuntimeError): class Config(config.ConfigNCQFTL): def __init__(self, confdic=None): - super(Config, self).__init__(confdic) + super().__init__(confdic) local_items = { ############## NKFTL (SAST) ############ "nkftl": { - 'n_blocks_in_data_group': 4, # number of blocks in a data block group - 'max_blocks_in_log_group': 1, # max number of blocks in a log block group - + "n_blocks_in_data_group": 4, # number of blocks in a data block group + "max_blocks_in_log_group": 1, # max number of blocks in a log block group "GC_threshold_ratio": 0.8, "GC_low_threshold_ratio": 0.7, - "max_ratio_of_log_blocks": 0.2, }, "write_gc_log": False, @@ -71,26 +70,28 @@ def __init__(self, confdic=None): self.update(local_items) def n_pages_per_data_group(self): - n_blocks_in_data_group = self['nkftl']['n_blocks_in_data_group'] + n_blocks_in_data_group = self["nkftl"]["n_blocks_in_data_group"] n_pages_per_block = self.n_pages_per_block n_pages_per_dg = n_blocks_in_data_group * n_pages_per_block return n_pages_per_dg def n_datagroups_per_dev(self): - n_blocks_in_data_group = self['nkftl']['n_blocks_in_data_group'] - return self.n_blocks_per_dev / n_blocks_in_data_group + n_blocks_in_data_group = self["nkftl"]["n_blocks_in_data_group"] + return old_div(self.n_blocks_per_dev, n_blocks_in_data_group) def nkftl_data_group_number_of_lpn(self, lpn): """ Given lpn, return its data group number """ - dgn = (lpn / self.n_pages_per_block) / \ - self['nkftl']['n_blocks_in_data_group'] + dgn = old_div( + (old_div(lpn, self.n_pages_per_block)), + self["nkftl"]["n_blocks_in_data_group"], + ) return dgn def nkftl_data_group_number_of_logical_block(self, logical_block_num): - dgn = logical_block_num / self['nkftl']['n_blocks_in_data_group'] + dgn = old_div(logical_block_num, self["nkftl"]["n_blocks_in_data_group"]) return dgn def nkftl_max_n_log_pages_in_data_group(self): @@ -98,8 +99,7 @@ def nkftl_max_n_log_pages_in_data_group(self): This is the max number of log pages in data group: max number of log blocks * number of pages in block """ - return self['nkftl']['max_blocks_in_log_group'] * \ - self.n_pages_per_block + return self["nkftl"]["max_blocks_in_log_group"] * self.n_pages_per_block def nkftl_allowed_num_of_data_blocks(self): """ @@ -112,11 +112,13 @@ def nkftl_allowed_num_of_data_blocks(self): such as merging. """ - raise RuntimeError("nkftl_set_flash_num_blocks_by_data_block_bytes()" - "should not be called anymore because it assumes the total number " - "of data blocks and log blocks in flash to be proportional " - "following N/K. In fact, the number of log blocks in flash can be " - "less than total.flash.block * K/(N+K).") + raise RuntimeError( + "nkftl_set_flash_num_blocks_by_data_block_bytes()" + "should not be called anymore because it assumes the total number " + "of data blocks and log blocks in flash to be proportional " + "following N/K. In fact, the number of log blocks in flash can be " + "less than total.flash.block * K/(N+K)." + ) def nkftl_set_flash_num_blocks_by_data_block_bytes(self, data_bytes): """ @@ -126,14 +128,16 @@ def nkftl_set_flash_num_blocks_by_data_block_bytes(self, data_bytes): log blocks. """ - raise RuntimeError("nkftl_set_flash_num_blocks_by_data_block_bytes()" - "should not be called anymore because it assumes the total number " - "of data blocks and log blocks in flash to be proportional " - "following N/K. In fact, the number of log blocks in flash can be " - "less than total.flash.block * K/(N+K).") + raise RuntimeError( + "nkftl_set_flash_num_blocks_by_data_block_bytes()" + "should not be called anymore because it assumes the total number " + "of data blocks and log blocks in flash to be proportional " + "following N/K. In fact, the number of log blocks in flash can be " + "less than total.flash.block * K/(N+K)." + ) -class GlobalHelper(object): +class GlobalHelper: """ In case you need some global variables. We put all global stuff here so it is easier to manage. (And you know all the bad things you did :) @@ -147,7 +151,7 @@ def incr_lba_op_timestamp(self): self.cur_lba_op_timestamp += 1 -class OutOfBandAreas(object): +class OutOfBandAreas: def __init__(self, confobj): self.conf = confobj @@ -163,13 +167,13 @@ def display_bitmap_by_block(self): npages_per_block = self.conf.n_pages_per_block nblocks = self.conf.n_blocks_per_dev totalpages = nblocks * npages_per_block - line = '' + line = "" for i in range(totalpages): - block_index = i / npages_per_block + block_index = old_div(i, npages_per_block) rem = i % npages_per_block if rem == 0: - line += ' ' + str(block_index) + ':' - line += str(self.states.page_state_human(i)) + '|' + line += " " + str(block_index) + ":" + line += str(self.states.page_state_human(i)) + "|" return line @@ -215,7 +219,7 @@ def lpns_of_block(self, flash_block): s, e = self.conf.block_to_page_range(flash_block) lpns = [] for ppn in range(s, e): - lpns.append(self.ppn_to_lpn.get(ppn, 'NA')) + lpns.append(self.ppn_to_lpn.get(ppn, "NA")) return lpns @@ -241,7 +245,7 @@ def are_all_pages_erased(self, flash_block): return True -class MappingBase(object): +class MappingBase: """ This class defines a __init__() that passes in necessary objects to the mapping object. @@ -255,8 +259,7 @@ def __init__(self, confobj, recorderobj, global_helper_obj): class DataBlockMappingTable(MappingBase): def __init__(self, confobj, recorderobj, global_helper_obj): - super(DataBlockMappingTable, self).__init__(confobj, recorderobj, - global_helper_obj) + super().__init__(confobj, recorderobj, global_helper_obj) self.logical_to_physical_block = bidict.bidict() @@ -309,8 +312,8 @@ def __str__(self): return str(self.logical_to_physical_block) -TDATA = 'TDATA' -TLOG = 'TLOG' +TDATA = "TDATA" +TLOG = "TLOG" class NKBlockPool(MultiChannelBlockPoolBase): @@ -330,8 +333,7 @@ def data_usedblocks(self): return blocks def pop_a_free_block_to_log_blocks(self, choice=LEAST_ERASED): - blocknum = self.pick_and_move(src=TFREE, dst=TLOG, - choice=choice) + blocknum = self.pick_and_move(src=TFREE, dst=TLOG, choice=choice) return blocknum @@ -339,8 +341,7 @@ def move_used_log_to_data_block(self, blocknum): self.change_tag(blocknum, src=TLOG, dst=TDATA) def pop_a_free_block_to_data_blocks(self, choice=LEAST_ERASED): - blocknum = self.pick_and_move(src=TFREE, dst=TDATA, - choice=choice) + blocknum = self.pick_and_move(src=TFREE, dst=TDATA, choice=choice) return blocknum def free_used_data_block(self, blocknum): @@ -358,7 +359,7 @@ def used_ratio(self): return (self.n_blocks_per_dev - nfree) / float(self.n_blocks_per_dev) -class LogGroup2(object): +class LogGroup2: """ - keep track of log blocks of this group - allocate pages from blocks of this group @@ -440,7 +441,7 @@ def remove_log_block(self, log_pbn): self._remove_block(log_pbn) def _remove_block(self, blocknum): - channel_id = blocknum / self.conf.n_blocks_per_channel + channel_id = old_div(blocknum, self.conf.n_blocks_per_channel) channel_blocks = self.log_channels[channel_id] to_del = None for cur_block in channel_blocks: @@ -482,8 +483,8 @@ def next_ppns(self, n, strip_unit_size): log blocks and make new ones available. """ remaining = n - if strip_unit_size == 'infinity': - strip_unit_size = float('inf') + if strip_unit_size == "infinity": + strip_unit_size = float("inf") ret_ppns = [] dead_channels = set() @@ -492,11 +493,12 @@ def next_ppns(self, n, strip_unit_size): # max, because it is impossible. We check before we allocate # block. cur_channel_id = self._get_cur_channel() - reqsize = min(remaining, - strip_unit_size, - strip_unit_size - self._cur_channel_used_pages) - ppns = self._next_ppns_in_channel_with_allocation( - reqsize, cur_channel_id) + reqsize = min( + remaining, + strip_unit_size, + strip_unit_size - self._cur_channel_used_pages, + ) + ppns = self._next_ppns_in_channel_with_allocation(reqsize, cur_channel_id) n_pages_allocated = len(ppns) ret_ppns.extend(ppns) @@ -512,8 +514,10 @@ def next_ppns(self, n, strip_unit_size): # advance channel when 1. current channel is full, or 2. current # usage_pages reaches strip_unit_size assert self._cur_channel_used_pages <= strip_unit_size - if n_pages_allocated < reqsize or \ - self._cur_channel_used_pages >= strip_unit_size: + if ( + n_pages_allocated < reqsize + or self._cur_channel_used_pages >= strip_unit_size + ): self._incr_cur_channel() self._cur_channel_used_pages = 0 @@ -540,7 +544,7 @@ def _next_ppns_in_channel_with_allocation(self, reqsize, channel_id): return ret_ppns def register_pbn(self, pbn): - channel_id = pbn / self.conf.n_blocks_per_channel + channel_id = old_div(pbn, self.conf.n_blocks_per_channel) curblock = CurrentBlock(self.n_pages_per_block, pbn) # set the curblock as fully used so nobody accidentally use it @@ -559,10 +563,12 @@ def _allocate_block_in_channel(self, channel_id): blocknum = self.block_pool.pick(tag=TFREE, channel_id=channel_id) self.block_pool.change_tag(blocknum, src=TFREE, dst=TLOG) self.log_channels[channel_id].append( - CurrentBlock(self.n_pages_per_block, blocknum)) + CurrentBlock(self.n_pages_per_block, blocknum) + ) - assert self.n_log_blocks() <= self.max_n_log_blocks, "{} > {}".format( - self.n_log_blocks(), self.max_n_log_blocks) + assert self.n_log_blocks() <= self.max_n_log_blocks, ( + f"{self.n_log_blocks()} > {self.max_n_log_blocks}" + ) return True def _next_ppns_in_channel(self, n, channel_id): @@ -584,28 +590,31 @@ def _next_ppns_in_channel(self, n, channel_id): class Translator(MappingBase): - def __init__(self, confobj, recorderobj, global_helper_obj, - log_mapping, data_block_mapping - ): - super(Translator, self).__init__(confobj, recorderobj, - global_helper_obj) + def __init__( + self, confobj, recorderobj, global_helper_obj, log_mapping, data_block_mapping + ): + super().__init__(confobj, recorderobj, global_helper_obj) self.data_block_mapping_table = data_block_mapping self.log_mapping_table = log_mapping def __str__(self): ret = [] ret.append( - '========================= MAPPING MANAGER ===========================') + "========================= MAPPING MANAGER ===========================" + ) ret.append( - '------------------------ Data Block Mapping Table -------------------') + "------------------------ Data Block Mapping Table -------------------" + ) ret.append(str(self.data_block_mapping_table)) ret.append( - '------------------------ Log Block Mapping Table --------------------') + "------------------------ Log Block Mapping Table --------------------" + ) ret.append(str(self.log_mapping_table)) ret.append( - '=====================================================================') + "=====================================================================" + ) - return '\n'.join(ret) + return "\n".join(ret) def lpn_to_ppn(self, lpn): """ @@ -636,8 +645,7 @@ def lpn_to_ppn(self, lpn): class LogMappingTable(MappingBase): def __init__(self, confobj, block_pool, recorderobj, global_helper_obj): - super(LogMappingTable, self).__init__(confobj, recorderobj, - global_helper_obj) + super().__init__(confobj, recorderobj, global_helper_obj) self.block_pool = block_pool @@ -647,7 +655,7 @@ def __init__(self, confobj, block_pool, recorderobj, global_helper_obj): def find_group_by_pbn(self, pbn): dgn = None loggroup = None - for i_dgn, i_loggroup in self.log_group_info.items(): + for i_dgn, i_loggroup in list(self.log_group_info.items()): if pbn in i_loggroup.log_block_numbers(): dgn = i_dgn loggroup = i_loggroup @@ -655,9 +663,14 @@ def find_group_by_pbn(self, pbn): return dgn, loggroup def next_ppns_to_program(self, dgn, n, strip_unit_size): - loggroup = self.log_group_info.setdefault(dgn, - LogGroup2(self.conf, self.block_pool, - max_n_log_blocks=self.conf['nkftl']['max_blocks_in_log_group'])) + loggroup = self.log_group_info.setdefault( + dgn, + LogGroup2( + self.conf, + self.block_pool, + max_n_log_blocks=self.conf["nkftl"]["max_blocks_in_log_group"], + ), + ) return loggroup.next_ppns(n, strip_unit_size=strip_unit_size) def add_mapping(self, lpn, ppn): @@ -692,32 +705,35 @@ def remove_log_block(self, data_group_no, log_pbn): self.log_group_info[data_group_no].remove_log_block(log_pbn) -class GcDecider(object): +class GcDecider: def __init__(self, confobj, block_pool, recorderobj): self.conf = confobj self.block_pool = block_pool self.recorder = recorderobj - self.high_watermark = self.conf['nkftl']['GC_threshold_ratio'] * \ - self.conf.n_blocks_per_dev - self.low_watermark = self.conf['nkftl']['GC_low_threshold_ratio'] * \ - self.conf.n_blocks_per_dev + self.high_watermark = ( + self.conf["nkftl"]["GC_threshold_ratio"] * self.conf.n_blocks_per_dev + ) + self.low_watermark = ( + self.conf["nkftl"]["GC_low_threshold_ratio"] * self.conf.n_blocks_per_dev + ) assert self.high_watermark > self.low_watermark def should_start(self): n_used_blocks = self.block_pool.total_used_blocks() n_used_log_blocks = len(self.block_pool.log_usedblocks) - log_block_high = self.conf.n_blocks_per_dev * \ - self.conf['nkftl']['max_ratio_of_log_blocks'] + log_block_high = ( + self.conf.n_blocks_per_dev * self.conf["nkftl"]["max_ratio_of_log_blocks"] + ) if n_used_blocks > self.high_watermark: - self.recorder.count_me('should_start', 'high_watermark') - print 'high_watermark' + self.recorder.count_me("should_start", "high_watermark") + print("high_watermark") return True if n_used_log_blocks > log_block_high: - self.recorder.count_me('should_start', 'log_block_high') + self.recorder.count_me("should_start", "log_block_high") # print 'log_block_high' return True @@ -762,13 +778,14 @@ def _freezed_too_long(self, cur_n_used_blocks): return ret -class BlockInfo(object): +class BlockInfo: """ This is for sorting blocks to clean the victim. """ - def __init__(self, block_type, block_num, last_used_time, - valid_ratio, data_group_no=None): + def __init__( + self, block_type, block_num, last_used_time, valid_ratio, data_group_no=None + ): self.valid_ratio = valid_ratio self.block_type = block_type self.block_num = block_num @@ -782,12 +799,19 @@ def __cmp__(self, other): return cmp(self.last_used_time, other.last_used_time) -class WearLevelingVictimBlocks(object): - TYPE_DATA = 'TYPE_DATA' - TYPE_LOG = 'TYPE_LOG' +class WearLevelingVictimBlocks: + TYPE_DATA = "TYPE_DATA" + TYPE_LOG = "TYPE_LOG" - def __init__(self, conf, block_pool, oob, n_victims, - log_mapping_table, data_block_mapping_table): + def __init__( + self, + conf, + block_pool, + oob, + n_victims, + log_mapping_table, + data_block_mapping_table, + ): self._conf = conf self._block_pool = block_pool self._oob = oob @@ -824,14 +848,15 @@ def iterator_verbose(self): break -class VictimBlocksBase(object): - def __init__(self, conf, block_pool, oob, rec, log_mapping_table, - data_block_mapping_table): +class VictimBlocksBase: + def __init__( + self, conf, block_pool, oob, rec, log_mapping_table, data_block_mapping_table + ): self.conf = conf self.block_pool = block_pool self.oob = oob self.rec = rec - self.priority_q = Queue.PriorityQueue() + self.priority_q = queue.PriorityQueue() self.log_mapping = log_mapping_table self.data_mapping = data_block_mapping_table @@ -856,7 +881,8 @@ def _init(self): block_type=TYPE_DATA_BLOCK, block_num=blocknum, valid_ratio=self.oob.states.block_valid_ratio(blocknum), - last_used_time=-1) # high priority + last_used_time=-1, + ) # high priority self.priority_q.put(blk_info) @@ -866,22 +892,36 @@ def _init(self): TODO: is there any log blocks that are not in log mapping but in log_usedblocks? """ - for data_group_no, log_group_info in self.log_mapping.log_group_info.items(): + for data_group_no, log_group_info in list( + self.log_mapping.log_group_info.items() + ): for curblock in log_group_info.cur_blocks(): blk_info = BlockInfo( block_type=TYPE_LOG_BLOCK, block_num=curblock.blocknum, - valid_ratio=self.oob.states.block_valid_ratio( - curblock.blocknum), + valid_ratio=self.oob.states.block_valid_ratio(curblock.blocknum), last_used_time=0, - data_group_no=data_group_no) + data_group_no=data_group_no, + ) self.priority_q.put(blk_info) -class GarbageCollector(object): - def __init__(self, confobj, block_pool, flashobj, oobobj, recorderobj, - translatorobj, global_helper_obj, log_mapping, data_block_mapping, - simpy_env, des_flash, logical_block_locks): +class GarbageCollector: + def __init__( + self, + confobj, + block_pool, + flashobj, + oobobj, + recorderobj, + translatorobj, + global_helper_obj, + log_mapping, + data_block_mapping, + simpy_env, + des_flash, + logical_block_locks, + ): self.conf = confobj self.flash = flashobj self.oob = oobobj @@ -900,8 +940,8 @@ def __init__(self, confobj, block_pool, flashobj, oobobj, recorderobj, self.decider = GcDecider(self.conf, self.block_pool, self.recorder) - n_cleaners = self.conf['n_gc_procs'] - print 'n_cleaners:', n_cleaners + n_cleaners = self.conf["n_gc_procs"] + print(("n_cleaners:", n_cleaners)) self._cleaner_res = simpy.Resource(self.env, capacity=n_cleaners) self.gcid = 0 @@ -917,7 +957,7 @@ def clean(self, forced=False, merge=True): procs = [] for dgn in range(self.conf.n_datagroups_per_dev()): - if dgn in self.log_mapping_table.log_group_info.keys(): + if dgn in list(self.log_mapping_table.log_group_info.keys()): p = self.env.process(self.clean_data_group(dgn, merge=merge)) procs.append(p) yield simpy.AllOf(self.env, procs) @@ -945,13 +985,15 @@ def clean_data_group(self, data_group_no, merge=True): self.gcid += 1 if self.gc_time_recorded == False: - self.recorder.set_result_by_one_key('gc_trigger_timestamp', - self.env.now / float(SEC)) + self.recorder.set_result_by_one_key( + "gc_trigger_timestamp", self.env.now / float(SEC) + ) self.gc_time_recorded = True - print 'GC time recorded!........!' + print("GC time recorded!........!") - log_block_list = copy.copy(self.log_mapping_table - .log_group_info[data_group_no].log_block_numbers()) + log_block_list = copy.copy( + self.log_mapping_table.log_group_info[data_group_no].log_block_numbers() + ) procs = [] for log_block in log_block_list: # A log block may not be a log block anymore after the loop starts @@ -961,8 +1003,9 @@ def clean_data_group(self, data_group_no, merge=True): log_pbn=log_block, data_group_no=data_group_no, tag=TAG_WRITE_DRIVEN, - merge=merge - )) + merge=merge, + ) + ) procs.append(p) yield simpy.AllOf(self.env, procs) @@ -986,20 +1029,27 @@ def clean_log_block(self, log_pbn, data_group_no, tag, merge=True): # Check to see if it is really the log block of the speicfed data # group - if log_pbn not in self.translator.log_mapping_table\ - .log_group_info[data_group_no].log_block_numbers(): + if ( + log_pbn + not in self.translator.log_mapping_table.log_group_info[ + data_group_no + ].log_block_numbers() + ): self._cleaner_res.release(req) return valid_ratio = self.oob.states.block_valid_ratio(log_pbn) - self.recorder.count_me('victim_valid_ratio', round(valid_ratio, 2)) + self.recorder.count_me("victim_valid_ratio", round(valid_ratio, 2)) erased_ratio = self.oob.states.block_erased_ratio(log_pbn) - self.recorder.count_me('victim_erased_ratio', round(erased_ratio, 2)) + self.recorder.count_me("victim_erased_ratio", round(erased_ratio, 2)) # Just free it? if not self.oob.is_any_page_valid(log_pbn): - yield self.env.process(self._recycle_empty_log_block( - data_group_no, log_pbn, tag=TAG_SIMPLE_ERASE)) + yield self.env.process( + self._recycle_empty_log_block( + data_group_no, log_pbn, tag=TAG_SIMPLE_ERASE + ) + ) self._cleaner_res.release(req) return @@ -1012,17 +1062,18 @@ def clean_log_block(self, log_pbn, data_group_no, tag, merge=True): is_mergable, logical_block = self.is_switch_mergable(log_pbn) if is_mergable == True: yield self.env.process( - self.switch_merge(log_pbn=log_pbn, - logical_block=logical_block)) + self.switch_merge(log_pbn=log_pbn, logical_block=logical_block) + ) self._cleaner_res.release(req) return - is_mergable, logical_block, offset = self.is_partial_mergable( - log_pbn) + is_mergable, logical_block, offset = self.is_partial_mergable(log_pbn) if is_mergable == True: yield self.env.process( - self.partial_merge(log_pbn=log_pbn, lbn=logical_block, - first_free_offset=offset)) + self.partial_merge( + log_pbn=log_pbn, lbn=logical_block, first_free_offset=offset + ) + ) self._cleaner_res.release(req) return @@ -1037,7 +1088,7 @@ def full_merge(self, log_pbn): find in this log block, we iterate all LPNs to and copy their data to a new free block. """ - self.recorder.count_me("garbage_collection", 'full_merge') + self.recorder.count_me("garbage_collection", "full_merge") # Find all the logical blocks ppn_start, ppn_end = self.conf.block_to_page_range(log_pbn) @@ -1053,7 +1104,8 @@ def full_merge(self, log_pbn): # Move all the pages of a logical block to new block for logical_block in logical_blocks: yield self.env.process( - self.aggregate_logical_block(logical_block, TAG_FULL_MERGE)) + self.aggregate_logical_block(logical_block, TAG_FULL_MERGE) + ) self.asserts() def aggregate_logical_block(self, lbn, tag): @@ -1107,31 +1159,33 @@ def aggregate_logical_block(self, lbn, tag): lpn_start, lpn_end = self.conf.block_to_page_range(lbn) for lpn in range(lpn_start, lpn_end): in_block_page_off = lpn - lpn_start - dst_ppn = self.conf.block_off_to_page(dst_phy_block_num, - in_block_page_off) + dst_ppn = self.conf.block_off_to_page(dst_phy_block_num, in_block_page_off) data_group_no = self.conf.nkftl_data_group_number_of_lpn(lpn) found, src_ppn, loc = self.translator.lpn_to_ppn(lpn) if found == True and self.oob.states.is_page_valid(src_ppn): data = self.flash.page_read(src_ppn, cat=tag) yield self.env.process( - self.des_flash.rw_ppns([src_ppn], 'read', tag=tag)) + self.des_flash.rw_ppns([src_ppn], "read", tag=tag) + ) self.flash.page_write(dst_ppn, cat=tag, data=data) yield self.env.process( - self.des_flash.rw_ppns([dst_ppn], 'write', tag=tag)) + self.des_flash.rw_ppns([dst_ppn], "write", tag=tag) + ) - self.oob.remap(lpn=lpn, old_ppn=src_ppn, - new_ppn=dst_ppn) + self.oob.remap(lpn=lpn, old_ppn=src_ppn, new_ppn=dst_ppn) src_block, _ = self.conf.page_to_block_off(src_ppn) - if self.conf['write_gc_log'] is True: - self.recorder.write_file('gc.log', - gcid=self.gcid, - blocknum=src_block, - lpn=lpn, - ppn=src_ppn, - merge_type='full', - valid=True) + if self.conf["write_gc_log"] is True: + self.recorder.write_file( + "gc.log", + gcid=self.gcid, + blocknum=src_block, + lpn=lpn, + ppn=src_ppn, + merge_type="full", + valid=True, + ) # Now you've moved lpn, you need to remove lpn mapping if it is # in log blocks @@ -1145,15 +1199,19 @@ def aggregate_logical_block(self, lbn, tag): if not self.oob.is_any_page_valid(src_pbn): if loc == IN_DATA_BLOCK: yield self.env.process( - self.recycle_empty_data_block(data_block=src_pbn, - tag=TAG_FULL_MERGE)) + self.recycle_empty_data_block( + data_block=src_pbn, tag=TAG_FULL_MERGE + ) + ) elif loc == IN_LOG_BLOCK: yield self.env.process( self._recycle_empty_log_block( data_group_no=data_group_no, log_pbn=src_pbn, - tag=TAG_FULL_MERGE)) + tag=TAG_FULL_MERGE, + ) + ) else: # This lpn does not exist, so we just invalidate the @@ -1165,14 +1223,15 @@ def aggregate_logical_block(self, lbn, tag): # Now we have all the pages in new block, we make the new block # the data block for lbn - found, old_pbn = self.translator.data_block_mapping_table\ - .lbn_to_pbn(lbn) + found, old_pbn = self.translator.data_block_mapping_table.lbn_to_pbn(lbn) if found == True: # old_pbn must not have any valid pages, so we free it yield self.env.process( - self.recycle_empty_data_block(old_pbn, tag=TAG_FULL_MERGE)) + self.recycle_empty_data_block(old_pbn, tag=TAG_FULL_MERGE) + ) self.translator.data_block_mapping_table.add_data_block_mapping( - lbn=lbn, pbn=dst_phy_block_num) + lbn=lbn, pbn=dst_phy_block_num + ) self.logical_block_locks.release_request(lbn, req) @@ -1265,10 +1324,12 @@ def partial_merge(self, log_pbn, lbn, first_free_offset): req = self.logical_block_locks.get_request(lbn) yield req - is_mergable, logical_block_ret, offset_ret = self.is_partial_mergable( - log_pbn) - if is_mergable is False or logical_block_ret != lbn or \ - offset_ret != first_free_offset: + is_mergable, logical_block_ret, offset_ret = self.is_partial_mergable(log_pbn) + if ( + is_mergable is False + or logical_block_ret != lbn + or offset_ret != first_free_offset + ): # we need to double check here since while we wait, things may # have changed. We check logical_block_ret != logical_block # because we only lock logical block, it is possible that @@ -1276,13 +1337,11 @@ def partial_merge(self, log_pbn, lbn, first_free_offset): self.logical_block_locks.release_request(lbn, req) return - self.recorder.count_me("garbage_collection", 'partial_merge') + self.recorder.count_me("garbage_collection", "partial_merge") - data_group_no = self.conf.nkftl_data_group_number_of_logical_block( - lbn) + data_group_no = self.conf.nkftl_data_group_number_of_logical_block(lbn) # Copy - for offset in range(first_free_offset, - self.conf.n_pages_per_block): + for offset in range(first_free_offset, self.conf.n_pages_per_block): lpn = self.conf.block_off_to_page(lbn, offset) dst_ppn = self.conf.block_off_to_page(log_pbn, offset) @@ -1296,21 +1355,24 @@ def partial_merge(self, log_pbn, lbn, first_free_offset): src_block, _ = self.conf.page_to_block_off(src_ppn) data = self.flash.page_read(src_ppn, cat=TAG_PARTIAL_MERGE) yield self.env.process( - self.des_flash.rw_ppns([src_ppn], 'read', tag=TAG_PARTIAL_MERGE)) - self.flash.page_write(dst_ppn, cat=TAG_PARTIAL_MERGE, - data=data) + self.des_flash.rw_ppns([src_ppn], "read", tag=TAG_PARTIAL_MERGE) + ) + self.flash.page_write(dst_ppn, cat=TAG_PARTIAL_MERGE, data=data) yield self.env.process( - self.des_flash.rw_ppns([dst_ppn], 'write', tag=TAG_PARTIAL_MERGE)) + self.des_flash.rw_ppns([dst_ppn], "write", tag=TAG_PARTIAL_MERGE) + ) self.oob.remap(lpn, old_ppn=src_ppn, new_ppn=dst_ppn) - if self.conf['write_gc_log'] is True: - self.recorder.write_file('gc.log', - gcid=self.gcid, - blocknum=src_block, - lpn=lpn, - ppn=src_ppn, - merge_type='partial-data', - valid=True) + if self.conf["write_gc_log"] is True: + self.recorder.write_file( + "gc.log", + gcid=self.gcid, + blocknum=src_block, + lpn=lpn, + ppn=src_ppn, + merge_type="partial-data", + valid=True, + ) # This branch may never be called because the none of the rest # of the pages is valid, thus you don't have the change @@ -1335,51 +1397,60 @@ def partial_merge(self, log_pbn, lbn, first_free_offset): # on partial merge. You can just find out which one it is and # try to recycle it once. # self.recycle_empty_data_block(src_block, - # tag = TAG_PARTIAL_MERGE ) # check and then recycle + # tag = TAG_PARTIAL_MERGE ) # check and then recycle elif found == True and location == IN_LOG_BLOCK: src_block, _ = self.conf.page_to_block_off(src_ppn) # If the lpn is in log block data = self.flash.page_read(src_ppn, cat=TAG_PARTIAL_MERGE) yield self.env.process( - self.des_flash.rw_ppns([src_ppn], 'read', tag=TAG_PARTIAL_MERGE)) - self.flash.page_write(dst_ppn, cat=TAG_PARTIAL_MERGE, - data=data) + self.des_flash.rw_ppns([src_ppn], "read", tag=TAG_PARTIAL_MERGE) + ) + self.flash.page_write(dst_ppn, cat=TAG_PARTIAL_MERGE, data=data) yield self.env.process( - self.des_flash.rw_ppns([dst_ppn], 'write', tag=TAG_PARTIAL_MERGE)) + self.des_flash.rw_ppns([dst_ppn], "write", tag=TAG_PARTIAL_MERGE) + ) self.oob.remap(lpn, old_ppn=src_ppn, new_ppn=dst_ppn) - if self.conf['write_gc_log'] is True: - self.recorder.write_file('gc.log', - gcid=self.gcid, - blocknum=src_block, - lpn=lpn, - ppn=src_ppn, - merge_type='partial-log', - valid=True) + if self.conf["write_gc_log"] is True: + self.recorder.write_file( + "gc.log", + gcid=self.gcid, + blocknum=src_block, + lpn=lpn, + ppn=src_ppn, + merge_type="partial-log", + valid=True, + ) # you need to remove lpn from log mapping here self.translator.log_mapping_table.remove_lpn(lpn=lpn) yield self.env.process( - self._recycle_empty_log_block(data_group_no=data_group_no, - log_pbn=src_block, tag=TAG_PARTIAL_MERGE)) + self._recycle_empty_log_block( + data_group_no=data_group_no, + log_pbn=src_block, + tag=TAG_PARTIAL_MERGE, + ) + ) # If there is an old data block, we need to recycle it because we # now have a new one. - found, old_pbn = self.translator.data_block_mapping_table\ - .lbn_to_pbn(lbn=lbn) + found, old_pbn = self.translator.data_block_mapping_table.lbn_to_pbn(lbn=lbn) if found: yield self.env.process( - self.recycle_empty_data_block(old_pbn, tag=TAG_PARTIAL_MERGE)) + self.recycle_empty_data_block(old_pbn, tag=TAG_PARTIAL_MERGE) + ) # Now the log block lgo_pbn has all the content of lbn # Now add the new mapping - self.translator.data_block_mapping_table\ - .add_data_block_mapping(lbn=lbn, pbn=log_pbn) + self.translator.data_block_mapping_table.add_data_block_mapping( + lbn=lbn, pbn=log_pbn + ) # Remove log_pbn from log group self.translator.log_mapping_table.remove_log_block( - data_group_no=data_group_no, log_pbn=log_pbn) + data_group_no=data_group_no, log_pbn=log_pbn + ) # move from log pool to data pool self.block_pool.move_used_log_to_data_block(log_pbn) @@ -1458,32 +1529,37 @@ def switch_merge(self, log_pbn, logical_block): self.logical_block_locks.release_request(logical_block, req) return - self.recorder.count_me("garbage_collection", 'switch_merge') + self.recorder.count_me("garbage_collection", "switch_merge") # erase old data block - found, old_physical_block = self.translator.data_block_mapping_table\ - .lbn_to_pbn(logical_block) + found, old_physical_block = self.translator.data_block_mapping_table.lbn_to_pbn( + logical_block + ) if found: # clean up old_physical_block yield self.env.process( - self.recycle_empty_data_block(old_physical_block, tag=TAG_SWITCH_MERGE)) + self.recycle_empty_data_block(old_physical_block, tag=TAG_SWITCH_MERGE) + ) # update data block mapping table # This will override the old mapping if there is one self.translator.data_block_mapping_table.add_data_block_mapping( - logical_block, log_pbn) + logical_block, log_pbn + ) # Update log mapping table, remove log block data_group_no = self.conf.nkftl_data_group_number_of_logical_block( - logical_block) - yield self.env.process(self._remove_log_block(data_group_no, log_pbn, - tag=TAG_SWITCH_MERGE)) + logical_block + ) + yield self.env.process( + self._remove_log_block(data_group_no, log_pbn, tag=TAG_SWITCH_MERGE) + ) # Need to mark the log block as used data block now try: self.block_pool.move_used_log_to_data_block(log_pbn) except ValueError: - print 'log_pbn............', log_pbn + print(("log_pbn............", log_pbn)) raise self.logical_block_locks.release_request(logical_block, req) @@ -1500,12 +1576,14 @@ def recycle_empty_data_block(self, data_block, tag): self.oob.erase_block(data_block) self.flash.block_erase(data_block, cat=tag) # need to remove data block mapping - self.translator.data_block_mapping_table\ - .remove_data_block_mapping_by_pbn(data_block) + self.translator.data_block_mapping_table.remove_data_block_mapping_by_pbn( + data_block + ) self.block_pool.free_used_data_block(data_block) yield self.env.process( - self.des_flash.erase_pbn_extent(data_block, 1, tag=tag)) + self.des_flash.erase_pbn_extent(data_block, 1, tag=tag) + ) self._phy_block_locks.release_request(data_block, req) @@ -1519,12 +1597,14 @@ def _remove_log_block(self, data_group_no, log_pbn, tag): is_log_block = log_pbn in self.block_pool.log_usedblocks has_valid_pages = self.oob.is_any_page_valid(log_pbn) - is_in_dg = log_pbn in self.log_mapping_table\ - .log_group_info[data_group_no].log_block_numbers() + is_in_dg = ( + log_pbn + in self.log_mapping_table.log_group_info[data_group_no].log_block_numbers() + ) if all([is_log_block, has_valid_pages, is_in_dg]): self.translator.log_mapping_table.remove_log_block( - data_group_no=data_group_no, - log_pbn=log_pbn) + data_group_no=data_group_no, log_pbn=log_pbn + ) self._phy_block_locks.release_request(log_pbn, req) @@ -1538,8 +1618,10 @@ def _recycle_empty_log_block(self, data_group_no, log_pbn, tag): is_log_block = log_pbn in self.block_pool.log_usedblocks no_valid_pages = not self.oob.is_any_page_valid(log_pbn) - is_in_dg = log_pbn in self.log_mapping_table\ - .log_group_info[data_group_no].log_block_numbers() + is_in_dg = ( + log_pbn + in self.log_mapping_table.log_group_info[data_group_no].log_block_numbers() + ) if all([is_log_block, no_valid_pages, is_in_dg]): # all true, it should be OK to delete it @@ -1549,12 +1631,11 @@ def _recycle_empty_log_block(self, data_group_no, log_pbn, tag): self.block_pool.free_used_log_block(log_pbn) # remove log mapping TODO: Try? self.translator.log_mapping_table.remove_log_block( - data_group_no=data_group_no, - log_pbn=log_pbn) + data_group_no=data_group_no, log_pbn=log_pbn + ) self.flash.block_erase(log_pbn, cat=tag) - yield self.env.process( - self.des_flash.erase_pbn_extent(log_pbn, 1, tag=tag)) + yield self.env.process(self.des_flash.erase_pbn_extent(log_pbn, 1, tag=tag)) self._phy_block_locks.release_request(log_pbn, req) @@ -1567,44 +1648,49 @@ def level_wear(self): req = self._cleaning_lock.request() yield req - victim_blocks = WearLevelingVictimBlocks(self.conf, - self.block_pool, self.oob, 0.1 * self.conf.n_blocks_per_dev, - data_block_mapping_table=self.data_block_mapping_table, - log_mapping_table=self.log_mapping_table - ) + victim_blocks = WearLevelingVictimBlocks( + self.conf, + self.block_pool, + self.oob, + 0.1 * self.conf.n_blocks_per_dev, + data_block_mapping_table=self.data_block_mapping_table, + log_mapping_table=self.log_mapping_table, + ) for valid_ratio, block_type, block_num in victim_blocks.iterator_verbose(): if valid_ratio == 0: if block_type == victim_blocks.TYPE_DATA: yield self.env.process( - self.recycle_empty_data_block(block_num, 'wearleveling')) + self.recycle_empty_data_block(block_num, "wearleveling") + ) elif block_type == victim_blocks.TYPE_LOG: - dgn, loggroup = self.log_mapping_table.find_group_by_pbn( - block_num) + dgn, loggroup = self.log_mapping_table.find_group_by_pbn(block_num) yield self.env.process( - self._recycle_empty_log_block(dgn, block_num, 'wearleveling')) + self._recycle_empty_log_block(dgn, block_num, "wearleveling") + ) else: raise RuntimeError() else: dst_pbn = self.block_pool.pop_a_free_block_to_data_blocks( - choice=MOST_ERASED) + choice=MOST_ERASED + ) if dst_pbn is None: # out of space, let's skip continue if block_type == victim_blocks.TYPE_DATA: - found, lbn = self.data_block_mapping_table.pbn_to_lbn( - block_num) + found, lbn = self.data_block_mapping_table.pbn_to_lbn(block_num) assert found == True yield self.env.process( - self._move_data_block(src_pbn=block_num, dst_pbn=dst_pbn)) + self._move_data_block(src_pbn=block_num, dst_pbn=dst_pbn) + ) elif block_type == victim_blocks.TYPE_LOG: yield self.env.process( - self._move_log_block(src_pbn=block_num, dst_pbn=dst_pbn)) + self._move_log_block(src_pbn=block_num, dst_pbn=dst_pbn) + ) else: - raise RuntimeError('Block type not recognized: {}'.format( - block_type)) + raise RuntimeError(f"Block type not recognized: {block_type}") self._cleaning_lock.release(req) @@ -1630,7 +1716,8 @@ def _move_log_block(self, src_pbn, dst_pbn): offset = src_ppn - start dst_ppn = self.conf.block_off_to_page(dst_pbn, offset) yield self.env.process( - self._move_page(src_ppn, dst_ppn, tag='wearleveling')) + self._move_page(src_ppn, dst_ppn, tag="wearleveling") + ) # update page-level mapping try: @@ -1642,8 +1729,7 @@ def _move_log_block(self, src_pbn, dst_pbn): self.logical_block_locks.release_request(lbn, req) - yield self.env.process( - self._recycle_empty_log_block(dgn, src_pbn, '')) + yield self.env.process(self._recycle_empty_log_block(dgn, src_pbn, "")) def _move_data_block(self, src_pbn, dst_pbn): """ @@ -1656,10 +1742,11 @@ def _move_data_block(self, src_pbn, dst_pbn): # copy valid pages yield self.env.process( - self._move_block_data(src_pbn, dst_pbn, tag='wearleveling')) + self._move_block_data(src_pbn, dst_pbn, tag="wearleveling") + ) # need to recyle the old data block - yield self.env.process(self.recycle_empty_data_block(src_pbn, tag='')) + yield self.env.process(self.recycle_empty_data_block(src_pbn, tag="")) # add new mapping in data block mapping self.data_block_mapping_table.add_data_block_mapping(lbn, dst_pbn) @@ -1670,51 +1757,52 @@ def _move_block_data(self, src_pbn, dst_pbn, tag): if self.oob.states.is_page_valid(src_ppn): offset = src_ppn - start dst_ppn = self.conf.block_off_to_page(dst_pbn, offset) - yield self.env.process( - self._move_page(src_ppn, dst_ppn, tag=tag)) + yield self.env.process(self._move_page(src_ppn, dst_ppn, tag=tag)) - def _move_page(self, src_ppn, dst_ppn, tag=''): + def _move_page(self, src_ppn, dst_ppn, tag=""): lpn = self.oob.translate_ppn_to_lpn(src_ppn) data = self.flash.page_read(src_ppn, cat=tag) - yield self.env.process( - self.des_flash.rw_ppns([src_ppn], 'read', tag=tag)) + yield self.env.process(self.des_flash.rw_ppns([src_ppn], "read", tag=tag)) self.flash.page_write(dst_ppn, cat=tag, data=data) - yield self.env.process( - self.des_flash.rw_ppns([dst_ppn], 'write', tag=tag)) + yield self.env.process(self.des_flash.rw_ppns([dst_ppn], "write", tag=tag)) - self.oob.remap(lpn=lpn, old_ppn=src_ppn, - new_ppn=dst_ppn) + self.oob.remap(lpn=lpn, old_ppn=src_ppn, new_ppn=dst_ppn) def assert_mapping(self, lpn): # Try log blocks log_found, log_ppn = self.translator.log_mapping_table.lpn_to_ppn(lpn) # Try data blocks - data_found, data_ppn = self.translator.data_block_mapping_table\ - .lpn_to_ppn(lpn) - - if log_found == True and self.oob.states.is_page_valid(log_ppn) and \ - data_found == True and self.oob.states.is_page_valid(data_ppn): - raise RuntimeError("lpn:{} is valid in log mapping (ppn:{}) " - "and data mapping (ppn:{})".format(lpn, log_ppn, data_ppn)) + data_found, data_ppn = self.translator.data_block_mapping_table.lpn_to_ppn(lpn) + + if ( + log_found == True + and self.oob.states.is_page_valid(log_ppn) + and data_found == True + and self.oob.states.is_page_valid(data_ppn) + ): + raise RuntimeError( + f"lpn:{lpn} is valid in log mapping (ppn:{log_ppn}) " + f"and data mapping (ppn:{data_ppn})" + ) def asserts(self): # number of log blocks in logblockinfo should be equal to # used log blocks in block_pool return log_block_cnt = 0 - for dgn, loggroupinfo in self.translator.log_mapping_table\ - .log_group_info.items(): + for dgn, loggroupinfo in list( + self.translator.log_mapping_table.log_group_info.items() + ): logblocks = loggroupinfo.log_block_numbers() log_block_cnt += len(logblocks) - assert len( - logblocks) <= self.conf['nkftl']['max_blocks_in_log_group'] + assert len(logblocks) <= self.conf["nkftl"]["max_blocks_in_log_group"] if not log_block_cnt == len(self.block_pool.log_usedblocks): raise RuntimeError( - "not log_block_cnt{} == len(self.block_pool.log_usedblocks{})" - .format(log_block_cnt, len(self.block_pool.log_usedblocks))) + f"not log_block_cnt{log_block_cnt} == len(self.block_pool.log_usedblocks{len(self.block_pool.log_usedblocks)})" + ) # number of data blocks in data_block_mapping_table should be equal to # used data blocks in block_pool @@ -1722,11 +1810,11 @@ def asserts(self): # full merge, or others, is going on. An asserts() check could be # triggered when a transaction is going on. # data_blocks_in_map = len(self.translator.data_block_mapping_table\ - # .logical_to_physical_block) + # .logical_to_physical_block) # if not data_blocks_in_map == len(self.block_pool.data_usedblocks): - # raise RuntimeError( - # "not data_blocks_in_map ({}) == len(self.block_pool.data_usedblocks) ({})"\ - # .format(data_blocks_in_map, len(self.block_pool.data_usedblocks))) + # raise RuntimeError( + # "not data_blocks_in_map ({}) == len(self.block_pool.data_usedblocks) ({})"\ + # .format(data_blocks_in_map, len(self.block_pool.data_usedblocks))) class Ftl(ftlbuilder.FtlBuilder): @@ -1736,7 +1824,7 @@ class Ftl(ftlbuilder.FtlBuilder): """ def __init__(self, confobj, recorderobj, flashobj, simpy_env, des_flash): - super(Ftl, self).__init__(confobj, recorderobj, flashobj) + super().__init__(confobj, recorderobj, flashobj) self.des_flash = des_flash self.env = simpy_env @@ -1745,23 +1833,23 @@ def __init__(self, confobj, recorderobj, flashobj, simpy_env, des_flash): n_blocks_per_channel=self.conf.n_blocks_per_channel, n_pages_per_block=self.conf.n_pages_per_block, tags=[TDATA, TLOG], - leveling_factor=self.conf['wear_leveling_factor'], - leveling_diff=self.conf['wear_leveling_diff'] + leveling_factor=self.conf["wear_leveling_factor"], + leveling_diff=self.conf["wear_leveling_diff"], ) self.oob = OutOfBandAreas(confobj) self.global_helper = GlobalHelper(confobj) - self.data_block_mapping_table = DataBlockMappingTable(confobj, - recorderobj, self.global_helper) + self.data_block_mapping_table = DataBlockMappingTable( + confobj, recorderobj, self.global_helper + ) - self.log_mapping_table = LogMappingTable(confobj, - self.block_pool, recorderobj, self.global_helper) + self.log_mapping_table = LogMappingTable( + confobj, self.block_pool, recorderobj, self.global_helper + ) self.logical_block_locks = LockPool(self.env) self.datacache = DataCache(0.9 * MB, self.conf.page_size) - self.buffer = WriteBuffer( - self.des_flash, 16*self.conf.n_pages_per_block) - + self.buffer = WriteBuffer(self.des_flash, 16 * self.conf.n_pages_per_block) ###### the managers ###### self.translator = Translator( @@ -1769,7 +1857,7 @@ def __init__(self, confobj, recorderobj, flashobj, simpy_env, des_flash): recorderobj=recorderobj, global_helper_obj=self.global_helper, log_mapping=self.log_mapping_table, - data_block_mapping=self.data_block_mapping_table + data_block_mapping=self.data_block_mapping_table, ) # Garbage collector is considered to be in the highest level @@ -1785,7 +1873,7 @@ def __init__(self, confobj, recorderobj, flashobj, simpy_env, des_flash): data_block_mapping=self.data_block_mapping_table, simpy_env=self.env, des_flash=self.des_flash, - logical_block_locks=self.logical_block_locks + logical_block_locks=self.logical_block_locks, ) self.written_bytes = 0 @@ -1804,24 +1892,33 @@ def lpn_to_ppn(self, lpn): def lba_read(self, lpn): content = yield self.env.process(self.read_ext(Extent(lpn, 1))) - self.env.exit(content[0]) + return content[0] def read_ext(self, extent): req_size = extent.lpn_count * self.conf.page_size - self.recorder.add_to_general_accumulater('traffic', 'read', req_size) + self.recorder.add_to_general_accumulater("traffic", "read", req_size) self.read_bytes += req_size if self.read_bytes > self.pre_read_bytes + self.display_interval: - print 'Read (MB)', self.pre_read_bytes / MB, 'reading', round(float(req_size) / MB, 2) + print( + ( + "Read (MB)", + old_div(self.pre_read_bytes, MB), + "reading", + round(float(req_size) / MB, 2), + ) + ) sys.stdout.flush() self.pre_read_bytes = self.read_bytes extents = split_ext(self.conf.n_pages_per_block, extent) ext_data = [] for logical_block_ext in extents: - ret_data = yield self.env.process(self.read_logical_block(logical_block_ext)) + ret_data = yield self.env.process( + self.read_logical_block(logical_block_ext) + ) ext_data.extend(ret_data) - self.env.exit(ext_data) + return ext_data def read_logical_block(self, extent): lpns_to_read = [] @@ -1849,24 +1946,33 @@ def read_logical_block(self, extent): contents.append(content) yield self.env.process( - self.des_flash.rw_ppns(ppns_to_read, 'read', tag="Unknown")) + self.des_flash.rw_ppns(ppns_to_read, "read", tag="Unknown") + ) self.logical_block_locks.release_request(block_id, req) - self.env.exit(contents) + return contents def lba_write(self, lpn, data=None): yield self.env.process( - self.write_ext(Extent(lpn_start=lpn, lpn_count=1), [data])) + self.write_ext(Extent(lpn_start=lpn, lpn_count=1), [data]) + ) yield self.env.process(self.garbage_collector.clean()) def write_ext(self, extent, data=None): req_size = extent.lpn_count * self.conf.page_size - self.recorder.add_to_general_accumulater('traffic', 'write', req_size) + self.recorder.add_to_general_accumulater("traffic", "write", req_size) self.written_bytes += req_size if self.written_bytes > self.pre_written_bytes + self.display_interval: - print 'Written (MB)', self.pre_written_bytes / MB, 'writing', round(float(req_size) / MB, 2) + print( + ( + "Written (MB)", + old_div(self.pre_written_bytes, MB), + "writing", + round(float(req_size) / MB, 2), + ) + ) sys.stdout.flush() self.pre_written_bytes = self.written_bytes for lpn in extent.lpn_iter(): @@ -1885,7 +1991,9 @@ def write_ext(self, extent, data=None): fp = tp + 1 tp = tp + 1 for singleextent in exts: - extent_data_group = split_ext(self.conf.n_pages_per_data_group(), singleextent) + extent_data_group = split_ext( + self.conf.n_pages_per_data_group(), singleextent + ) for single_data_group in extent_data_group: extents.append(single_data_group) data_group_procs = [] @@ -1894,9 +2002,11 @@ def write_ext(self, extent, data=None): data_group_data = None else: data_group_data = self._sub_ext_data( - data, extent, data_group_ext) + data, extent, data_group_ext + ) p = self.env.process( - self.write_data_group(data_group_ext, data=data_group_data)) + self.write_data_group(data_group_ext, data=data_group_data) + ) data_group_procs.append(p) yield simpy.AllOf(self.env, data_group_procs) @@ -1905,7 +2015,7 @@ def _block_iter_of_extent(self, extent): block_start, _ = self.conf.page_to_block_off(extent.lpn_start) block_last, _ = self.conf.page_to_block_off(extent.last_lpn()) - return range(block_start, block_last + 1) + return list(range(block_start, block_last + 1)) def write_data_group(self, extent, data=None): """ @@ -1914,11 +2024,9 @@ def write_data_group(self, extent, data=None): write_data_group() locks all the logical blocks it about to write then it gets all the ppns, write the data and modify the mapping """ - data_group_no = self.conf.nkftl_data_group_number_of_lpn( - extent.lpn_start) + data_group_no = self.conf.nkftl_data_group_number_of_lpn(extent.lpn_start) for lpn in extent.lpn_iter(): - assert self.conf.nkftl_data_group_number_of_lpn( - lpn) == data_group_no + assert self.conf.nkftl_data_group_number_of_lpn(lpn) == data_group_no # lock all logical blocks in this extent reqs = [] @@ -1935,30 +2043,31 @@ def write_data_group(self, extent, data=None): ppns = self.log_mapping_table.next_ppns_to_program( dgn=data_group_no, n=loop_ext.lpn_count, - strip_unit_size=self.conf['stripe_size']) + strip_unit_size=self.conf["stripe_size"], + ) n_ppns = len(ppns) - mappings = OrderedDict(zip(loop_ext.lpn_iter(), ppns)) + mappings = OrderedDict(list(zip(loop_ext.lpn_iter(), ppns))) assert len(mappings) == n_ppns if data is None: loop_data = None else: loop_data = self._sub_ext_data(data, extent, loop_ext) - yield self.env.process( - self._write_log_ppns(mappings, data=loop_data)) + yield self.env.process(self._write_log_ppns(mappings, data=loop_data)) if n_ppns < loop_ext.lpn_count: # we cannot find available pages in log blocks - for block_id, req in reversed(zip(block_ids, reqs)): + for block_id, req in reversed(list(zip(block_ids, reqs))): self.logical_block_locks.release_request(block_id, req) gc_req = self.garbage_collector._cleaning_lock.request() yield gc_req - print 'start cleaning for write_ext()' + print("start cleaning for write_ext()") yield self.env.process( - self.garbage_collector.clean_data_group(data_group_no)) + self.garbage_collector.clean_data_group(data_group_no) + ) self.garbage_collector._cleaning_lock.release(gc_req) @@ -1972,51 +2081,50 @@ def write_data_group(self, extent, data=None): loop_ext.lpn_count -= n_ppns # unlock all logical blocks - for block_id, req in reversed(zip(block_ids, reqs)): + for block_id, req in reversed(list(zip(block_ids, reqs))): self.logical_block_locks.release_request(block_id, req) def print_mappings(self, mappings): block = 1136 - for lpn, ppn in mappings.items(): + for lpn, ppn in list(mappings.items()): blk, _ = self.conf.page_to_block_off(lpn) # blk, _ = self.conf.page_to_block_off(ppn) if blk == block: - print lpn, '->', ppn + print((lpn, "->", ppn)) def write_logical_block(self, extent, data=None): block_id, _ = self.conf.page_to_block_off(extent.lpn_start) req = self.logical_block_locks.get_request(block_id) yield req - data_group_no = self.conf.nkftl_data_group_number_of_lpn( - extent.lpn_start) + data_group_no = self.conf.nkftl_data_group_number_of_lpn(extent.lpn_start) for lpn in extent.lpn_iter(): - assert self.conf.nkftl_data_group_number_of_lpn( - lpn) == data_group_no + assert self.conf.nkftl_data_group_number_of_lpn(lpn) == data_group_no loop_ext = copy.copy(extent) while loop_ext.lpn_count > 0: ppns = self.log_mapping_table.next_ppns_to_program( dgn=data_group_no, n=loop_ext.lpn_count, - strip_unit_size=self.conf['stripe_size']) + strip_unit_size=self.conf["stripe_size"], + ) n_ppns = len(ppns) - mappings = OrderedDict(zip(loop_ext.lpn_iter(), ppns)) + mappings = OrderedDict(list(zip(loop_ext.lpn_iter(), ppns))) assert len(mappings) == n_ppns if data is None: loop_data = None else: loop_data = self._sub_ext_data(data, extent, loop_ext) - yield self.env.process( - self._write_log_ppns(mappings, data=loop_data)) + yield self.env.process(self._write_log_ppns(mappings, data=loop_data)) if n_ppns < loop_ext.lpn_count: # we cannot find vailable pages in log blocks self.logical_block_locks.release_request(block_id, req) yield self.env.process( - self.garbage_collector.clean_data_group(data_group_no)) + self.garbage_collector.clean_data_group(data_group_no) + ) req = self.logical_block_locks.get_request(block_id) yield req @@ -2032,14 +2140,14 @@ def write_logical_block(self, extent, data=None): def _sub_ext_data(self, data, extent, sub_ext): start = sub_ext.lpn_start - extent.lpn_start count = sub_ext.lpn_count - sub_data = data[start:(start + count)] + sub_data = data[start : (start + count)] return sub_data def _write_log_ppns(self, mappings, data=None): """ The ppns in mappings is obtained from loggroup.next_ppns() """ - for ppn in mappings.values(): + for ppn in list(mappings.values()): assert self.oob.states.is_page_erased(ppn) # data block mapping @@ -2067,18 +2175,18 @@ def _write_log_ppns(self, mappings, data=None): # des flash yield self.env.process( - self.des_flash.rw_ppns(mappings.values(), 'write', - tag="Unknown")) + self.des_flash.rw_ppns(list(mappings.values()), "write", tag="Unknown") + ) def _update_log_mappings(self, mappings): """ The ppns in mappings must have been get by loggroup.next_ppns() """ - for lpn, ppn in mappings.items(): + for lpn, ppn in list(mappings.items()): self.log_mapping_table.add_mapping(lpn, ppn) def _remap_oob(self, new_mappings): - for lpn, new_ppn in new_mappings.items(): + for lpn, new_ppn in list(new_mappings.items()): found, old_ppn, loc = self.translator.lpn_to_ppn(lpn) if found == False: old_ppn = None @@ -2089,16 +2197,23 @@ def lba_discard(self, lpn): def discard_ext(self, extent): req_size = extent.lpn_count * self.conf.page_size - self.recorder.add_to_general_accumulater( - 'traffic', 'discard', req_size) + self.recorder.add_to_general_accumulater("traffic", "discard", req_size) self.discarded_bytes += req_size if self.discarded_bytes > self.pre_discarded_bytes + self.display_interval: - print 'Discarded (MB)', self.pre_discarded_bytes / MB, 'discarding', round(float(req_size) / MB, 2) + print( + ( + "Discarded (MB)", + old_div(self.pre_discarded_bytes, MB), + "discarding", + round(float(req_size) / MB, 2), + ) + ) sys.stdout.flush() self.pre_discarded_bytes = self.discarded_bytes - self.recorder.add_to_general_accumulater('traffic', 'discard', - extent.lpn_count*self.conf.page_size) + self.recorder.add_to_general_accumulater( + "traffic", "discard", extent.lpn_count * self.conf.page_size + ) extents = split_ext(self.conf.n_pages_per_block, extent) for logical_block_ext in extents: @@ -2126,8 +2241,8 @@ def clean(self, forced=False, merge=True): def is_wear_leveling_needed(self): factor, diff = self.block_pool.get_wear_status() - self.recorder.append_to_value_list('wear_diff', diff) - print 'ddddddddddddddddddddiiiiiiiiiiifffffffffff', diff + self.recorder.append_to_value_list("wear_diff", diff) + print(("ddddddddddddddddddddiiiiiiiiiiifffffffffff", diff)) return self.block_pool.need_wear_leveling() @@ -2139,20 +2254,21 @@ def is_cleaning_needed(self): def snapshot_erasure_count_dist(self): dist = self.block_pool.get_erasure_count_dist() - print self.env.now - print dist - self.recorder.append_to_value_list('ftl_func_erasure_count_dist', - dist) + print(self.env.now) + print(dist) + self.recorder.append_to_value_list("ftl_func_erasure_count_dist", dist) def snapshot_user_traffic(self): return - self.recorder.append_to_value_list('ftl_func_user_traffic', - {'timestamp': self.env.now/float(SEC), - 'write_traffic_size': self.written_bytes, - 'read_traffic_size': self.read_bytes, - 'discard_traffic_size': self.discarded_bytes, - }, - ) + self.recorder.append_to_value_list( + "ftl_func_user_traffic", + { + "timestamp": self.env.now / float(SEC), + "write_traffic_size": self.written_bytes, + "read_traffic_size": self.read_bytes, + "discard_traffic_size": self.discarded_bytes, + }, + ) def split_ext(n_pages_in_zone, extent): @@ -2163,7 +2279,7 @@ def split_ext(n_pages_in_zone, extent): cur_ext = None exts = [] for lpn in extent.lpn_iter(): - seg_id = lpn / n_pages_in_zone + seg_id = old_div(lpn, n_pages_in_zone) if seg_id == last_seg_id: cur_ext.lpn_count += 1 else: diff --git a/wiscsee/wiscsim/recorder.py b/wiscsee/wiscsim/recorder.py index d9ce13a..2ea50c0 100644 --- a/wiscsee/wiscsim/recorder.py +++ b/wiscsee/wiscsim/recorder.py @@ -5,28 +5,35 @@ from utilities import utils -FILE_TARGET, STDOUT_TARGET = ('file', 'stdout') +FILE_TARGET, STDOUT_TARGET = ("file", "stdout") def switchable(function): "decrator for class Recorder's method, so they can be switched on/off" + def wrapper(self, *args, **kwargs): if self.enabled == None: - raise RuntimeError("You need to explicity enable/disable Recorder." + raise RuntimeError( + "You need to explicity enable/disable Recorder." " We raise exception here because we think you will create" - " unexpected behaviors that are hard to debug.") + " unexpected behaviors that are hard to debug." + ) if self.enabled == False: return else: return function(self, *args, **kwargs) + return wrapper -class Recorder(object): - def __init__(self, output_target, - output_directory = None, - verbose_level = 1, - print_when_finished = False): +class Recorder: + def __init__( + self, + output_target, + output_directory=None, + verbose_level=1, + print_when_finished=False, + ): self.output_target = output_target self.output_directory = output_directory self.verbose_level = verbose_level @@ -34,12 +41,12 @@ def __init__(self, output_target, assert len(self.output_target) > 0 - self.file_pool = {} # {filename:descriptor} - self.file_colnames = {} # {filename:[colname1, 2, ...] + self.file_pool = {} # {filename:descriptor} + self.file_colnames = {} # {filename:[colname1, 2, ...] # {set name: collections.counter} self.general_accumulator = {} - self.result_dict = {'general_accumulator': self.general_accumulator} + self.result_dict = {"general_accumulator": self.general_accumulator} self.enabled = None @@ -48,10 +55,11 @@ def __init__(self, output_target, self._unique_num = 0 self._tag_groups = { - 'read_user': 'foreground', - 'write_user': 'foreground', - 'read_trans': 'background', - 'prog_trans': 'background'} + "read_user": "foreground", + "write_user": "foreground", + "read_trans": "background", + "prog_trans": "background", + } def close(self): self.__close_log_file() @@ -60,21 +68,21 @@ def close(self): self._close_file_pool() def enable(self): - print "....Recorder is enabled...." + print("....Recorder is enabled....") self.enabled = True def disable(self): "Note that this will not clear the previous records" - print "....Recorder is DIS-abled. Now not counting anything." + print("....Recorder is DIS-abled. Now not counting anything.") self.enabled = False def _close_file_pool(self): - for _, file_handle in self.file_pool.items(): + for _, file_handle in list(self.file_pool.items()): os.fsync(file_handle) file_handle.close() def __save_result_dict(self): - result_path = os.path.join(self.output_directory, 'recorder.json') + result_path = os.path.join(self.output_directory, "recorder.json") utils.dump_json(self.result_dict, result_path) def __close_log_file(self): @@ -84,21 +92,19 @@ def __close_log_file(self): def __open_log_file(self): # open log file - log_path = os.path.join(self.output_directory, 'recorder.log') + log_path = os.path.join(self.output_directory, "recorder.log") utils.prepare_dir_for_path(log_path) - self.log_handle = open(log_path, 'w') + self.log_handle = open(log_path, "w") def __save_accumulator(self): - counter_set_path = os.path.join(self.output_directory, - 'accumulator_table.txt') + counter_set_path = os.path.join(self.output_directory, "accumulator_table.txt") utils.prepare_dir_for_path(counter_set_path) - general_accumulator_table = self._parse_accumulator( - self.general_accumulator) + general_accumulator_table = self._parse_accumulator(self.general_accumulator) utils.table_to_file(general_accumulator_table, counter_set_path) def __write_log(self, *args): - line = ' '.join( str(x) for x in args) - line += '\n' + line = " ".join(str(x) for x in args) + line += "\n" if self.output_target == FILE_TARGET: self.log_handle.write(line) else: @@ -131,15 +137,14 @@ def count_me(self, counter_name, item): def get_count_me(self, counter_name, item): return self.get_general_accumulater_cnt(counter_name, item) - def get_general_accumulater_cnt(self, - counter_set_name, item_name): - counter_dict = self.general_accumulator.setdefault(counter_set_name, - collections.Counter()) + def get_general_accumulater_cnt(self, counter_set_name, item_name): + counter_dict = self.general_accumulator.setdefault( + counter_set_name, collections.Counter() + ) return counter_dict[item_name] @switchable - def add_to_general_accumulater(self, - counter_set_name, item_name, addition): + def add_to_general_accumulater(self, counter_set_name, item_name, addition): """ {counter set 1: {counter 1: ##, @@ -149,8 +154,9 @@ def add_to_general_accumulater(self, counter 2: #}, } """ - counter_dict = self.general_accumulator.setdefault(counter_set_name, - collections.Counter()) + counter_dict = self.general_accumulator.setdefault( + counter_set_name, collections.Counter() + ) counter_dict[item_name] += addition @switchable @@ -164,11 +170,11 @@ def get_unique_num(self): def get_tag(self, op, op_id): # return '-'.join([op, str(op_id)]) - return {'op': op, 'op_id':op_id} + return {"op": op, "op_id": op_id} def tag_group(self, tag): try: - return self._tag_groups[tag['op']] + return self._tag_groups[tag["op"]] except (KeyError, TypeError): return tag # return 'TagGroupUnknown' @@ -189,11 +195,13 @@ def _parse_accumulator(self, counter_sets): counter.name item.name count """ table = [] - for counter_set_name, counter_set in counter_sets.items(): - for counter_name, count in counter_set.items(): - d = {'counter.set.name': counter_set_name, - 'counter.name' : counter_name, - 'count' : count} + for counter_set_name, counter_set in list(counter_sets.items()): + for counter_name, count in list(counter_set.items()): + d = { + "counter.set.name": counter_set_name, + "counter.name": counter_name, + "count": count, + } table.append(d) return table @@ -206,35 +214,33 @@ def write_file(self, filename, **kwargs): provide keys in the parameter as they become columns in the file. """ width = 20 - if not self.file_pool.has_key(filename): - fd = open( os.path.join( self.output_directory, filename ), 'wr+') + if filename not in self.file_pool: + fd = open(os.path.join(self.output_directory, filename), "wr+") self.file_pool[filename] = fd - self.file_colnames[filename] = kwargs.keys() - colnames = [str(colname).rjust(width) for colname in kwargs.keys()] - fd.write(' '.join(colnames) + '\n') + self.file_colnames[filename] = list(kwargs.keys()) + colnames = [str(colname).rjust(width) for colname in list(kwargs.keys())] + fd.write(" ".join(colnames) + "\n") else: fd = self.file_pool[filename] colnames = self.file_colnames[filename] args = [str(kwargs[colname]).rjust(width) for colname in colnames] - fd.write(' '.join(args) + '\n') + fd.write(" ".join(args) + "\n") def debug(self, *args): if self.verbose_level >= 3: - self.__write_log('DEBUG', *args) + self.__write_log("DEBUG", *args) @switchable def put(self, operation, page_num, category): # do statistics - item = '.'.join((operation, category)) + item = ".".join((operation, category)) self.add_to_general_accumulater("put", item, 1) def warning(self, *args): if self.verbose_level >= 2: - self.__write_log('WARNING', *args) + self.__write_log("WARNING", *args) def error(self, *args): if self.verbose_level >= 0: - self.__write_log('ERROR', *args) - - + self.__write_log("ERROR", *args) diff --git a/wiscsee/wiscsim/sftl-standalone.py b/wiscsee/wiscsim/sftl-standalone.py index 06550ee..8f33cf2 100755 --- a/wiscsee/wiscsim/sftl-standalone.py +++ b/wiscsee/wiscsim/sftl-standalone.py @@ -1,8 +1,10 @@ #!/usr/bin/env python3 +from past.utils import old_div import random import math -from datacache import * +from .datacache import * from collections import defaultdict + SSD_capacity = 33554432 # how many pages are there in a SSD # how many LPN to PPN mapping entries a translation page could hold trans_page_entry = 2048 @@ -21,7 +23,7 @@ # events = [] # # Dict> -# format_config = {"MSR" : (1, 100, ","), "blktrace" : (512, 1000**3, " "), "systor" : (1, 1000**3, ","), "normal" : (1, 1000, " "), "FIU" : (512, 1, " ")} +# format_config = {"MSR" : (1, 100, ","), "blktrace" : (512, 1000**3, " "), "systor" : (1, 1000**3, ","), "normal" : (1, 1000, " "), "FIU" : (512, 1, " ")} # size_scale = format_config[format][0] # delimeter = format_config[format][2] @@ -63,12 +65,20 @@ # return events + def split_lpns(offset, size): - lpns = [lpn for lpn in range(int(math.floor(offset/float(page_size))), int(math.ceil((offset+size)/float(page_size))))] + lpns = [ + lpn + for lpn in range( + int(math.floor(offset / float(page_size))), + int(math.ceil((offset + size) / float(page_size))), + ) + ] return lpns -def parse_events(filename, lineno=float('inf'), format="MSR"): + +def parse_events(filename, lineno=float("inf"), format="MSR"): if "rocksdb" in filename: format = "blktrace" if "systor17" in filename: @@ -81,7 +91,14 @@ def parse_events(filename, lineno=float('inf'), format="MSR"): format = "Financial" # Dict> - format_config = {"MSR" : (1, 100, ","), "blktrace" : (512, 1000**3, " "), "systor" : (1, 1000**3, ","), "normal" : (1, 1000, " "), "FIU" : (512, 1, " "), "Financial" : (1, 1000**3, ",", 512)} + format_config = { + "MSR": (1, 100, ","), + "blktrace": (512, 1000**3, " "), + "systor": (1, 1000**3, ","), + "normal": (1, 1000, " "), + "FIU": (512, 1, " "), + "Financial": (1, 1000**3, ",", 512), + } size_scale = format_config[format][0] offset_scale = size_scale time_scale = format_config[format][1] @@ -89,7 +106,6 @@ def parse_events(filename, lineno=float('inf'), format="MSR"): if len(format_config[format]) > 3: offset_scale = format_config[format][3] - with open(filename) as fp: t_start = None last_t = 0 @@ -98,7 +114,6 @@ def parse_events(filename, lineno=float('inf'), format="MSR"): warm_up_writes = [] events = [] for i, raw in enumerate(fp): - # parse trace line = raw.strip().split(delimeter) if format == "MSR": @@ -106,9 +121,15 @@ def parse_events(filename, lineno=float('inf'), format="MSR"): t, d, offset, size, t0 = int(t), int(d), int(offset), int(size), int(t0) if format == "normal": t, d, offset, size, mode = line - t, d, offset, size, mode = int(t), int(d), int(offset), int(size), int(mode) + t, d, offset, size, mode = ( + int(t), + int(d), + int(offset), + int(size), + int(mode), + ) elif format == "blktrace": - line = filter(lambda _: _ != '', line) + line = [_ for _ in line if _ != ""] raise NotImplementedError elif format == "systor": if i == 0: @@ -116,10 +137,16 @@ def parse_events(filename, lineno=float('inf'), format="MSR"): t, t0, mode, d, offset, size = line if t0 == "": t0 = 0.0 - t, d, offset, size, t0 = float(t), int(d), int(offset), int(size), float(t0) + t, d, offset, size, t0 = ( + float(t), + int(d), + int(offset), + int(size), + float(t0), + ) elif format == "Financial": app, offset, size, mode, t = line - if int(app)!=0: + if int(app) != 0: continue t, offset, size = float(t), int(offset), int(size) elif format == "FIU": @@ -134,19 +161,21 @@ def parse_events(filename, lineno=float('inf'), format="MSR"): # scale trace offset *= offset_scale size *= size_scale - t = int(t*time_scale) + t = int(t * time_scale) if size == 0: continue - if mode in ["Read", "R", 0, 'r']: + if mode in ["Read", "R", 0, "r"]: should_warm_up = False for lpn in split_lpns(offset, size): if lpn not in exist_lpns: should_warm_up = True exist_lpns[lpn] = None if should_warm_up: - warm_up_writes += [(offset // page_size, math.ceil(size / page_size))] - elif mode in ["Write", "W", 1, 'w']: + warm_up_writes += [ + (offset // page_size, math.ceil(old_div(size, page_size))) + ] + elif mode in ["Write", "W", 1, "w"]: for lpn in split_lpns(offset, size): exist_lpns[lpn] = None @@ -154,34 +183,39 @@ def parse_events(filename, lineno=float('inf'), format="MSR"): if t < last_t: continue # events += [ControlEvent(OP_SLEEP, arg1=t - last_t)] - events += [(offset // page_size, math.ceil(size / page_size))] + events += [(offset // page_size, math.ceil(old_div(size, page_size)))] active_events += 1 last_t = t - + # termination if i > lineno: break - + # timestamp from traces might not be sorted! (now we abort the unsorted ones) # events = sorted(events, key=lambda event: event.timestamp) # for i in range(0, len(events)): # events.insert(i*2, ControlEvent(OP_SLEEP, arg1=None)) # last_t = 0 # for i in range(0, len(events), 2): - # sleep, event = events[i], events[i+1] + # sleep, event = events[i], events[i+1] # sleep.arg1 = event.timestamp - last_t # last_t = event.timestamp events = warm_up_writes + events return events + def split_write_to_page(start_lpn, size): writes = [] pages_to_write = size current_write_point = start_lpn while pages_to_write > 0: - current_write_size = min(pages_to_write, trans_page_entry * - (current_write_point // trans_page_entry) - current_write_point + trans_page_entry) + current_write_size = min( + pages_to_write, + trans_page_entry * (current_write_point // trans_page_entry) + - current_write_point + + trans_page_entry, + ) writes.append((current_write_point, current_write_size)) pages_to_write = pages_to_write - current_write_size current_write_point = current_write_size + current_write_point @@ -209,6 +243,7 @@ def __str__(self): # str([x[0] for x in enumerate(self.meta) # if x[1].is_head and not x[1].is_default]) return str(self.segment) + # def split_write_to_segment(self, n_start_lpn, size): # writes = [] # pages_to_write = size @@ -248,12 +283,19 @@ def memory_size(self): heads = [x.i for x in self.meta if x.is_head] additional_heads = [] for end_point in list(zip(*self.segment))[1]: - if end_point+1 not in heads: + if end_point + 1 not in heads: additional_heads.append(end_point) if self.has_overwrite: - return bytes_PPN * len(heads+additional_heads) + trans_page_entry * 2 / 8. + bytes_PPN - return bytes_PPN * len(heads+additional_heads) + trans_page_entry * 2 / 8. + bytes_PPN - + return ( + bytes_PPN * len(heads + additional_heads) + + trans_page_entry * 2 / 8.0 + + bytes_PPN + ) + return ( + bytes_PPN * len(heads + additional_heads) + + trans_page_entry * 2 / 8.0 + + bytes_PPN + ) # def split(self, left_split, n_lpn, left_segment=None): # if not left_split: @@ -294,8 +336,12 @@ def write(self, n_lpn, size): def _write(self, n_lpn, size): left_segment = self.find_in_segment(n_lpn) right_segment = self.find_in_segment(n_lpn + size - 1) - overwrite = (left_segment == right_segment) and ( - left_segment != -1) and (n_lpn != self.segment[left_segment][0]) and (n_lpn + size - 1 != self.segment[left_segment][1]) + overwrite = ( + (left_segment == right_segment) + and (left_segment != -1) + and (n_lpn != self.segment[left_segment][0]) + and (n_lpn + size - 1 != self.segment[left_segment][1]) + ) if overwrite: self.meta[n_lpn].is_default, self.meta[n_lpn].is_head = False, True for i in range(n_lpn + 1, n_lpn + size): @@ -308,21 +354,25 @@ def _write(self, n_lpn, size): self.meta[n_lpn].is_default, self.meta[n_lpn].is_head = True, True for i in range(n_lpn + 1, n_lpn + size): self.meta[i].is_default, self.meta[i].is_head = True, False - - if (right_segment != -1): + + if right_segment != -1: initial_segment_end = self.segment[right_segment][1] if (left_segment != -1) and (n_lpn != self.segment[left_segment][0]): - self.segment[left_segment] = ( - self.segment[left_segment][0], n_lpn - 1) + self.segment[left_segment] = (self.segment[left_segment][0], n_lpn - 1) if (right_segment != -1) and (n_lpn + size - 1 != initial_segment_end): - self.segment[right_segment] = ( - n_lpn + size, initial_segment_end) + self.segment[right_segment] = (n_lpn + size, initial_segment_end) insert_position = len([x for x in self.segment if x[1] < n_lpn]) - elements_to_delete = len([x for x in self.segment if ( - x[1] <= n_lpn + size - 1) and (x[0] >= n_lpn)]) + elements_to_delete = len( + [ + x + for x in self.segment + if (x[1] <= n_lpn + size - 1) and (x[0] >= n_lpn) + ] + ) self.segment.insert(insert_position, (n_lpn, n_lpn + size - 1)) - del self.segment[insert_position + - 1:insert_position + 1 + elements_to_delete] + del self.segment[ + insert_position + 1 : insert_position + 1 + elements_to_delete + ] # default_head_removed = False # # left_segment = self.find_in_segment(n_lpn) # # right_segment = self.find_in_segment(n_lpn + size - 1) @@ -354,6 +404,7 @@ def _write(self, n_lpn, size): # if right_should_split: # self.split(False, n_lpn + size - 1) + def group_consecutives(vals, step=1): """Return list of consecutive lists of numbers from vals (number list).""" run = [] @@ -366,8 +417,8 @@ def group_consecutives(vals, step=1): run = [v] results.append(run) expect = v + step - + for i, result in enumerate(results): results[i] = (result[0], len(result)) - return results \ No newline at end of file + return results diff --git a/wiscsee/wiscsim/sftl.py b/wiscsee/wiscsim/sftl.py index bcd10e8..2524d14 100644 --- a/wiscsee/wiscsim/sftl.py +++ b/wiscsee/wiscsim/sftl.py @@ -6,12 +6,14 @@ # trans_page_entry = 512 bytes_PPN = 4 # how many bytes a PPN takes + class MetaInfo: def __init__(self, i, is_head=None, is_default=None): self.is_head = is_head self.is_default = is_default self.i = i + class SFTLPage: def __init__(self, index, trans_page_entry): self.trans_page_entry = trans_page_entry @@ -40,7 +42,7 @@ def group_consecutives(self, vals, step=1): run = [v] results.append(run) expect = v + step - + for i, result in enumerate(results): results[i] = (result[0], len(result)) @@ -50,7 +52,7 @@ def group_mapping_into_entries(self): last_k = None groups_of_entries = [] entries = [] - for k,v in self.mapping.items(): + for k, v in list(self.mapping.items()): if not last_k or k == last_k + 1: entries.append((k, v)) else: @@ -63,9 +65,7 @@ def group_mapping_into_entries(self): groups_of_entries.append(entries) entries = [] - return groups_of_entries - def find_in_segment(self, n_lpn): left_pointer = 0 @@ -99,13 +99,17 @@ def memory(self): return 0 additional_heads = [] for end_point in list(zip(*self.segment))[1]: - if end_point+1 not in heads: + if end_point + 1 not in heads: additional_heads.append(end_point) - + # if self.has_overwrite: # return bytes_PPN * len(heads+additional_heads) + trans_page_entry * 2 / 8. + bytes_PPN - return bytes_PPN * len(heads+additional_heads) + self.trans_page_entry * 2 / 8. + bytes_PPN - + return ( + bytes_PPN * len(heads + additional_heads) + + self.trans_page_entry * 2 / 8.0 + + bytes_PPN + ) + def update(self, entries, blocknum=-1): entries = [(self.lpn_offset(lpn), ppn) for lpn, ppn in entries] for lpn, ppn in entries: @@ -115,7 +119,6 @@ def update(self, entries, blocknum=-1): for n_lpn, size in lpns: self._write(n_lpn, size) - def lookup(self, lpn, first=True): lpn = self.lpn_offset(lpn) if lpn in self.mapping: @@ -129,12 +132,12 @@ def compact(self, promote=False): def merge(self, other): for entries in self.group_mapping_into_entries(): other.update(entries) - + self.meta = other.meta self.segment = other.segment self.has_overwrite = other.has_overwrite - for k,v in other.mapping.items(): + for k, v in list(other.mapping.items()): if k not in self.mapping: self.mapping[k] = v @@ -147,8 +150,12 @@ def gc(self): def _write(self, n_lpn, size): left_segment = self.find_in_segment(n_lpn) right_segment = self.find_in_segment(n_lpn + size - 1) - overwrite = (left_segment == right_segment) and ( - left_segment != -1) and (n_lpn != self.segment[left_segment][0]) and (n_lpn + size - 1 != self.segment[left_segment][1]) + overwrite = ( + (left_segment == right_segment) + and (left_segment != -1) + and (n_lpn != self.segment[left_segment][0]) + and (n_lpn + size - 1 != self.segment[left_segment][1]) + ) if overwrite: self.meta[n_lpn].is_default, self.meta[n_lpn].is_head = False, True for i in range(n_lpn + 1, n_lpn + size): @@ -161,21 +168,25 @@ def _write(self, n_lpn, size): self.meta[n_lpn].is_default, self.meta[n_lpn].is_head = True, True for i in range(n_lpn + 1, n_lpn + size): self.meta[i].is_default, self.meta[i].is_head = True, False - - if (right_segment != -1): + + if right_segment != -1: initial_segment_end = self.segment[right_segment][1] if (left_segment != -1) and (n_lpn != self.segment[left_segment][0]): - self.segment[left_segment] = ( - self.segment[left_segment][0], n_lpn - 1) + self.segment[left_segment] = (self.segment[left_segment][0], n_lpn - 1) if (right_segment != -1) and (n_lpn + size - 1 != initial_segment_end): - self.segment[right_segment] = ( - n_lpn + size, initial_segment_end) + self.segment[right_segment] = (n_lpn + size, initial_segment_end) insert_position = len([x for x in self.segment if x[1] < n_lpn]) - elements_to_delete = len([x for x in self.segment if ( - x[1] <= n_lpn + size - 1) and (x[0] >= n_lpn)]) + elements_to_delete = len( + [ + x + for x in self.segment + if (x[1] <= n_lpn + size - 1) and (x[0] >= n_lpn) + ] + ) self.segment.insert(insert_position, (n_lpn, n_lpn + size - 1)) - del self.segment[insert_position + - 1:insert_position + 1 + elements_to_delete] + del self.segment[ + insert_position + 1 : insert_position + 1 + elements_to_delete + ] class DFTLPage: @@ -197,7 +208,7 @@ def segments(self): @property def memory(self): return 2 * bytes_PPN * len(self.mapping) - + def update(self, entries, blocknum=-1): for lpn, ppn in entries: self.mapping[lpn] = ppn @@ -212,7 +223,7 @@ def compact(self, promote=False): pass def merge(self, other): - for k,v in other.mapping.items(): + for k, v in list(other.mapping.items()): if k not in self.mapping: self.mapping[k] = v @@ -222,23 +233,26 @@ def promote(self): def gc(self): pass + if __name__ == "__main__": - trans_pages = dict() - entries = [(i,i) for i in range(256)] - page_no = 0 - - trans_pages = {i : SFTLPage(i) for i in range(2)} + entries = [(i, i) for i in range(256)] + page_no = 0 + + trans_pages = {i: SFTLPage(i) for i in range(2)} trans_pages[0].update(entries, 0) trans_pages[1].update(entries[:128], 1) trans_pages[0].merge(trans_pages[1]) # print(trans_pages[singlewrite[0] // trans_page_entry]) print("SFTL Memory consumption in Byte is:") - print(sum([x.memory for x in trans_pages.values()])) - print(len([x for x in trans_pages.values() if x.has_overwrite]) / float(len([x for x in trans_pages.values()]))) + print(sum([x.memory for x in list(trans_pages.values())])) + print( + len([x for x in list(trans_pages.values()) if x.has_overwrite]) + / float(len([x for x in list(trans_pages.values())])) + ) page_no, page = list(trans_pages.keys())[0], list(trans_pages.values())[0] print(page_no) print(page.segment) # print([x.i for x in page.meta if x.is_head]) - # print(len(list(trans_pages.values()))) \ No newline at end of file + # print(len(list(trans_pages.values()))) diff --git a/wiscsee/wiscsim/simulator.py b/wiscsee/wiscsim/simulator.py index 3f73c3a..dff3674 100755 --- a/wiscsee/wiscsim/simulator.py +++ b/wiscsee/wiscsim/simulator.py @@ -9,17 +9,17 @@ import pprint import config -import ssdframework -import dftlext -import flash -import nkftl2 -import recorder -import hostevent -import dftldes -import ftlcounter +from . import ssdframework +from . import dftlext +from . import flash +from . import nkftl2 +from . import recorder +from . import hostevent +from . import dftldes +from . import ftlcounter from commons import * -from ftlsim_commons import * +from .ftlsim_commons import * from .host import Host from utilities import utils @@ -27,9 +27,8 @@ from pyreuse.fsutils import ext4dumpextents from .gc_analysis import GcLog -class Simulator(object): - __metaclass__ = abc.ABCMeta +class Simulator(metaclass=abc.ABCMeta): @abc.abstractmethod def run(self): return @@ -41,31 +40,32 @@ def get_sim_type(self): def __init__(self, conf, event_iter): "conf is class Config" if not isinstance(conf, config.Config): - raise TypeError("conf is not config.Config, it is {}". - format(type(conf).__name__)) + raise TypeError(f"conf is not config.Config, it is {type(conf).__name__}") self.conf = conf self.event_iter = event_iter # initialize recorder - self.recorder = recorder.Recorder(output_target = self.conf['output_target'], - output_directory = self.conf['result_dir'], - verbose_level = self.conf['verbose_level'], - print_when_finished = self.conf['print_when_finished'] - ) - - if self.conf.has_key('enable_e2e_test'): + self.recorder = recorder.Recorder( + output_target=self.conf["output_target"], + output_directory=self.conf["result_dir"], + verbose_level=self.conf["verbose_level"], + print_when_finished=self.conf["print_when_finished"], + ) + + if "enable_e2e_test" in self.conf: raise RuntimeError("enable_e2e_test is deprecated") class SimulatorDESNew(Simulator): def __init__(self, conf, event_iter): - super(SimulatorDESNew, self).__init__(conf, event_iter) + super().__init__(conf, event_iter) self.env = simpy.Environment() self.host = Host(self.conf, self.env, event_iter) - self.ssd = ssdframework.Ssd(self.conf, self.env, - self.host.get_ncq(), self.recorder) + self.ssd = ssdframework.Ssd( + self.conf, self.env, self.host.get_ncq(), self.recorder + ) def run(self): self.env.process(self.host.run()) @@ -79,19 +79,21 @@ def get_sim_type(self): return "SimulatorDESNew" def record_post_run_stats(self): - self.recorder.set_result_by_one_key( - 'simulation_duration', self.env.now) - #pprint.pprint(self.recorder.get_result_summary()) + self.recorder.set_result_by_one_key("simulation_duration", self.env.now) + # pprint.pprint(self.recorder.get_result_summary()) self.recorder.close() - gclog = GcLog(device_path=self.conf['device_path'], - result_dir=self.conf['result_dir'], - flash_page_size=self.conf.page_size - ) - if self.conf['filesystem'] == 'ext4' and \ - os.path.exists(gclog.gclog_path) and \ - os.path.exists(gclog.extents_path): + gclog = GcLog( + device_path=self.conf["device_path"], + result_dir=self.conf["result_dir"], + flash_page_size=self.conf.page_size, + ) + if ( + self.conf["filesystem"] == "ext4" + and os.path.exists(gclog.gclog_path) + and os.path.exists(gclog.extents_path) + ): gclog.classify_lpn_in_gclog() @@ -102,28 +104,28 @@ def create_simulator(simulator_class, conf, event_iter): def random_data(addr): randnum = random.randint(0, 10000) - content = "{}.{}".format(addr, randnum) + content = f"{addr}.{randnum}" return content -class SimulatorNonDES(Simulator): - __metaclass__ = abc.ABCMeta - +class SimulatorNonDES(Simulator, metaclass=abc.ABCMeta): def __init__(self, conf, event_iter): - super(SimulatorNonDES, self).__init__(conf, event_iter) + super().__init__(conf, event_iter) - if self.conf['ftl_type'] == 'dftlext': + if self.conf["ftl_type"] == "dftlext": ftl_class = dftlext.Dftl - elif self.conf['ftl_type'] == 'nkftl2': + elif self.conf["ftl_type"] == "nkftl2": ftl_class = nkftl2.Ftl - elif self.conf['ftl_type'] == 'ftlcounter': + elif self.conf["ftl_type"] == "ftlcounter": ftl_class = ftlcounter.Ftl else: - raise ValueError("ftl_type {} is not defined"\ - .format(self.conf['ftl_type'])) + raise ValueError("ftl_type {} is not defined".format(self.conf["ftl_type"])) - self.ftl = ftl_class(self.conf, self.recorder, - flash.Flash(recorder = self.recorder, confobj = self.conf)) + self.ftl = ftl_class( + self.conf, + self.recorder, + flash.Flash(recorder=self.recorder, confobj=self.conf), + ) def run(self): """ @@ -134,7 +136,7 @@ def run(self): self.process_event(event) cnt += 1 if cnt % 5000 == 0: - print '|', + print("|", end=" ") sys.stdout.flush() self.ftl.post_processing() @@ -142,7 +144,7 @@ def run(self): self.recorder.close() def process_event(self, event): - if event.action != 'D': + if event.action != "D": return if event.operation == OP_READ: @@ -157,15 +159,20 @@ def process_event(self, event): self.ftl.disable_recording() elif event.operation == OP_WORKLOADSTART: self.ftl.pre_workload() - elif event.operation in ['finish', OP_BARRIER, OP_REC_TIMESTAMP, OP_CLEAN, - OP_NOOP]: + elif event.operation in [ + "finish", + OP_BARRIER, + OP_REC_TIMESTAMP, + OP_CLEAN, + OP_NOOP, + ]: # ignore this pass else: pass # print event # raise RuntimeError("operation '{}' is not supported".format( - # event.operation)) + # event.operation)) class SimulatorNonDESSpeed(SimulatorNonDES): @@ -173,27 +180,21 @@ class SimulatorNonDESSpeed(SimulatorNonDES): This one does not do e2e test It uses extents """ + def get_sim_type(self): return "NonDESSpeed" def write(self, event): - self.ftl.sec_write( - sector = event.sector, - count = event.sector_count, - data = None) + self.ftl.sec_write(sector=event.sector, count=event.sector_count, data=None) def read(self, event): """ read extent from flash and check if the data is correct. """ - self.ftl.sec_read( - sector = event.sector, - count = event.sector_count) + self.ftl.sec_read(sector=event.sector, count=event.sector_count) def discard(self, event): - self.ftl.sec_discard( - sector = event.sector, - count = event.sector_count) + self.ftl.sec_discard(sector=event.sector, count=event.sector_count) class SimulatorNonDESe2e(SimulatorNonDES): @@ -201,8 +202,9 @@ class SimulatorNonDESe2e(SimulatorNonDES): This one does not do e2e test It uses extents """ + def __init__(self, conf, event_iter): - super(SimulatorNonDESe2e, self).__init__(conf, event_iter) + super().__init__(conf, event_iter) self.lsn_to_data = {} @@ -221,37 +223,30 @@ def write(self, event): self.lsn_to_data[sec] = content data.append(content) - self.ftl.sec_write( - sector = event.sector, - count = event.sector_count, - data = data) + self.ftl.sec_write(sector=event.sector, count=event.sector_count, data=data) def read(self, event): """ read extent from flash and check if the data is correct. """ - data = self.ftl.sec_read( - sector = event.sector, - count = event.sector_count) + data = self.ftl.sec_read(sector=event.sector, count=event.sector_count) self.check_read(event, data) def check_read(self, event, data): for sec, sec_data in zip( - range(event.sector, event.sector + event.sector_count), data): + list(range(event.sector, event.sector + event.sector_count)), data + ): if self.lsn_to_data.get(sec, None) != sec_data: - msg = "Data is not correct. Got: {read}, "\ - "Correct: {correct}. sector={sec}".format( - read = sec_data, - correct = self.lsn_to_data.get(sec, None), - sec = sec) - print msg + msg = ( + f"Data is not correct. Got: {sec_data}, " + f"Correct: {self.lsn_to_data.get(sec, None)}. sector={sec}" + ) + print(msg) # raise RuntimeError(msg) def discard(self, event): - self.ftl.sec_discard( - sector = event.sector, - count = event.sector_count) + self.ftl.sec_discard(sector=event.sector, count=event.sector_count) for sec in range(event.sector, event.sector + event.sector_count): try: @@ -265,7 +260,7 @@ def __init__(self, conf, event_iters): """ event_iters is list of event iterators """ - super(SimulatorDESSync, self).__init__(conf, None) + super().__init__(conf, None) if not isinstance(event_iters, list): raise RuntimeError("event_iters must be a list of iterators.") @@ -273,7 +268,9 @@ def __init__(self, conf, event_iters): self.event_iters = event_iters self.env = simpy.Environment() - self.ssdframework = ssdframework.SSDFramework(self.conf, self.recorder, self.env) + self.ssdframework = ssdframework.SSDFramework( + self.conf, self.recorder, self.env + ) def host_proc(self, pid, event_iter): """ @@ -282,8 +279,7 @@ def host_proc(self, pid, event_iter): # this token is acquired before we issue request to queue. # it effectively control the queue depth of this process - token = simpy.Resource(self.env, - capacity = self.conf['process_queue_depth']) + token = simpy.Resource(self.env, capacity=self.conf["process_queue_depth"]) for event in event_iter: event.token = token @@ -293,7 +289,7 @@ def host_proc(self, pid, event_iter): yield self.ssdframework.ncq.queue.put(event) - for i in range(self.conf['SSDFramework']['ncq_depth']): + for i in range(self.conf["SSDFramework"]["ncq_depth"]): event = hostevent.ControlEvent(OP_SHUT_SSD) event.token = token @@ -321,6 +317,3 @@ def read(self): def discard(self): raise NotImplementedError() - - - diff --git a/wiscsee/wiscsim/ssdframework.py b/wiscsee/wiscsim/ssdframework.py index 5830e00..74a8391 100644 --- a/wiscsee/wiscsim/ssdframework.py +++ b/wiscsee/wiscsim/ssdframework.py @@ -1,3 +1,4 @@ +from past.utils import old_div import bitarray from collections import deque, Counter, defaultdict import datetime @@ -5,7 +6,7 @@ import random import os import pprint -import Queue +import queue import sys import simpy import copy @@ -15,22 +16,23 @@ import config from commons import * -from ftlsim_commons import * -import flash -import controller -import ftlbuilder -import hostevent -import lrulist -import recorder +from .ftlsim_commons import * +from . import flash +from . import controller +from . import ftlbuilder +from . import hostevent +from . import lrulist +from . import recorder from utilities import utils -import dftldes -import nkftl2 -import learnedftl +from . import dftldes +from . import nkftl2 +from . import learnedftl from wiscsim.utils import * from pyreuse.sysutils import blocktrace, blockclassifiers, dumpe2fsparser -class SsdBase(object): + +class SsdBase: def _process(self, pid): raise NotImplementedError() @@ -43,41 +45,49 @@ def __init__(self, conf, simpy_env, ncq, rec_obj): self.conf = conf self.env = simpy_env self.recorder = rec_obj - self.ncq = ncq # should be initialized in Simulator + self.ncq = ncq # should be initialized in Simulator self.n_processes = self.ncq.ncq_depth self.flash_controller = controller.Controller3( - self.env, self.conf, self.recorder) + self.env, self.conf, self.recorder + ) - log_msg('initializing ssd...........', self.conf['ftl_type']) + log_msg("initializing ssd...........", self.conf["ftl_type"]) self.ftl = self._create_ftl() - self._snapshot_valid_ratios = self.conf['snapshot_valid_ratios'] - self._snapshot_erasure_count_dist = self.conf['snapshot_erasure_count_dist'] - self._snapshot_interval = self.conf['snapshot_interval'] + self._snapshot_valid_ratios = self.conf["snapshot_valid_ratios"] + self._snapshot_erasure_count_dist = self.conf["snapshot_erasure_count_dist"] + self._snapshot_interval = self.conf["snapshot_interval"] self._snapshot_user_traffic = True - self._do_wear_leveling = self.conf['do_wear_leveling'] - self._wear_leveling_check_interval = self.conf['wear_leveling_check_interval'] + self._do_wear_leveling = self.conf["do_wear_leveling"] + self._wear_leveling_check_interval = self.conf["wear_leveling_check_interval"] self.gc_sleep_timer = 0 self.gc_sleep_duration = 10 def _create_ftl(self): - if self.conf['ftl_type'] == 'dftldes': - return dftldes.Ftl(self.conf, self.recorder, self.flash_controller, - self.env) - elif self.conf['ftl_type'] == 'nkftl2': - print 'we will use nkftl2' + if self.conf["ftl_type"] == "dftldes": + return dftldes.Ftl( + self.conf, self.recorder, self.flash_controller, self.env + ) + elif self.conf["ftl_type"] == "nkftl2": + print("we will use nkftl2") simpleflash = flash.Flash(recorder=self.recorder, confobj=self.conf) - return nkftl2.Ftl(self.conf, self.recorder, simpleflash, self.env, - self.flash_controller) - elif self.conf['ftl_type'] == 'learnedftl': + return nkftl2.Ftl( + self.conf, self.recorder, simpleflash, self.env, self.flash_controller + ) + elif self.conf["ftl_type"] == "learnedftl": simpleflash = flash.Flash(recorder=self.recorder, confobj=self.conf) - return learnedftl.Ftl(self.conf, self.recorder, simpleflash, self.env, - self.flash_controller, self.ncq) - + return learnedftl.Ftl( + self.conf, + self.recorder, + simpleflash, + self.env, + self.flash_controller, + self.ncq, + ) def _barrier(self): """ @@ -102,7 +112,7 @@ def _process(self, pid): if len(self.ncq.queue) == 0: break host_event, _ = self.ncq.queue.popitem(last=False) - #print(host_event) + # print(host_event) slot_req = self.ncq.slots.request() yield slot_req @@ -122,7 +132,7 @@ def _process(self, pid): pass elif operation == OP_SHUT_SSD: - print 'got shut_ssd' + print("got shut_ssd") sys.stdout.flush() self.ftl.end_ssd() self._end_all_processes() @@ -149,86 +159,95 @@ def _process(self, pid): pass elif operation == OP_CALC_GC_DURATION: - dur = self.recorder.get_result_by_one_key('gc_end') - \ - self.recorder.get_result_by_one_key('gc_start') - self.recorder.set_result_by_one_key('gc_duration', dur) - self.recorder.set_result_by_one_key('gc_duration_sec', dur/SEC) + dur = self.recorder.get_result_by_one_key( + "gc_end" + ) - self.recorder.get_result_by_one_key("gc_start") + self.recorder.set_result_by_one_key("gc_duration", dur) + self.recorder.set_result_by_one_key( + "gc_duration_sec", old_div(dur, SEC) + ) elif operation == OP_CALC_NON_MERGE_GC_DURATION: - dur = self.recorder.get_result_by_one_key('non_merge_gc_end') - \ - self.recorder.get_result_by_one_key('non_merge_gc_start') - self.recorder.set_result_by_one_key('non_merge_gc_duration', dur) - self.recorder.set_result_by_one_key('non_merge_gc_duration_sec', dur/SEC) + dur = self.recorder.get_result_by_one_key( + "non_merge_gc_end" + ) - self.recorder.get_result_by_one_key("non_merge_gc_start") + self.recorder.set_result_by_one_key("non_merge_gc_duration", dur) + self.recorder.set_result_by_one_key( + "non_merge_gc_duration_sec", old_div(dur, SEC) + ) elif operation == OP_FLUSH_TRANS_CACHE: - if self.conf['ftl_type'] == 'dftldes': + if self.conf["ftl_type"] == "dftldes": yield self.env.process(self.ftl.flush_trans_cache()) elif operation == OP_PURGE_TRANS_CACHE: - if self.conf['ftl_type'] == 'dftldes': + if self.conf["ftl_type"] == "dftldes": yield self.env.process(self.ftl.purge_trans_cache()) elif operation == OP_DROP_TRANS_CACHE: - if self.conf['ftl_type'] == 'dftldes': + if self.conf["ftl_type"] == "dftldes": self.ftl.drop_trans_cache() elif operation == OP_REC_TIMESTAMP: - self.recorder.set_result_by_one_key(host_event.arg1, - self.env.now) + self.recorder.set_result_by_one_key(host_event.arg1, self.env.now) elif operation == OP_REC_FLASH_OP_CNT: result_dict = self.recorder.get_result_summary() flashops = copy.deepcopy( - result_dict['general_accumulator'].get('flash_ops', {})) - self.recorder.set_result_by_one_key(host_event.arg1, - flashops) + result_dict["general_accumulator"].get("flash_ops", {}) + ) + self.recorder.set_result_by_one_key(host_event.arg1, flashops) elif operation == OP_REC_FOREGROUND_OP_CNT: result_dict = self.recorder.get_result_summary() traffic = copy.deepcopy( - result_dict['general_accumulator'].get('traffic', {})) - self.recorder.set_result_by_one_key(host_event.arg1, - traffic) + result_dict["general_accumulator"].get("traffic", {}) + ) + self.recorder.set_result_by_one_key(host_event.arg1, traffic) elif operation == OP_REC_CACHE_HITMISS: result_dict = self.recorder.get_result_summary() data = copy.deepcopy( - result_dict['general_accumulator'].get('Mapping_Cache', {})) - self.recorder.set_result_by_one_key(host_event.arg1, - data) + result_dict["general_accumulator"].get("Mapping_Cache", {}) + ) + self.recorder.set_result_by_one_key(host_event.arg1, data) elif operation == OP_END_SSD_PROCESS: self.ncq.slots.release(slot_req) break elif operation == OP_CLEAN: - print 'start cleaning' + print("start cleaning") yield self.env.process(self._cleaner_process(forced=True)) elif operation == OP_REC_BW: - dur = self.recorder.get_result_by_one_key('interest_workload_end') - \ - self.recorder.get_result_by_one_key('interest_workload_start') - self.recorder.set_result_by_one_key('workload_duration_nsec', dur) - self.recorder.set_result_by_one_key('workload_duration_sec', float(dur)/SEC) + dur = self.recorder.get_result_by_one_key( + "interest_workload_end" + ) - self.recorder.get_result_by_one_key("interest_workload_start") + self.recorder.set_result_by_one_key("workload_duration_nsec", dur) + self.recorder.set_result_by_one_key( + "workload_duration_sec", float(dur) / SEC + ) - write_traffic = self.recorder.get_general_accumulater_cnt( - 'traffic', 'write') / MB + write_traffic = old_div( + self.recorder.get_general_accumulater_cnt("traffic", "write"), MB + ) - self.recorder.set_result_by_one_key('workload_duration_nsec', dur) + self.recorder.set_result_by_one_key("workload_duration_nsec", dur) if dur == 0: write_bw = "NA" else: - write_bw = float(write_traffic)/(float(dur) / SEC) + write_bw = float(write_traffic) / (float(dur) / SEC) - self.recorder.set_result_by_one_key('write_bandwidth', write_bw) - print '>>>>>>>>>> Bandwidth (MB/s) <<<<<<<<<<<', write_bw - print '>>>>>>>>>> Traffic (MB) <<<<<<<<<<<', write_traffic - print '>>>>>>>>>> Duration (sec) <<<<<<<<<<<', float(dur) / SEC + self.recorder.set_result_by_one_key("write_bandwidth", write_bw) + print((">>>>>>>>>> Bandwidth (MB/s) <<<<<<<<<<<", write_bw)) + print((">>>>>>>>>> Traffic (MB) <<<<<<<<<<<", write_traffic)) + print((">>>>>>>>>> Duration (sec) <<<<<<<<<<<", float(dur) / SEC)) elif operation == OP_NON_MERGE_CLEAN: - print 'start non merge cleaning' - if self.conf['ftl_type'] == 'nkftl2': + print("start non merge cleaning") + if self.conf["ftl_type"] == "nkftl2": yield self.env.process(self.ftl.clean(forced=True, merge=False)) elif operation == OP_READ: @@ -238,7 +257,9 @@ def _process(self, pid): # procs.append(proc) # procs_to_events[proc] = (host_event, OP_READ) - yield self.env.process(self.ftl.read_ext(host_event.get_lpn_extent(self.conf))) + yield self.env.process( + self.ftl.read_ext(host_event.get_lpn_extent(self.conf)) + ) elif operation == OP_WRITE: # rw_events[host_event].append('write') @@ -247,8 +268,9 @@ def _process(self, pid): # procs.append(proc) # procs_to_events[proc] = (host_event, OP_WRITE) - yield self.env.process(self.ftl.write_ext(host_event.get_lpn_extent(self.conf))) - + yield self.env.process( + self.ftl.write_ext(host_event.get_lpn_extent(self.conf)) + ) # elif operation == OP_SLEEP: # procs += [self.env.timeout(delay=host_event.arg1, value="sleep")] @@ -268,16 +290,18 @@ def _process(self, pid): # # procs_to_events.pop(proc) # procs.remove(proc) - elif operation == OP_DISCARD: + elif operation == OP_DISCARD: yield self.env.process( - self.ftl.discard_ext(host_event.get_lpn_extent(self.conf))) + self.ftl.discard_ext(host_event.get_lpn_extent(self.conf)) + ) elif operation in [OP_FALLOCATE]: pass else: - raise NotImplementedError("Operation {} not supported."\ - .format(host_event.operation)) + raise NotImplementedError( + f"Operation {host_event.operation} not supported." + ) # if req_i >= 10000: # exit(0) @@ -310,7 +334,7 @@ def _process(self, pid): # # rw_events[event].append(self.env.now) # # procs_to_events.pop(proc) # procs.remove(proc) - + # TODO: move this to FTL # import math # read_latencies = [] @@ -329,8 +353,8 @@ def _process(self, pid): # if mode == 'write': # write_latencies.append(per_page_latency) - # fp.write("%s %d %d %d %d %.3f\n" % (mode, event.size, event.offset, start_time, end_time, per_page_latency)) - # fp.write("%d %d %d %d\n" % (event.size, event.offset, start_time, end_time)) + # fp.write("%s %d %d %d %d %.3f\n" % (mode, event.size, event.offset, start_time, end_time, per_page_latency)) + # fp.write("%d %d %d %d\n" % (event.size, event.offset, start_time, end_time)) # print(read_events) # def plot_ecdf(arr, path): @@ -341,7 +365,6 @@ def _process(self, pid): # plt.grid(True) # fig.savefig(path) - # if len(read_latencies) > 0: # log_msg("End-to-end read response time per page: %.2fus" % np.average(read_latencies)) # if len(write_latencies) > 0: @@ -349,8 +372,7 @@ def _process(self, pid): def _end_all_processes(self): for i in range(self.n_processes): - self.ncq.queue[ - hostevent.ControlEvent(OP_END_SSD_PROCESS)] = "" + self.ncq.queue[hostevent.ControlEvent(OP_END_SSD_PROCESS)] = "" self._snapshot_valid_ratios = False self._snapshot_erasure_count_dist = False self._do_wear_leveling = False @@ -362,16 +384,15 @@ def _cleaner_process(self, forced=False): yield self.env.process(self.ftl.clean(forced)) def _wear_leveling_process(self): - log_msg('wear leveling process start') + log_msg("wear leveling process start") while self._do_wear_leveling is True: yield self.env.timeout(self._wear_leveling_check_interval) if self.ftl.is_wear_leveling_needed() is True: - log_msg('start wear leveling...') + log_msg("start wear leveling...") yield self.env.process(self.ftl.level_wear()) else: - log_msg('skip wear leveling') - log_msg('wear leveling process ends') - + log_msg("skip wear leveling") + log_msg("wear leveling process ends") def _valid_ratio_snapshot_process(self): while self._snapshot_valid_ratios is True: @@ -381,7 +402,7 @@ def _valid_ratio_snapshot_process(self): def _user_traffic_size_snapshot_process(self): while self._snapshot_user_traffic is True: self.ftl.snapshot_user_traffic() - yield self.env.timeout(0.1*SEC) + yield self.env.timeout(0.1 * SEC) def _erasure_count_dist_snapshot_process(self): while self._snapshot_erasure_count_dist is True: @@ -391,63 +412,25 @@ def _erasure_count_dist_snapshot_process(self): def run(self): procs = [] for i in range(self.n_processes): - p = self.env.process( self._process(i) ) + p = self.env.process(self._process(i)) procs.append(p) - p = self.env.process( self._valid_ratio_snapshot_process() ) + p = self.env.process(self._valid_ratio_snapshot_process()) procs.append(p) - p = self.env.process( self._erasure_count_dist_snapshot_process() ) + p = self.env.process(self._erasure_count_dist_snapshot_process()) procs.append(p) - p = self.env.process( self._wear_leveling_process() ) + p = self.env.process(self._wear_leveling_process()) procs.append(p) - p = self.env.process( self._user_traffic_size_snapshot_process() ) + p = self.env.process(self._user_traffic_size_snapshot_process()) procs.append(p) yield simpy.events.AllOf(self.env, procs) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -class SSDFramework(object): +class SSDFramework: """ The interface of this FTL for the host is a queue (NCQ). The host puts requests to the queue with certain time intervals according to the @@ -458,24 +441,28 @@ class SSDFramework(object): realftl should provide clean_garbage() """ + def __init__(self, confobj, recorderobj, simpy_env): self.conf = confobj self.recorder = recorderobj self.env = simpy_env self.ncq = NCQSingleQueue( - ncq_depth = self.conf['SSDFramework']['ncq_depth'], - simpy_env = self.env) + ncq_depth=self.conf["SSDFramework"]["ncq_depth"], simpy_env=self.env + ) self.flash_controller = controller.Controller3( - self.env, self.conf, self.recorder) + self.env, self.conf, self.recorder + ) - if self.conf['ftl_type'] == 'dftldes': - self.realftl = dftldes.Dftl(self.conf, self.recorder, - self.flash_controller, self.env) + if self.conf["ftl_type"] == "dftldes": + self.realftl = dftldes.Dftl( + self.conf, self.recorder, self.flash_controller, self.env + ) else: - raise RuntimeError("ftl_type {} is not supported.".format( - self.conf['ftl_type'])) + raise RuntimeError( + "ftl_type {} is not supported.".format(self.conf["ftl_type"]) + ) self.recorder.disable() @@ -488,8 +475,8 @@ def access_flash(self, flash_reqs): for flash_req in flash_reqs: # This will have the requests automatically queued at channel p = self.env.process( - self.flash_controller.execute_request(flash_req, - tag = TAG_FOREGROUND)) + self.flash_controller.execute_request(flash_req, tag=TAG_FOREGROUND) + ) ctrl_procs.append(p) all_ctrl_procs = simpy.events.AllOf(self.env, ctrl_procs) @@ -508,8 +495,7 @@ def process(self, pid): if host_event.operation == OP_ENABLE_RECORDER: self.realftl.recorder.enable() - self.recorder.set_result_by_one_key('workload_start_time', - self.env.now) + self.recorder.set_result_by_one_key("workload_start_time", self.env.now) self.release_token(host_event) continue @@ -521,71 +507,75 @@ def process(self, pid): # Translate for the reqests s = self.env.now - flash_reqs = yield self.env.process( - self.realftl.translate(ssd_req, pid) ) + flash_reqs = yield self.env.process(self.realftl.translate(ssd_req, pid)) e = self.env.now - self.recorder.add_to_timer("translation_time-w_wait", pid, - e - s) + self.recorder.add_to_timer("translation_time-w_wait", pid, e - s) # Access flash s = self.env.now - yield self.env.process( - self.access_flash(flash_reqs)) + yield self.env.process(self.access_flash(flash_reqs)) e = self.env.now - self.recorder.add_to_timer("forground_flash_access_time-w_wait", pid, - e - s) + self.recorder.add_to_timer("forground_flash_access_time-w_wait", pid, e - s) # Try clean garbage - self.env.process( - self.realftl.clean_garbage()) + self.env.process(self.realftl.clean_garbage()) if pid == 0 and req_index % 100 == 0: - print self.env.now / float(SEC) + print(self.env.now / float(SEC)) self.release_token(host_event) def run(self): procs = [] - for i in range(self.conf['SSDFramework']['ncq_depth']): - p = self.env.process( self.process(i) ) + for i in range(self.conf["SSDFramework"]["ncq_depth"]): + p = self.env.process(self.process(i)) procs.append(p) e = simpy.events.AllOf(self.env, procs) yield e - self.recorder.set_result_by_one_key( - "simulation_duration", self.env.now) + self.recorder.set_result_by_one_key("simulation_duration", self.env.now) self.record_blkparse_bw() self.print_statistics() def record_blkparse_bw(self): raw_blkparse_file_path = os.path.join( - self.conf['result_dir'], 'blkparse-output.txt') + self.conf["result_dir"], "blkparse-output.txt" + ) if not os.path.exists(raw_blkparse_file_path): return blkresult = blocktrace.BlktraceResult( - self.conf['sector_size'], - self.conf['event_file_column_names'], - raw_blkparse_file_path, None) + self.conf["sector_size"], + self.conf["event_file_column_names"], + raw_blkparse_file_path, + None, + ) self.recorder.set_result_by_one_key( - 'blkparse_read_bw', - blkresult.get_bandwidth_mb('read')) + "blkparse_read_bw", blkresult.get_bandwidth_mb("read") + ) self.recorder.set_result_by_one_key( - 'blkparse_write_bw', - blkresult.get_bandwidth_mb('write')) + "blkparse_write_bw", blkresult.get_bandwidth_mb("write") + ) self.recorder.set_result_by_one_key( - 'blkparse_duration', - blkresult.get_duration()) + "blkparse_duration", blkresult.get_duration() + ) def print_statistics(self): - print '++++++++++++++++++++ statistics ++++++++++++++++++' - print 'sim duration', self.recorder.result_dict['simulation_duration'] - print 'workload start', self.recorder.result_dict['workload_start_time'] - print 'workload duration', \ - self.recorder.result_dict['simulation_duration'] - \ - self.recorder.result_dict['workload_start_time'] - print 'blkparse_duration', self.recorder.result_dict.get( - 'blkparse_duration', None) - + print("++++++++++++++++++++ statistics ++++++++++++++++++") + print(("sim duration", self.recorder.result_dict["simulation_duration"])) + print(("workload start", self.recorder.result_dict["workload_start_time"])) + print( + ( + "workload duration", + self.recorder.result_dict["simulation_duration"] + - self.recorder.result_dict["workload_start_time"], + ) + ) + print( + ( + "blkparse_duration", + self.recorder.result_dict.get("blkparse_duration", None), + ) + ) diff --git a/wiscsee/wiscsim/tagblockpool.py b/wiscsee/wiscsim/tagblockpool.py index 206f619..9565ad8 100644 --- a/wiscsee/wiscsim/tagblockpool.py +++ b/wiscsee/wiscsim/tagblockpool.py @@ -1,18 +1,18 @@ from collections import Counter -TFREE = 'TAGFREE' +TFREE = "TAGFREE" -LEAST_ERASED = 'least' -MOST_ERASED = 'most' +LEAST_ERASED = "least" +MOST_ERASED = "most" -class TagBlockPool(object): +class TagBlockPool: def __init__(self, n, tags): - self._tag_subpool = {tag:[] for tag in tags} - self._tag_subpool[TFREE] = range(n) + self._tag_subpool = {tag: [] for tag in tags} + self._tag_subpool[TFREE] = list(range(n)) # {blocknum: count} - self._erasure_cnt = {tag:Counter() for tag in tags} + self._erasure_cnt = {tag: Counter() for tag in tags} # have to put the block number in the counter # otherwise, if a free block is never used, it won't # appear in the counter. @@ -51,7 +51,7 @@ def pick_and_move(self, src, dst, choice=LEAST_ERASED): def get_erasure_count(self, blocknum=None): all_cnt = dict() - for tag, cnt in self._erasure_cnt.items(): + for tag, cnt in list(self._erasure_cnt.items()): all_cnt.update(cnt) if blocknum is None: return self.all_cnt @@ -89,10 +89,10 @@ def get_least_or_most_erased_blocks(self, tag, choice, nblocks): return blocks def get_erasure_count_dist(self): - return Counter(self.get_erasure_count().values()) + return Counter(list(self.get_erasure_count().values())) -class CurrentBlock(object): +class CurrentBlock: def __init__(self, n_pages_per_block, blocknum): self.n_pages_per_block = n_pages_per_block self.blocknum = blocknum @@ -117,12 +117,12 @@ def is_full(self): class BlockPoolWithCurBlocks(TagBlockPool): def __init__(self, n, tags, n_pages_per_block): - super(BlockPoolWithCurBlocks, self).__init__(n, tags) + super().__init__(n, tags) self._n_pages_per_block = n_pages_per_block # {TAG1: {0: CurrentBlock obj, 1: CurrentBlock obj}, # TAG2: {0: CurrentBlock obj, 1: CurrentBlock obj}} - self._cur_blocks = {tag:{} for tag in tags} + self._cur_blocks = {tag: {} for tag in tags} def get_cur_block_obj(self, tag, block_index=None): """ @@ -132,7 +132,7 @@ def get_cur_block_obj(self, tag, block_index=None): if block_index is None: # return all cur block objs of a tag if block_index is not # specified. - return [obj for obj in self._cur_blocks[tag].values()] + return [obj for obj in list(self._cur_blocks[tag].values())] else: return self._cur_blocks[tag].get(block_index, None) @@ -153,10 +153,12 @@ def remove_full_cur_blocks(self): If cur block is full, we mark it as NON cur block. So garbage collector can clean it. """ - for tag, cur_obj_dict in self._cur_blocks.items(): + for tag, cur_obj_dict in list(self._cur_blocks.items()): to_del_block_index = [ - block_index for block_index, obj in cur_obj_dict.items() - if obj.is_full()] + block_index + for block_index, obj in list(cur_obj_dict.items()) + if obj.is_full() + ] for block_index in to_del_block_index: del cur_obj_dict[block_index] @@ -170,5 +172,3 @@ def set_new_cur_block(self, tag, block_index, blocknum): block_obj = CurrentBlock(self._n_pages_per_block, blocknum=blocknum) self._cur_blocks[tag][block_index] = block_obj return block_obj - - diff --git a/wiscsee/wiscsim/utils.py b/wiscsee/wiscsim/utils.py index 3219cf5..e96d395 100644 --- a/wiscsee/wiscsim/utils.py +++ b/wiscsee/wiscsim/utils.py @@ -6,12 +6,18 @@ GB = 1024**3 DEBUG = True + + def log_msg(*msg): - ''' + """ Log a message with the current time stamp. - ''' + """ msg = [str(_) for _ in msg] if DEBUG: - print"[%s] %s" % ((dt.datetime.now().strftime("%Y-%m-%d %H:%M:%S")), " ".join(msg)) + print( + "[{}] {}".format( + (dt.datetime.now().strftime("%Y-%m-%d %H:%M:%S")), " ".join(msg) + ) + ) - sys.stdout.flush() \ No newline at end of file + sys.stdout.flush() diff --git a/wiscsee/wiscsim/warmup.py b/wiscsee/wiscsim/warmup.py index ab3dfe6..3e27cdb 100755 --- a/wiscsee/wiscsim/warmup.py +++ b/wiscsee/wiscsim/warmup.py @@ -1,3 +1,4 @@ +from past.utils import old_div from itertools import combinations import bitarray import bitarray.util @@ -5,11 +6,11 @@ from wiscsim.lsm_tree.bloom_filter import BloomFilter from collections import defaultdict from bisect import bisect_left, insort_left -from datacache import LRUCache +from .datacache import LRUCache from wiscsim.utils import * from wiscsim.workload_parser import parse_events, create_config, mix_events -from ftlsim_commons import Extent -from datacache import WriteBuffer, RWCache +from .ftlsim_commons import Extent +from .datacache import WriteBuffer, RWCache from workflow import * from wiscsim.sftl import SFTLPage import glob @@ -29,15 +30,14 @@ TB = 1024.0**4 - -class PFTL(object): +class PFTL: def __init__(self): # store update history for verification purpose self.mapping_table = defaultdict(list) def set(self, lpn, ppn): self.mapping_table[lpn].append(ppn) - + def get(self, lpn): # force check; since we are using defaultdict we don't want to create empty entry if lpn not in self.mapping_table: @@ -53,15 +53,14 @@ def get_all(self, lpn): @property def memory(self): - return len(self.mapping_table)*4 + return len(self.mapping_table) * 4 @property def pages(self): return len(self.mapping_table) - -class SimpleSegment(): +class SimpleSegment: def __init__(self, k, b, x1, x2): self.b = b self.k = k @@ -69,19 +68,21 @@ def __init__(self, k, b, x1, x2): self.x2 = x2 def __str__(self): - return "(%d, %.2f, %d, %d)" % (self.b, self.k, self.x1, self.x2) + return f"({self.b}, {self.k:.2f}, {self.x1}, {self.x2})" def __repr__(self): return str(self) def get_y(self, x): - predict = int(round(x*self.k + self.b)) + predict = int(round(x * self.k + self.b)) return predict @staticmethod def intersection(s1, s2): - p = (float(s2.b - s1.b) / (s1.k - s2.k), - float(s1.k * s2.b - s2.k * s1.b) / (s1.k - s2.k)) + p = ( + float(s2.b - s1.b) / (s1.k - s2.k), + float(s1.k * s2.b - s2.k * s1.b) / (s1.k - s2.k), + ) return p @staticmethod @@ -107,7 +108,7 @@ def frompoints(p1, p2): return SimpleSegment(k, b, p1[0], p2[0]) -class Segment(): +class Segment: FPR = 0.01 PAGE_PER_BLOCK = 256 BITMAP = True @@ -126,7 +127,7 @@ def __init__(self, k, b, x1, x2, points=None): if not consecutive: if Segment.BITMAP: - self.filter = bitarray.bitarray(self.x2 - self.x1 + 1) + self.filter = bitarray.bitarray(self.x2 - self.x1 + 1) self.filter.setall(0) for pt in points: self.filter[pt[0] - self.x1] = 1 @@ -134,7 +135,7 @@ def __init__(self, k, b, x1, x2, points=None): self.filter = BloomFilter(len(points), Segment.FPR) for pt in points: self.filter.add(pt[0]) - + # if LPN_TO_DEBUG in zip(*points)[0]: # log_msg("new seg", self) @@ -142,18 +143,20 @@ def __init__(self, k, b, x1, x2, points=None): # print(self.full_str()) def __str__(self): - return "%.4f, %d, [%d, %d], memory: %dB, accuracy: %s, bitmap: %s" \ - % (self.k, self.b, self.x1, self.x2, self.memory, self.accurate, self.filter) + return f"{self.k:.4f}, {self.b}, [{self.x1}, {self.x2}], memory: {self.memory}B, accuracy: {self.accurate}, bitmap: {self.filter}" def __repr__(self): return str(self) - return "(%d, %.4f, %d, %d, %s)" % (self.b, self.k, self.x1, self.x2, self.accurate) + return f"({self.b}, {self.k:.4f}, {self.x1}, {self.x2}, {self.accurate})" def short(self): - return "%d,%d,%.3f,%d" % (self.x1, self.x2 - self.x1, self.k, self.b) + return f"{self.x1},{self.x2 - self.x1},{self.k:.3f},{self.b}" def full_str(self): - return "(%d, %.4f, %d, %d, %s) " % (self.b, self.k, self.x1, self.x2, self.accurate) + str(self._points) + return ( + f"({self.b}, {self.k:.4f}, {self.x1}, {self.x2}, {self.accurate}) " + + str(self._points) + ) def is_valid(self, x): if not (self.x1 <= x and x <= self.x2): @@ -168,7 +171,7 @@ def is_valid(self, x): def get_y(self, x, check=True): if not check or self.is_valid(x): - predict = int(round(x*self.k + self.b)) + predict = int(round(x * self.k + self.b)) # lowbound = self.blocknum * Segment.PAGE_PER_BLOCK # upbound = (self.blocknum + 1) * Segment.PAGE_PER_BLOCK - 1 # return max(min(predict, upbound), lowbound) @@ -183,7 +186,7 @@ def check_properties(self, points): is_accruate = False # if abs(self.get_y(pt[0], check=False) - pt[1]) > 5: # print(self, self.get_y(pt[0], check=False), pt[1]) - if len(np.unique(np.diff(zip(*points)[0]))) > 1: + if len(np.unique(np.diff(list(zip(*points))[0]))) > 1: is_consecutive = False return is_accruate, is_consecutive @@ -194,7 +197,6 @@ def overlaps(self, other): def overlaps_with_range(self, x1, x2): return min(self.x2, x2) - max(self.x1, x1) >= 0 - # check whether two segments can be put into the same level # if they can be put in the same level, return False # (here we assume other is older than self) @@ -240,44 +242,43 @@ def merge(new, old): else: return new, old, False - @staticmethod def bitwise_merge(new, old): lo, hi = min(old.x1, new.x1), max(old.x2, new.x2) - new_bm = bitarray.bitarray(hi-lo+1) - old_bm = bitarray.bitarray(hi-lo+1) + new_bm = bitarray.bitarray(hi - lo + 1) + old_bm = bitarray.bitarray(hi - lo + 1) new_bm.setall(0) old_bm.setall(0) if new.consecutive: - new_bm[new.x1-lo : new.x2-lo+1 : new.rec_k] = 1 + new_bm[new.x1 - lo : new.x2 - lo + 1 : new.rec_k] = 1 elif Segment.BITMAP: - new_bm[new.x1-lo : new.x2-lo+1] = new.filter - + new_bm[new.x1 - lo : new.x2 - lo + 1] = new.filter + if old.consecutive: - old_bm[old.x1-lo : old.x2-lo+1 : old.rec_k] = 1 + old_bm[old.x1 - lo : old.x2 - lo + 1 : old.rec_k] = 1 elif Segment.BITMAP: - old_bm[old.x1-lo : old.x2-lo+1] = old.filter - + old_bm[old.x1 - lo : old.x2 - lo + 1] = old.filter + try: old_bm = old_bm & (~new_bm) except: - print(lo, hi) - print(old, new) - print(old._points, new._points) - print(old_bm, new_bm) + print((lo, hi)) + print((old, new)) + print((old._points, new._points)) + print((old_bm, new_bm)) exit(0) first_valid = old_bm.find(1) if first_valid == -1: return new, None last_valid = bitarray.util.rindex(old_bm, 1) - old.x1 = first_valid + lo + old.x1 = first_valid + lo old.x2 = last_valid + lo if not old.consecutive and Segment.BITMAP: - old.filter = old_bm[first_valid : last_valid+1] - assert(old.filter != None) + old.filter = old_bm[first_valid : last_valid + 1] + assert old.filter != None # TODO: re-check accuracy and consecutive @@ -293,39 +294,61 @@ def mergable(self): @property def length(self): - return (self.x2-self.x1) // self.rec_k + 1 + return (self.x2 - self.x1) // self.rec_k + 1 @property def memory(self): if self.x1 == self.x2: - return SUBLPN_BYTES + PPN_BYTES # + FLOAT16_BYTES + LENGTH_BYTES + return SUBLPN_BYTES + PPN_BYTES # + FLOAT16_BYTES + LENGTH_BYTES else: if self.consecutive: - return SUBLPN_BYTES + PPN_BYTES + FLOAT16_BYTES + LENGTH_BYTES # 4+4+2+1 + return ( + SUBLPN_BYTES + PPN_BYTES + FLOAT16_BYTES + LENGTH_BYTES + ) # 4+4+2+1 else: if Segment.BITMAP: # filter_size = len(self.filter) / 8.0 ones = len([e for e in self.filter if e]) - non_consec_ones = len([i for i in range(len(self.filter)) if i > 0 and i < len(self.filter)-1 and self.filter[i] is not self.filter[i-1]]) + non_consec_ones = len( + [ + i + for i in range(len(self.filter)) + if i > 0 + and i < len(self.filter) - 1 + and self.filter[i] is not self.filter[i - 1] + ] + ) # print(self.filter, [i for i in range(len(self.filter)) if i > 0 and i < len(self.filter)-1 and self.filter[i] is not self.filter[i-1]]) # zeros = len(self.filter) - ones # sparse_encoding_size = min(ones, zeros) * 1 + LENGTH_BYTES - return SUBLPN_BYTES + PPN_BYTES + FLOAT16_BYTES + non_consec_ones * 1 + LENGTH_BYTES - #return SUBLPN_BYTES + PPN_BYTES + FLOAT16_BYTES + min(filter_size, sparse_encoding_size) + return ( + SUBLPN_BYTES + + PPN_BYTES + + FLOAT16_BYTES + + non_consec_ones * 1 + + LENGTH_BYTES + ) + # return SUBLPN_BYTES + PPN_BYTES + FLOAT16_BYTES + min(filter_size, sparse_encoding_size) else: - return SUBLPN_BYTES + PPN_BYTES + FLOAT16_BYTES + round(self.filter.bit_array_size / 8.0) - + return ( + SUBLPN_BYTES + + PPN_BYTES + + FLOAT16_BYTES + + round(self.filter.bit_array_size / 8.0) + ) + @property def rec_k(self): - return int(round(1.0/self.k)) + return int(round(1.0 / self.k)) + @property def blocknum(self): mid = (self.x2 + self.x1) / 2.0 predict = int(round(mid * self.k + self.b)) - return int(predict / Segment.PAGE_PER_BLOCK) + return int(old_div(predict, Segment.PAGE_PER_BLOCK)) -class PLR(): +class PLR: FIRST = "first" SECOND = "second" READY = "ready" @@ -357,7 +380,7 @@ def learn(self, points): self.segments.append(seg) if rej != None: rejs.append(rej) - + seg = self.build_segment() if seg != None: self.segments.append(seg) @@ -375,16 +398,18 @@ def build_segment(self): if self.state == PLR.FIRST: seg = None elif self.state == PLR.SECOND: - seg = Segment(1, self.s0[1] - self.s0[0], self.s0[0], self.s0[0], - points=self.points) + seg = Segment( + 1, self.s0[1] - self.s0[0], self.s0[0], self.s0[0], points=self.points + ) elif self.state == PLR.READY: # avg_slope = np.float16((self.rho_lower.k + self.rho_upper.k) / 2.0) avg_slope = (self.rho_lower.k + self.rho_upper.k) / 2.0 # rec_k = round(1.0/avg_slope) intercept = -self.sint[0] * avg_slope + self.sint[1] - seg = Segment(avg_slope, intercept, self.s0[0], self.s1[0], - points=self.points) - + seg = Segment( + avg_slope, intercept, self.s0[0], self.s1[0], points=self.points + ) + return seg def process(self, point): @@ -403,16 +428,23 @@ def process(self, point): else: self.s1 = point self.state = PLR.READY - self.rho_lower = SimpleSegment.frompoints(SimpleSegment.get_upper_bound(self.s0, self.gamma), - SimpleSegment.get_lower_bound(self.s1, self.gamma)) - self.rho_upper = SimpleSegment.frompoints(SimpleSegment.get_lower_bound(self.s0, self.gamma), - SimpleSegment.get_upper_bound(self.s1, self.gamma)) - self.sint = SimpleSegment.intersection( - self.rho_upper, self.rho_lower) + self.rho_lower = SimpleSegment.frompoints( + SimpleSegment.get_upper_bound(self.s0, self.gamma), + SimpleSegment.get_lower_bound(self.s1, self.gamma), + ) + self.rho_upper = SimpleSegment.frompoints( + SimpleSegment.get_lower_bound(self.s0, self.gamma), + SimpleSegment.get_upper_bound(self.s1, self.gamma), + ) + self.sint = SimpleSegment.intersection(self.rho_upper, self.rho_lower) self.state = PLR.READY elif self.state == PLR.READY: - if not SimpleSegment.is_above(point, self.rho_lower) or not SimpleSegment.is_below(point, self.rho_upper) or self.should_stop(point): + if ( + not SimpleSegment.is_above(point, self.rho_lower) + or not SimpleSegment.is_below(point, self.rho_upper) + or self.should_stop(point) + ): prev_segment = self.build_segment() self.s0 = point self.state = PLR.SECOND @@ -430,7 +462,6 @@ def process(self, point): # self.sint = SimpleSegment.intersection( # self.rho_upper, self.rho_lower) - self.points.append(point) return (prev_segment, None) @@ -449,7 +480,7 @@ def __len__(self): return len(self.it) -class LogPLR(): +class LogPLR: def __init__(self, gamma, frame_no): self.plr = PLR(gamma) # one run is one level of segments with non-overlapping intervals @@ -457,7 +488,7 @@ def __init__(self, gamma, frame_no): self.frame_no = frame_no # mapping from block to segments # self.block_map = defaultdict(list) - + def update(self, entries, blocknum): # make sure no same LPNs exist in the entries sorted_entries = sorted(entries) @@ -471,7 +502,7 @@ def update(self, entries, blocknum): return [], [] def merge(self, old_plr): - assert(self.frame_no == old_plr.frame_no) + assert self.frame_no == old_plr.frame_no self.runs.extend(old_plr.runs) # bottleneck @@ -497,7 +528,7 @@ def lookup(self, LPA, first=True): for level in sorted(empty_levels, reverse=True): del self.runs[level] - + return results, lookup, [], [] def lookup_range(self, start, end): @@ -520,8 +551,6 @@ def lookup_range(self, start, end): break return results - - # recursively add segments to each level # bottleneck @@ -533,20 +562,18 @@ def add_segments(self, level, segments, recursive=True): conflicts = [] for new_seg in segments: if new_seg.get_y(LPN_TO_DEBUG): - log_msg("%s added to run %d" % (new_seg, level)) + log_msg(f"{new_seg} added to run {level}") if len(run) == 0: run.append(new_seg) continue - # run[index].x1 >= new_seg.x1 - index = bisect_left(KeyWrapper( - run, key=lambda seg: seg.x1), new_seg.x1) + index = bisect_left(KeyWrapper(run, key=lambda seg: seg.x1), new_seg.x1) run.insert(index, new_seg) overlaps = [] if index != 0: - overlaps.append((index-1, run[index-1])) - for i in range(index+1, len(run)): + overlaps.append((index - 1, run[index - 1])) + for i in range(index + 1, len(run)): if run[i].x1 > new_seg.x2: break overlaps.append((i, run[i])) @@ -555,18 +582,18 @@ def add_segments(self, level, segments, recursive=True): for index, old_seg in overlaps: to_print = old_seg.get_y(LPN_TO_DEBUG) if to_print: - log_msg("%s tries to merge with %s" % (new_seg, old_seg)) + log_msg(f"{new_seg} tries to merge with {old_seg}") # log_msg("%s tries to merge with %s" % (new_seg, old_seg)) new_seg, old_seg, same_level = Segment.merge(new_seg, old_seg) if not old_seg: indices_to_delete.append(index) if to_print: - log_msg("%s removed old seg" % (new_seg)) + log_msg(f"{new_seg} removed old seg") elif not same_level: conflicts.append(old_seg) indices_to_delete.append(index) if to_print: - log_msg("%s -> %s" % (new_seg, old_seg)) + log_msg(f"{new_seg} -> {old_seg}") for index in sorted(indices_to_delete, reverse=True): if run[index].get_y(LPN_TO_DEBUG): log_msg("removed old seg", (run[index])) @@ -574,18 +601,15 @@ def add_segments(self, level, segments, recursive=True): if recursive: if len(conflicts) > 0: - self.add_segments(level+1, conflicts) + self.add_segments(level + 1, conflicts) else: if len(conflicts) > 0: - self.runs.insert(level+1, conflicts) - - - + self.runs.insert(level + 1, conflicts) def __str__(self): repr = "" for level in range(len(self.runs)): - repr += "== level %d ==\n %s \n" % (level, str(self.runs[level])) + repr += f"== level {level} ==\n {str(self.runs[level])} \n" return repr @property @@ -594,7 +618,10 @@ def segments(self): @property def memory(self): - return sum([seg.memory for i, run in enumerate(self.runs) for seg in run]) + LPN_BYTES + return ( + sum([seg.memory for i, run in enumerate(self.runs) for seg in run]) + + LPN_BYTES + ) @property def levels(self): @@ -602,7 +629,7 @@ def levels(self): # now we only use compact def gc(self, blocknum): - return + return for seg in self.block_map[blocknum]: for run in reversed(self.runs): index = bisect_left(KeyWrapper(run, key=lambda _: _.x1), seg.x1) @@ -611,7 +638,7 @@ def gc(self, blocknum): log_msg("gc removed old seg", (seg)) run.remove(seg) break - + self.block_map[blocknum] = [] def promote(self): @@ -622,16 +649,18 @@ def promote(self): return layers = self.runs[:] - for i in range(1,len(layers)): + for i in range(1, len(layers)): lower_layer = layers[i] for old_seg in reversed(lower_layer): promoted_layer = i promoted_index = None - for j in reversed(range(0,i)): + for j in reversed(list(range(0, i))): upper_layer = layers[j] - index = bisect_left(KeyWrapper(upper_layer, key=lambda seg: seg.x1), old_seg.x1) + index = bisect_left( + KeyWrapper(upper_layer, key=lambda seg: seg.x1), old_seg.x1 + ) overlaps = False - for k in range(max(0, index-1), len(upper_layer)): + for k in range(max(0, index - 1), len(upper_layer)): if upper_layer[k].x1 > old_seg.x2: break if upper_layer[k].overlaps(old_seg): @@ -642,15 +671,15 @@ def promote(self): else: promoted_layer = j promoted_index = index - + if promoted_layer < i: # if self.frame_no == 3075: # log_msg("Promote %s to level %d" % (old_seg, promoted_layer)) layers[promoted_layer].insert(promoted_index, old_seg) lower_layer.remove(old_seg) - + self.runs = [run for run in self.runs if len(run) != 0] - + def compact(self, promote=False): if len(self.runs) == 0: return @@ -666,12 +695,14 @@ def compact(self, promote=False): def compact_range(self, start, end): results = self.lookup_range(start, end) - for upper_layer, new_segs in results.items(): - for lower_layer, old_segs in results.items(): + for upper_layer, new_segs in list(results.items()): + for lower_layer, old_segs in list(results.items()): if upper_layer < lower_layer: for new_seg in new_segs: for old_seg in old_segs: - new_seg, updated_old_seg, same_level = Segment.merge(new_seg, old_seg) + new_seg, updated_old_seg, same_level = Segment.merge( + new_seg, old_seg + ) if not updated_old_seg: self.runs[lower_layer].remove(old_seg) results[lower_layer].remove(old_seg) @@ -681,7 +712,15 @@ def compact_range(self, start, end): # Each LogPLR is responsible for one range class FrameLogPLR: ON_FLASH, CLEAN, DIRTY = "ON_FLASH", "CLEAN", "DIRTY" - def __init__(self, counter, gamma, max_size=1*1024**2, frame_length=256, ftl_type="learnedftl"): + + def __init__( + self, + counter, + gamma, + max_size=1 * 1024**2, + frame_length=256, + ftl_type="learnedftl", + ): global SUBLPN_BYTES SUBLPN_BYTES = 1 self.counter = counter @@ -714,7 +753,7 @@ def __init__(self, counter, gamma, max_size=1*1024**2, frame_length=256, ftl_typ def create_frame(self, frame_no): if self.type == "learnedftl": return LogPLR(self.gamma, frame_no) - + elif self.type == "sftl": return SFTLPage(frame_no, self.frame_length) @@ -732,7 +771,7 @@ def update(self, entries, blocknum): pages_to_write, pages_to_read = [], [] split_entries = FrameLogPLR.split_into_frame(self.frame_length, entries) frame_nos = [] - for frame_no, entries in split_entries.items(): + for frame_no, entries in list(split_entries.items()): frame_nos += [frame_no] if frame_no not in self.frames: self.frames[frame_no] = self.create_frame(frame_no) @@ -747,7 +786,7 @@ def update(self, entries, blocknum): self.dirty[frame_no] = True self.change_size_of_frame(frame_no, self.frames[frame_no].memory) - + if self.should_flush(): pages_to_write, pages_to_read = self.flush() @@ -765,7 +804,7 @@ def lookup(self, lpn, first=True): self.frames.move_to_head(frame_no, frame) # log_msg("Move to head", frame_no) results, lookup, _, _ = frame.lookup(lpn, first) - + if len(results) != 0: self.counter["mapping_table_read_hit"] += 1 else: @@ -785,7 +824,7 @@ def lookup(self, lpn, first=True): self.dirty[frame_no] = False self.frames[frame_no] = frame del self.frame_on_flash[frame_no] - + self.frames.move_to_head(frame_no, self.frames[frame_no]) # log_msg("Move to head", frame_no) # log_msg(self.frames.keys()) @@ -804,7 +843,7 @@ def lookup(self, lpn, first=True): # if should_print: # print("after",self.memory) - + return results, lookup, pages_to_write, pages_to_read # gc is currently replaced with compaction @@ -813,20 +852,19 @@ def gc(self, blocknum): def compact(self, promote=False, frame_nos=None): if not frame_nos: - for frame_no, frame in self.frames.items(): + for frame_no, frame in list(self.frames.items()): frame.compact(promote=promote) - self.change_size_of_frame(frame_no, frame.memory) + self.change_size_of_frame(frame_no, frame.memory) else: for frame_no in frame_nos: frame = self.frames[frame_no] frame.compact(promote=promote) - self.change_size_of_frame(frame_no, frame.memory) + self.change_size_of_frame(frame_no, frame.memory) def promote(self): - for frame in self.frames.values(): + for frame in list(self.frames.values()): frame.promote() - def should_flush(self): if self.memory > self.max_size: # n_pages = (self.memory - self.max_size) /self.conf.page_size @@ -839,27 +877,29 @@ def allocate_ppn_for_frame(self, frame_no): # self.current_trans_block = self.metadata.bvc.next_free_block() # self.metadata.pvb.validate_block(self.current_trans_block) - next_free_ppn = self.n_pages_per_block * self.current_trans_block + self.current_trans_page_offset + next_free_ppn = ( + self.n_pages_per_block * self.current_trans_block + + self.current_trans_page_offset + ) self.current_trans_page_offset += 1 if frame_no not in self.GTD: old_ppn = None self.GTD[frame_no] = next_free_ppn new_ppn = self.GTD[frame_no] - + else: old_ppn = self.GTD[frame_no] self.GTD[frame_no] = next_free_ppn new_ppn = self.GTD[frame_no] return new_ppn, old_ppn - + def flush(self): evicted_frames = [] pages_to_read = [] pages_to_write = [] - original_memory = self.memory # assert(self.memory == sum([frame.memory for frame in self.frames.values()])) while original_memory > self.max_size: @@ -867,7 +907,7 @@ def flush(self): # log_msg(frame_no, "evicted") freed_mem = self.memory_counter[frame_no] original_memory -= freed_mem - self.change_size_of_frame(frame_no, 0) + self.change_size_of_frame(frame_no, 0) evicted_frames.append(frame_no) new_ppn, old_ppn = self.allocate_ppn_for_frame(frame_no) @@ -883,7 +923,6 @@ def flush(self): evict_frame.merge(old_frame) self.frame_on_flash[frame_no] = evict_frame - # log_msg("%.2f miss ratio, %s evicted, %d memory, %d in cache, %d on flash" % (self.misses / float(self.misses + self.hits), evicted_frames, self.memory, len(self.frames), len(self.frame_on_flash))) # log_msg("%d miss, %d memory flushed, %s evicted, %d memory, %d in cache, %d on flash" % (self.misses, original_memory - self.memory, evicted_frames, self.memory, len(self.frames), len(self.frame_on_flash))) @@ -894,8 +933,8 @@ def change_size_of_frame(self, frame_no, new_mem): if frame_no in self.memory_counter: old_mem = self.memory_counter[frame_no] self.memory_counter[frame_no] = new_mem - self.total_memory += (new_mem - old_mem) - + self.total_memory += new_mem - old_mem + # TODO: bottleneck @property def memory(self): @@ -905,30 +944,41 @@ def memory(self): @property def segments(self): - return [seg for frame in self.frames.values() for seg in reversed(frame.segments)] + return [ + seg + for frame in list(self.frames.values()) + for seg in reversed(frame.segments) + ] @property def levels(self): if len(self.frames) == 0: return 0 - return max([frame.levels for frame in self.frames.values()]) + return max([frame.levels for frame in list(self.frames.values())]) @property def avg_levels(self): if len(self.frames) == 0: return 0, 0 - dist = [frame.levels for frame in self.frames.values() if frame.levels != 0] + dist = [ + frame.levels for frame in list(self.frames.values()) if frame.levels != 0 + ] return np.average(dist), np.std(dist) - + @property def groups(self): return len(self.frames) def dump(self, filename): with open(filename, "w") as outfile: - all_segments = [(frame_no, seg) for frame_no, frame in self.frames.items() for seg in reversed(frame.segments)] + all_segments = [ + (frame_no, seg) + for frame_no, frame in list(self.frames.items()) + for seg in reversed(frame.segments) + ] for frame_no, segment in all_segments: - outfile.write(str(frame_no)+","+segment.short()+"\n") + outfile.write(str(frame_no) + "," + segment.short() + "\n") + def split_ext(extent): if extent.lpn_count == 0: @@ -941,12 +991,13 @@ def split_ext(extent): return exts + if __name__ == "__main__": log_msg("warm-up starts") # traces = ["/home/js39/datasets/MSR-Cambridge/usr_0.csv"] - #trace = '/home/js39/datasets/FIU/homes/homes-110108-112108.1.blkparse' - #trace = "/home/js39/datasets/FIU/mail/cheetah.cs.fiu.edu-110108-113008.1.blkparse" - #trace = "/home/js39/datasets/rocksdb/ssdtrace-00" + # trace = '/home/js39/datasets/FIU/homes/homes-110108-112108.1.blkparse' + # trace = "/home/js39/datasets/FIU/mail/cheetah.cs.fiu.edu-110108-113008.1.blkparse" + # trace = "/home/js39/datasets/rocksdb/ssdtrace-00" # traces = ["MSR-Cambridge/usr_0.csv", "MSR-Cambridge/usr_1.csv", "MSR-Cambridge/usr_2.csv", "MSR-Cambridge/prn_0.csv", "MSR-Cambridge/src2_2.csv", "MSR-Cambridge/hm_0.csv", "MSR-Cambridge/prxy_0.csv", 'FIU/homes/homes-110108-112108.1.blkparse', 'FIU/mail/cheetah.cs.fiu.edu-110108-113008.1.blkparse'] # traces = ["/home/js39/datasets/"+_ for _ in traces] # traces = glob.glob("/home/js39/datasets/"+"MSR-Cambridge/*.csv") @@ -967,19 +1018,23 @@ def split_ext(extent): all_events = dict() for trace in traces: - events = parse_events(trace, page_size=page_size, max_writes=2000000, recorder=False) + events = parse_events( + trace, page_size=page_size, max_writes=2000000, recorder=False + ) all_events[trace] = events writes = mix_events(all_events, page_size, policy="RR_Single_Light_Shuffle") log_msg("Total # of writes:", len(writes)) def warm_up(ftl_type, gamma): - os.system("taskset -p 0xff %d > /dev/null" % os.getpid()) + os.system(f"taskset -p 0xff {os.getpid()} > /dev/null") if ftl_type == "learnedftl": - filter_ratio = 7.0/8.0 + filter_ratio = 7.0 / 8.0 elif ftl_type == "sftl": filter_ratio = 1.0 - buffer = RWCache(8*MB, page_size, 8*MB, filter_ratio) # WriteBuffer(8*256, 8*256, filtering=1.0) - counter = defaultdict(float) + buffer = RWCache( + 8 * MB, page_size, 8 * MB, filter_ratio + ) # WriteBuffer(8*256, 8*256, filtering=1.0) + counter = defaultdict(float) mapping_table = FrameLogPLR(counter, gamma=gamma, ftl_type=ftl_type) reference_mapping_table = PFTL() conf = create_config(ftl_type="learnedftl") @@ -991,7 +1046,15 @@ def warm_up(ftl_type, gamma): mapping_table.compact() mapping_table.promote() mapping_table.compact() - log_msg((ftl_type, gamma), "Processed # of writes:", op_i, "Memory:", mapping_table.memory, reference_mapping_table.memory, mapping_table.groups) + log_msg( + (ftl_type, gamma), + "Processed # of writes:", + op_i, + "Memory:", + mapping_table.memory, + reference_mapping_table.memory, + mapping_table.groups, + ) # mapping_table.compact() # mapping_table.promote() @@ -1004,23 +1067,29 @@ def warm_up(ftl_type, gamma): entry = (lpn, next_free_ppn) reference_mapping_table.set(entry[0], entry[1]) entries.append(entry) - mapping_pages_to_write, mapping_pages_to_read = mapping_table.update(entries, -1) + mapping_pages_to_write, mapping_pages_to_read = mapping_table.update( + entries, -1 + ) buffer.update_assigned(dict(entries)) mapping_table.compact() mapping_table.promote() mapping_table.compact() - mapping_table.dump("/home/js39/software/wiscsee/wiscsim/compaction/segments.txt") + mapping_table.dump( + "/home/js39/software/wiscsee/wiscsim/compaction/segments.txt" + ) log_msg("finish") return (mapping_table.memory, reference_mapping_table.memory) - params = [('learnedftl', 1e-4), ("sftl", 0), ('learnedftl', 16)] + params = [("learnedftl", 1e-4), ("sftl", 0), ("learnedftl", 16)] # params = [("learnedftl", 1e-4)] - results = Parallel(n_jobs=len(params), backend="multiprocessing")(delayed(warm_up)(ftl_type, gamma) for (ftl_type, gamma) in params) + results = Parallel(n_jobs=len(params), backend="multiprocessing")( + delayed(warm_up)(ftl_type, gamma) for (ftl_type, gamma) in params + ) print(results) @@ -1030,24 +1099,34 @@ def warm_up(ftl_type, gamma): log_msg(ftl_type, gamma) - log_msg("estimated %s memory footprint: %d B" % (ftl_type, mapping_table_memory)) - log_msg("estimated dftl memory footprint: %d B" % reference_mapping_table_memory) - log_msg("estimated storage consumption: %.2f GB" % float(reference_mapping_table_memory * page_size / 4.0 / GB)) + log_msg(f"estimated {ftl_type} memory footprint: {mapping_table_memory} B") + log_msg(f"estimated dftl memory footprint: {reference_mapping_table_memory} B") + log_msg( + f"estimated storage consumption: {float(reference_mapping_table_memory * page_size / 4.0 / GB):.2f} GB" + ) continue log_msg("== Scaled ==") storage = float(reference_mapping_table_memory / 4.0 * page_size) - factor = 2*TB/storage - - log_msg("estimated %s memory footprint for 2TB: %d MB" % (ftl_type, mapping_table_memory * factor / MB)) - log_msg("estimated dftl memory footprint for 2TB: %d MB" % (reference_mapping_table_memory * factor / MB)) + factor = old_div(2 * TB, storage) + + log_msg( + f"estimated {ftl_type} memory footprint for 2TB: {old_div(mapping_table_memory * factor, MB)} MB" + ) + log_msg( + f"estimated dftl memory footprint for 2TB: {old_div(reference_mapping_table_memory * factor, MB)} MB" + ) log_msg("") storage = float(reference_mapping_table_memory / 4.0 * page_size) - factor = 1*TB/storage - - log_msg("estimated %s memory footprint for 1TB: %d MB" % (ftl_type, mapping_table_memory * factor / MB)) - log_msg("estimated dftl memory footprint for 1TB: %d MB" % (reference_mapping_table_memory * factor / MB)) + factor = old_div(1 * TB, storage) + + log_msg( + f"estimated {ftl_type} memory footprint for 1TB: {old_div(mapping_table_memory * factor, MB)} MB" + ) + log_msg( + f"estimated dftl memory footprint for 1TB: {old_div(reference_mapping_table_memory * factor, MB)} MB" + ) log_msg("") # storage = float(reference_mapping_table_memory / 4.0 * page_size) @@ -1056,4 +1135,3 @@ def warm_up(ftl_type, gamma): # log_msg("estimated %s memory footprint for 0.25TB: %d MB" % (ftl_type, mapping_table_memory * factor / MB)) # log_msg("estimated dftl memory footprint for 0.25TB: %d MB" % (reference_mapping_table_memory * factor / MB)) # log_msg("") - diff --git a/wiscsee/wiscsim/workload_parser.py b/wiscsee/wiscsim/workload_parser.py index ce8193f..1caacef 100644 --- a/wiscsee/wiscsim/workload_parser.py +++ b/wiscsee/wiscsim/workload_parser.py @@ -1,54 +1,157 @@ +from past.utils import old_div +import struct from wiscsim.utils import * from wiscsim.hostevent import Event, ControlEvent from workflow import * import math from utilities import utils -from ftlsim_commons import Extent +from .ftlsim_commons import Extent import random from random import randrange random.seed(1000) + def create_config(ftl_type="dftldes"): if ftl_type == "dftldes" or ftl_type == "learnedftl" or ftl_type == "sftl": conf = wiscsim.dftldes.Config() - conf['ftl_type'] = "learnedftl" - conf['internal_ftl_type'] = ftl_type + conf["ftl_type"] = "learnedftl" + conf["internal_ftl_type"] = ftl_type else: raise NotImplementedError # ssd config - conf['flash_config']['n_pages_per_block'] = 256 - conf['flash_config']['n_blocks_per_plane'] = 2048 - conf['flash_config']['n_planes_per_chip'] = 64 - conf['flash_config']['n_chips_per_package'] = 1 - conf['flash_config']['n_packages_per_channel'] = 1 - conf['flash_config']['n_channels_per_dev'] = 8 + conf["flash_config"]["n_pages_per_block"] = 256 + conf["flash_config"]["n_blocks_per_plane"] = 2048 + conf["flash_config"]["n_planes_per_chip"] = 64 + conf["flash_config"]["n_chips_per_package"] = 1 + conf["flash_config"]["n_packages_per_channel"] = 1 + conf["flash_config"]["n_channels_per_dev"] = 8 # set ftl - conf['do_not_check_gc_setting'] = True + conf["do_not_check_gc_setting"] = True conf.GC_high_threshold_ratio = 0.96 conf.GC_low_threshold_ratio = 0.5 - conf['enable_simulation'] = True + conf["enable_simulation"] = True - utils.set_exp_metadata(conf, save_data = False, - expname = 'run_expname', - subexpname = 'run_subexpname') + utils.set_exp_metadata( + conf, save_data=False, expname="run_expname", subexpname="run_subexpname" + ) - conf['simulator_class'] = 'SimulatorDESNew' + conf["simulator_class"] = "SimulatorDESNew" utils.runtime_update(conf) return conf + def split_lpns(offset, size, page_size): page_size = float(page_size) - lpns = [lpn for lpn in range(int(math.floor(offset/page_size)), int(math.ceil((offset+size)/page_size)))] + lpns = [ + lpn + for lpn in range( + int(math.floor(old_div(offset, page_size))), + int(math.ceil(old_div((offset + size), page_size))), + ) + ] return lpns -def parse_events(filename, page_size, recorder=True, start_lineno=0, lineno=float('inf'), max_writes = float('inf'), max_write_size = float('inf'), write_only=False, format="MSR", capacity = 2*TB, shift_range=False): + +def parse_binary_events(filename, page_size, recorder=True): + events = [] + if recorder: + events.append(ControlEvent(OP_ENABLE_RECORDER)) + else: + events.append(ControlEvent(OP_DISABLE_RECORDER)) + + OP_GET = 0x00 + OP_SET = 0x01 + OP_MGET = 0x02 + OP_MSET = 0x03 + OP_RM = 0x04 + OP_MRM = 0x05 + + SECTOR_SIZE = 512 + # page_size passed in, typically 4096 + + with open(filename, 'rb') as f: + timestamp = 0 + while True: + # Read OP (1 byte) + op_bytes = f.read(1) + if not op_bytes: + break + op = struct.unpack('B', op_bytes)[0] + + # Read LBA (4 bytes) + lba_bytes = f.read(4) + if not lba_bytes: + break + lba = struct.unpack('> - format_config = {"MSR" : (1, 100, ","), "blktrace" : (512, 1000**3, " "), "systor" : (1, 1000**3, ","), "normal" : (1, 1000, " "), "FIU" : (512, 1, " "), "Financial" : (1, 1000**3, ",", 512)} + format_config = { + "MSR": (1, 100, ","), + "blktrace": (512, 1000**3, " "), + "systor": (1, 1000**3, ","), + "normal": (1, 1000, " "), + "FIU": (512, 1, " "), + "Financial": (1, 1000**3, ",", 512), + } size_scale = format_config[format][0] offset_scale = size_scale time_scale = format_config[format][1] @@ -95,29 +205,41 @@ def parse_events(filename, page_size, recorder=True, start_lineno=0, lineno=floa continue # parse trace line = raw.strip().split(delimeter) - line = list(filter(lambda x: x!= "", line)) + line = list([x for x in line if x != ""]) # print(line) if format == "MSR": t, p, d, mode, offset, size, t0 = line t, d, offset, size, t0 = int(t), int(d), int(offset), int(size), int(t0) elif format == "normal": t, d, offset, size, mode = line - t, d, offset, size, mode = int(t), int(d), int(offset), int(size), int(mode) + t, d, offset, size, mode = ( + int(t), + int(d), + int(offset), + int(size), + int(mode), + ) elif format == "blktrace": - if not len(line)==11: + if not len(line) == 11: continue - a, a2, a3, t, a4, a5, mode, offset, plus, size, a6 = line - t, offset, size= float(t), int(offset), int(size) + a, a2, a3, t, a4, a5, mode, offset, plus, size, a6 = line + t, offset, size = float(t), int(offset), int(size) elif format == "systor": if i == 0: continue t, t0, mode, d, offset, size = line if t0 == "": t0 = 0.0 - t, d, offset, size, t0 = float(t), int(d), int(offset), int(size), float(t0) + t, d, offset, size, t0 = ( + float(t), + int(d), + int(offset), + int(size), + float(t0), + ) elif format == "Financial": app, offset, size, mode, t = line - if int(app)!=0: + if int(app) != 0: continue t, offset, size = float(t), int(offset), int(size) elif format == "FIU": @@ -132,12 +254,12 @@ def parse_events(filename, page_size, recorder=True, start_lineno=0, lineno=floa # scale trace offset *= offset_scale size *= size_scale - t = int(t*time_scale) + t = int(t * time_scale) offset += offset_shift if size == 0: continue - if mode in ["Read", "R", 0, 'r', "RM"]: + if mode in ["Read", "R", 0, "r", "RM"]: if write_only: continue op = OP_READ @@ -147,9 +269,11 @@ def parse_events(filename, page_size, recorder=True, start_lineno=0, lineno=floa should_warm_up = True exist_lpns[lpn] = None if should_warm_up: - warm_up_writes += [Event(512, 0, OP_WRITE, offset, size, timestamp=0)] + warm_up_writes += [ + Event(512, 0, OP_WRITE, offset, size, timestamp=0) + ] num_writes += len(split_lpns(offset, size, page_size)) - elif mode in ["Write", "W", 1, 'w', "WS","WM"]: + elif mode in ["Write", "W", 1, "w", "WS", "WM"]: op = OP_WRITE for lpn in split_lpns(offset, size, page_size): exist_lpns[lpn] = None @@ -165,39 +289,38 @@ def parse_events(filename, page_size, recorder=True, start_lineno=0, lineno=floa events += [Event(512, 0, op, offset, size, timestamp=t)] active_events += 1 last_t = t - + # termination if i > lineno: break if num_writes >= max_writes: break - + if write_size >= max_write_size: break # if (i-start_lineno) % 1000000 == 0: # log_msg("parsed %d lines" % i) - + # timestamp from traces might not be sorted! (now we abort the unsorted ones) # events = sorted(events, key=lambda event: event.timestamp) # for i in range(0, len(events)): # events.insert(i*2, ControlEvent(OP_SLEEP, arg1=None)) # last_t = 0 # for i in range(0, len(events), 2): - # sleep, event = events[i], events[i+1] + # sleep, event = events[i], events[i+1] # sleep.arg1 = event.timestamp - last_t # last_t = event.timestamp events = [ControlEvent(OP_ENABLE_RECORDER)] + warm_up_writes + events - log_msg("Trace %s" % filename) - log_msg("Total warm-up events %d" % len(warm_up_writes)) - log_msg("Total active events %d" % active_events) + log_msg(f"Trace {filename}") + log_msg(f"Total warm-up events {len(warm_up_writes)}") + log_msg(f"Total active events {active_events}") return events - def split_ext(extent): if extent.lpn_count == 0: return None @@ -214,17 +337,17 @@ def partial_shuffle(l, factor=5): n = len(l) for _ in range(factor): a = randrange(n) - b = min(n-1, a + randrange(n // 1000)) + b = min(n - 1, a + randrange(n // 1000)) l[b], l[a] = l[a], l[b] -def mix_events(all_events, page_size, policy='RR'): +def mix_events(all_events, page_size, policy="RR"): all_writes = [] conf = create_config(ftl_type="learnedftl") # conf = create_config(ftl_type="learnedftl", page_size=page_size) if policy == "RR": - for i in range(max([len(l) for l in all_events.values()])): - for t in all_events.keys(): + for i in range(max([len(l) for l in list(all_events.values())])): + for t in list(all_events.keys()): if len(all_events[t]) > i: event = all_events[t][i] if event.get_operation() != OP_WRITE: @@ -233,9 +356,8 @@ def mix_events(all_events, page_size, policy='RR'): writes = split_ext(ext) all_writes += writes - if policy in ["RR_Single_Write", "RR_Single_Light_Shuffle"]: - for trace, events in all_events.items(): + for trace, events in list(all_events.items()): writes = [] for event in events: if event.get_operation() != OP_WRITE: @@ -244,8 +366,10 @@ def mix_events(all_events, page_size, policy='RR'): writes += split_ext(ext) all_events[trace] = writes - for i in range(sorted([len(l) for l in all_events.values()])[len(all_events)//2]): - for t in all_events.keys(): + for i in range( + sorted([len(l) for l in list(all_events.values())])[len(all_events) // 2] + ): + for t in list(all_events.keys()): if len(all_events[t]) > i: all_writes.append(all_events[t][i]) @@ -254,20 +378,20 @@ def mix_events(all_events, page_size, policy='RR'): if policy == "RR_Controlled_Random": threshold = 0.9 - dist_writes = [len(l) for l in all_events.values()] + dist_writes = [len(l) for l in list(all_events.values())] num_of_writes = sum(dist_writes) - progress = {t : 0 for t in all_events.keys()} + progress = {t: 0 for t in list(all_events.keys())} next_t, i = 0, 0 while i < num_of_writes: - tracefile = all_events.keys()[next_t] + tracefile = list(all_events.keys())[next_t] if progress[tracefile] < len(all_events[tracefile]): event = all_events[tracefile][progress[tracefile]] if event.get_operation() == OP_WRITE: ext = event.get_lpn_extent(conf) writes = split_ext(ext) all_writes += writes - + i += 1 progress[tracefile] += 1 @@ -277,7 +401,6 @@ def mix_events(all_events, page_size, policy='RR'): else: next_t += 1 - next_t %= len(all_events) - + next_t %= len(all_events) return all_writes diff --git a/wiscsee/workflow.py b/wiscsee/workflow.py index 8294579..201a226 100644 --- a/wiscsee/workflow.py +++ b/wiscsee/workflow.py @@ -14,7 +14,7 @@ def run_workflow(conf): wf.run() -class Workflow(object): +class Workflow: def __init__(self, conf): self.conf = conf @@ -32,38 +32,36 @@ def run_workload(self): return self._run_workload() def _save_conf(self): - confpath = os.path.join(self.conf['result_dir'], 'config.json') + confpath = os.path.join(self.conf["result_dir"], "config.json") prepare_dir_for_path(confpath) self.conf.dump_to_file(confpath) def _run_workload(self): - workload_src = self.conf['workload_src'] + workload_src = self.conf["workload_src"] if workload_src == WLRUNNER: runner = workrunner.wlrunner.WorkloadRunner(self.conf) event_iter = runner.run() elif workload_src == LBAGENERATOR: - classname = self.conf['lba_workload_class'] - cls = eval("workrunner.lbaworkloadgenerator.{}".format(classname)) + classname = self.conf["lba_workload_class"] + cls = eval(f"workrunner.lbaworkloadgenerator.{classname}") lbagen = cls(self.conf) event_iter = lbagen elif workload_src == LBAMULTIPROC: - classname = self.conf['lba_workload_class'] - cls = "workrunner.lbaworkloadgenerator.{}".format(classname) + classname = self.conf["lba_workload_class"] + cls = f"workrunner.lbaworkloadgenerator.{classname}" lbagen = cls(self.conf) event_iter = lbagen.get_iter_list() else: - raise RuntimeError("{} is not a valid workload source"\ - .format(workload_src)) + raise RuntimeError(f"{workload_src} is not a valid workload source") return event_iter def _run_simulator(self, event_iter): - if self.conf['enable_simulation'] is not True: + if self.conf["enable_simulation"] is not True: return - simulator = create_simulator(self.conf['simulator_class'], self.conf, - event_iter ) + simulator = create_simulator( + self.conf["simulator_class"], self.conf, event_iter + ) simulator.run() return simulator - - diff --git a/wiscsee/workrunner/__init__.py b/wiscsee/workrunner/__init__.py index 00d6731..6766eb2 100644 --- a/wiscsee/workrunner/__init__.py +++ b/wiscsee/workrunner/__init__.py @@ -1,2 +1,2 @@ -import wlrunner -import lbaworkloadgenerator +from . import wlrunner +from . import lbaworkloadgenerator diff --git a/wiscsee/workrunner/cpuhandler.py b/wiscsee/workrunner/cpuhandler.py index 1d2d1f4..772a7b6 100644 --- a/wiscsee/workrunner/cpuhandler.py +++ b/wiscsee/workrunner/cpuhandler.py @@ -1,76 +1,86 @@ import glob import os + def get_possible_cpus(): - f = open("/sys/devices/system/cpu/possible", 'r') + f = open("/sys/devices/system/cpu/possible") line = f.readline() f.close() # assuming format of 0-2,4,6-63 - items = line.split(',') + items = line.split(",") cpus = [] for item in items: - if '-' in item: - a,b = item.split('-') + if "-" in item: + a, b = item.split("-") a = int(a) b = int(b) - cpus.extend(range(a, b+1)) + cpus.extend(list(range(a, b + 1))) else: cpus.append(int(item)) return cpus + def get_available_cpu_dirs(): "Counting dirs is more accurate than */cpu/possible, at least on emulab" - cpudirs = [name for name in glob.glob("/sys/devices/system/cpu/cpu[0-9]*") \ - if os.path.isdir(name)] + cpudirs = [ + name + for name in glob.glob("/sys/devices/system/cpu/cpu[0-9]*") + if os.path.isdir(name) + ] return cpudirs + def get_online_cpuids(): - with open('/sys/devices/system/cpu/online', 'r') as f: + with open("/sys/devices/system/cpu/online") as f: line = f.readline().strip() # assuming format of 0-2,4,6-63 - items = line.split(',') + items = line.split(",") cpus = [] for item in items: - if '-' in item: - a,b = item.split('-') + if "-" in item: + a, b = item.split("-") a = int(a) b = int(b) - cpus.extend(range(a, b+1)) + cpus.extend(list(range(a, b + 1))) else: cpus.append(int(item)) return cpus + def switch_cpu(cpuid, mode): path = "/sys/devices/system/cpu/cpu{cpuid}/online" path = path.format(cpuid=cpuid) - modedict = {'ON':'1', 'OFF':'0'} + modedict = {"ON": "1", "OFF": "0"} - f = open(path, 'w') + f = open(path, "w") f.write(modedict[mode]) f.flush() f.close() return + def enable_all_cpus(): possible_cpus = get_possible_cpus() enable_n_cpus(len(possible_cpus)) + def set_cpus(n): - if n == 'NOOP' or n == None: + if n == "NOOP" or n == None: return - if n == 'all': + if n == "all": enable_all_cpus() return enable_n_cpus(n) + def enable_n_cpus(n): """ Enable n CPUs @@ -83,23 +93,20 @@ def enable_n_cpus(n): elif n_online > n: # more than wanted is online, disable some for cpuid in online_cpus[n:]: - switch_cpu(cpuid, 'OFF') + switch_cpu(cpuid, "OFF") else: # we need some more CPU to be online need = n - n_online possible_cpus = get_possible_cpus() for cpuid in possible_cpus: if not cpuid in online_cpus: - switch_cpu(cpuid, 'ON') + switch_cpu(cpuid, "ON") need -= 1 if need == 0: break if need > 0: - raise RuntimeError("Need {} CPUS, but only got {}".format( - n, n - need)) + raise RuntimeError(f"Need {n} CPUS, but only got {n - need}") online_cpus = get_online_cpuids() assert len(online_cpus) == n - - diff --git a/wiscsee/workrunner/filesystem.py b/wiscsee/workrunner/filesystem.py index 17aaa6c..f44451c 100644 --- a/wiscsee/workrunner/filesystem.py +++ b/wiscsee/workrunner/filesystem.py @@ -1,10 +1,11 @@ import abc import os -import fshelper +from . import fshelper from utilities import utils -class LoopDevice(object): + +class LoopDevice: def __init__(self, dev_path, tmpfs_mount_point, size_mb, img_file=None): self.dev_path = dev_path self.tmpfs_mount_point = tmpfs_mount_point @@ -12,15 +13,15 @@ def __init__(self, dev_path, tmpfs_mount_point, size_mb, img_file=None): self.img_file = img_file def create(self): - fshelper.make_loop_device(self.dev_path, self.tmpfs_mount_point, - self.size_mb, self.img_file) + fshelper.make_loop_device( + self.dev_path, self.tmpfs_mount_point, self.size_mb, self.img_file + ) def delete(self): fshelper.delLoopDev(self.dev_path) -class FileSystemBase(object): - __metaclass__ = abc.ABCMeta +class FileSystemBase(metaclass=abc.ABCMeta): def __init__(self, device, mount_point): self.dev = device self.mount_point = mount_point @@ -34,21 +35,23 @@ def mount(self, opt_list=None): opt_str = mountoption_to_str(opt_list) utils.prepare_dir(self.mount_point) - ret = utils.shcmd('mount {opt} {dev} {mp}'.format( - opt = opt_str, dev = self.dev, mp = self.mount_point), - ignore_error = True) + ret = utils.shcmd( + f"mount {opt_str} {self.dev} {self.mount_point}", ignore_error=True + ) if ret != 0: - raise RuntimeError("Failed to mount dev:{} to dir:{}".format( - self.dev, self.mount_point)) + raise RuntimeError( + f"Failed to mount dev:{self.dev} to dir:{self.mount_point}" + ) def umount(self): ret = fshelper.umountFS(self.mount_point) if ret != 0: - raise RuntimeError("Failed to umount {}".format(self.mount_point)) + raise RuntimeError(f"Failed to umount {self.mount_point}") def sync(self): common.shcmd("sync") + def opts_to_str(opt_dic): """ This function translate opt_dic to a string complying command requirement @@ -60,19 +63,20 @@ def opts_to_str(opt_dic): "-O has_journal,^uninit_bg -X xx" """ if opt_dic == None or len(opt_dic) == 0: - return '' + return "" opt_list = [] - for opt, values in opt_dic.items(): + for opt, values in list(opt_dic.items()): values = [str(s) for s in values] - value_str = ','.join(values) - tmp = ' '.join((opt, value_str)) + value_str = ",".join(values) + tmp = " ".join((opt, value_str)) opt_list.append(tmp) - opt_str = ' '.join(opt_list) + opt_str = " ".join(opt_list) return opt_str + def mountoption_to_str(options): """ options is a list of dictionaries: @@ -98,41 +102,43 @@ def mountoption_to_str(options): """ if options == None: - return '' + return "" strs = [] - for _, opt in options.items(): - if opt['value'] != None: - if opt['include_name'] == True: - itemstr = opt['opt_name'] + '=' + str(opt['value']) + for _, opt in list(options.items()): + if opt["value"] != None: + if opt["include_name"] == True: + itemstr = opt["opt_name"] + "=" + str(opt["value"]) else: - itemstr = str(opt['value']) + itemstr = str(opt["value"]) strs.append(itemstr) if len(options) > 0: - opt_str = '-o ' + ','.join(strs) + opt_str = "-o " + ",".join(strs) else: - opt_str = '' + opt_str = "" return opt_str + class Ext4(FileSystemBase): def make(self, opt_dic=None): opt_str = opts_to_str(opt_dic) - ret = utils.shcmd('mkfs.ext4 {opt_str} -E nodiscard {dev}'.format( - opt_str = opt_str, dev = self.dev), ignore_error = True) + ret = utils.shcmd( + f"mkfs.ext4 {opt_str} -E nodiscard {self.dev}", ignore_error=True + ) if ret != 0: - raise RuntimeError("Failed to make dev:{}".format(self.dev)) + raise RuntimeError(f"Failed to make dev:{self.dev}") + class F2fs(FileSystemBase): def make(self, opt_dic=None): opt_str = opts_to_str(opt_dic) - ret = utils.shcmd('mkfs.f2fs -t 0 {opt} {dev}'.format( - opt=opt_str, dev = self.dev), ignore_error = True) + ret = utils.shcmd(f"mkfs.f2fs -t 0 {opt_str} {self.dev}", ignore_error=True) if ret != 0: - raise RuntimeError("Failed to make dev:{}".format(self.dev)) + raise RuntimeError(f"Failed to make dev:{self.dev}") def mount(self, opt_list=None): """ @@ -142,47 +148,56 @@ def mount(self, opt_list=None): opt_str = mountoption_to_str(opt_list) utils.prepare_dir(self.mount_point) - ret = utils.shcmd('mount -t f2fs {opt} {dev} {mp}'.format( - opt = opt_str, dev = self.dev, mp = self.mount_point), ignore_error = True) + ret = utils.shcmd( + f"mount -t f2fs {opt_str} {self.dev} {self.mount_point}", ignore_error=True + ) if ret != 0: - raise RuntimeError("Failed to mount dev:{} to dir:{}".format( - self.dev, self.mount_point)) + raise RuntimeError( + f"Failed to mount dev:{self.dev} to dir:{self.mount_point}" + ) + def sysfs_setup(self, option, value): """ This function sets up the parameters in sysfs. Option is the file name in sysfs. """ devname = os.path.basename(self.dev) - folder = '/sys/fs/f2fs/{dev}'.format(dev = devname) + folder = f"/sys/fs/f2fs/{devname}" path = os.path.join(folder, option) - with open(path, 'w') as f: + with open(path, "w") as f: f.write(str(value)) + class Btrfs(FileSystemBase): def make(self, opt_dic=None): opt_str = opts_to_str(opt_dic) - ret = utils.shcmd('mkfs.btrfs -f {opt} --nodiscard {dev}'.format( - opt=opt_str, dev = self.dev), ignore_error = True) + ret = utils.shcmd( + f"mkfs.btrfs -f {opt_str} --nodiscard {self.dev}", ignore_error=True + ) if ret != 0: - raise RuntimeError("Failed to make dev:{}".format(self.dev)) + raise RuntimeError(f"Failed to make dev:{self.dev}") + class Xfs(FileSystemBase): def make(self, opt_dic=None): if opt_dic == None: - opt_str = '' + opt_str = "" else: - items = [ ' '.join([k,v]) for k,v in opt_dic.items() ] - opt_str = ' '.join(items) + items = [" ".join([k, v]) for k, v in list(opt_dic.items())] + opt_str = " ".join(items) - ret = utils.shcmd("mkfs.xfs {opt} -K -f -s size=4096 -b size=4096 {dev}"\ - .format(opt = opt_str, dev = self.dev), ignore_error = True) + ret = utils.shcmd( + f"mkfs.xfs {opt_str} -K -f -s size=4096 -b size=4096 {self.dev}", + ignore_error=True, + ) if ret != 0: - raise RuntimeError("Failed to make dev:{}".format(self.dev)) + raise RuntimeError(f"Failed to make dev:{self.dev}") + # loopdev = LoopDevice(dev_path = '/dev/loop0', tmpfs_mount_point = '/mnt/tmpfs', - # size_mb = 4096) +# size_mb = 4096) # loopdev.create() # ext4 = Ext4(device='/dev/loop0', mount_point='/mnt/fsonloop') @@ -192,5 +207,3 @@ def make(self, opt_dic=None): # f2fs = F2fs(device='/dev/loop0', mount_point='/mnt/fsonloop') # f2fs.make() # f2fs.mount() - - diff --git a/wiscsee/workrunner/fshelper.py b/wiscsee/workrunner/fshelper.py index 2a50dec..fecc995 100644 --- a/wiscsee/workrunner/fshelper.py +++ b/wiscsee/workrunner/fshelper.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +from past.utils import old_div import json import os import re @@ -8,31 +9,36 @@ from utilities import utils from commons import * + def umountFS(mountpoint): cmd = ["umount", mountpoint] p = subprocess.Popen(cmd) p.wait() return p.returncode -def ext4_make(devname, blocksize=4096, makeopts=None): +def ext4_make(devname, blocksize=4096, makeopts=None): if makeopts == None: - cmd = ["mkfs.ext4", - "-b", blocksize, - "-O", "^has_journal,extent,huge_file,flex_bg,uninit_bg,dir_nlink,extra_isize", - devname] + cmd = [ + "mkfs.ext4", + "-b", + blocksize, + "-O", + "^has_journal,extent,huge_file,flex_bg,uninit_bg,dir_nlink,extra_isize", + devname, + ] else: - cmd = ["mkfs.ext4", - "-b", blocksize] + cmd = ["mkfs.ext4", "-b", blocksize] cmd.extend(makeopts) cmd.extend([devname]) cmd = [str(x) for x in cmd] p = subprocess.Popen(cmd) p.wait() - print "makeExt4:", p.returncode + print(("makeExt4:", p.returncode)) return p.returncode + def ext4_mount(devname, mountpoint): if not os.path.exists(mountpoint): os.makedirs(mountpoint) @@ -40,62 +46,68 @@ def ext4_mount(devname, mountpoint): cmd = ["mount", "-t", "ext4", "-o", "discard", devname, mountpoint] p = subprocess.Popen(cmd) p.wait() - print "mountExt4:", p.returncode + print(("mountExt4:", p.returncode)) return p.returncode + # def ext4_create_on_loop(): - # makeLoopDevice(config["loop_path"], config["tmpfs_mount_point"], 4096, img_file=None) - # ext4_make(config["loop_path"], blocksize=4096, makeopts=None) - # ext4_mount(devname=config["loop_path"], mountpoint=config["fs_mount_point"]) +# makeLoopDevice(config["loop_path"], config["tmpfs_mount_point"], 4096, img_file=None) +# ext4_make(config["loop_path"], blocksize=4096, makeopts=None) +# ext4_mount(devname=config["loop_path"], mountpoint=config["fs_mount_point"]) + def ext4_make_simple(): ret = ext4_make(config["loop_path"], blocksize=4096, makeopts=None) if ret != 0: - print 'error in ext4_make_simple()' + print("error in ext4_make_simple()") exit(1) + def ext4_mount_simple(): ret = ext4_mount(devname=config["loop_path"], mountpoint=config["fs_mount_point"]) if ret != 0: - print 'error in ext4_mount_simple()' + print("error in ext4_mount_simple()") exit(1) + def mkLoopDevOnFile(devname, filepath): - cmd = ['losetup', devname, filepath] + cmd = ["losetup", devname, filepath] cmd = [str(x) for x in cmd] - print " ".join(cmd), "......" + print((" ".join(cmd), "......")) proc = subprocess.Popen(cmd) proc.wait() return proc.returncode + def delLoopDev(devname): - cmd = ['losetup', '-d', devname] + cmd = ["losetup", "-d", devname] cmd = [str(x) for x in cmd] - print " ".join(cmd), "......" + print((" ".join(cmd), "......")) proc = subprocess.Popen(cmd) proc.wait() return proc.returncode + def isMounted(name): "only check is a name is in mounted list" - name = name.rstrip('/') - print "isMounted: name:", name - with open('/etc/mtab', 'r') as f: + name = name.rstrip("/") + print(("isMounted: name:", name)) + with open("/etc/mtab") as f: for line in f: - #print "line:", line, - line = " " + line + " " # a hack - if re.search(r'\s'+name+r'\s', line): - #print " YES" + # print "line:", line, + line = " " + line + " " # a hack + if re.search(r"\s" + name + r"\s", line): + # print " YES" return True - #print " NO" + # print " NO" return False + def isLoopDevUsed(path): - cmd = ['losetup','-f'] - proc = subprocess.Popen(cmd, - stdout=subprocess.PIPE) + cmd = ["losetup", "-f"] + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE) proc.wait() outstr = proc.communicate()[0] @@ -105,16 +117,20 @@ def isLoopDevUsed(path): else: return False + def umountFS(mountpoint): cmd = ["umount", mountpoint] p = subprocess.Popen(cmd) p.wait() return p.returncode + def make_loop_device(devname, tmpfs_mountpoint, sizeMB, img_file=None): "size is in MB. The tmpfs for this device might be bigger than sizeMB" - if not devname.startswith('/dev/loop'): - raise RuntimeError('you are requesting to create loop device on a non-loop device path') + if not devname.startswith("/dev/loop"): + raise RuntimeError( + "you are requesting to create loop device on a non-loop device path" + ) if not os.path.exists(tmpfs_mountpoint): os.makedirs(tmpfs_mountpoint) @@ -122,58 +138,57 @@ def make_loop_device(devname, tmpfs_mountpoint, sizeMB, img_file=None): # umount the FS mounted on loop dev if isMounted(devname): if umountFS(devname) != 0: - raise RuntimeError("unable to umount {}".format(devname)) + raise RuntimeError(f"unable to umount {devname}") else: - print devname, 'umounted' + print((devname, "umounted")) else: - print devname, "is not mounted" + print((devname, "is not mounted")) # delete the loop device if isLoopDevUsed(devname): if delLoopDev(devname) != 0: raise RuntimeError("!!!!!!!!!!!!! Failed to delete loop device") else: - print devname, 'is deleted' + print((devname, "is deleted")) else: - print devname, "is not in use" - + print((devname, "is not in use")) # umount the tmpfs the loop device is on if isMounted(tmpfs_mountpoint): if umountFS(tmpfs_mountpoint) != 0: - raise RuntimeError("unable to umount tmpfs at {}".format(tmpfs_mountpoint)) - print tmpfs_mountpoint, "umounted" + raise RuntimeError(f"unable to umount tmpfs at {tmpfs_mountpoint}") + print((tmpfs_mountpoint, "umounted")) else: - print tmpfs_mountpoint, "is not mounted" + print((tmpfs_mountpoint, "is not mounted")) - - mountTmpfs(tmpfs_mountpoint, int(sizeMB*1024*1024*1.1)) + mountTmpfs(tmpfs_mountpoint, int(sizeMB * 1024 * 1024 * 1.1)) imgpath = os.path.join(tmpfs_mountpoint, "disk.img") if img_file == None: mkImageFile(imgpath, sizeMB) else: - cmd = ['cp', img_file, imgpath] + cmd = ["cp", img_file, imgpath] subprocess.call(cmd) ret = mkLoopDevOnFile(devname, imgpath) if ret != 0: raise RuntimeError("Failed at losetup") + def mkImageFile(filepath, size): "size is in MB" - cmd = ['truncate', '-s', str(size*1024*1024), filepath] - print " ".join(cmd), "......" + cmd = ["truncate", "-s", str(size * 1024 * 1024), filepath] + print((" ".join(cmd), "......")) proc = subprocess.Popen(cmd) proc.wait() return proc.returncode + def mountTmpfs(mountpoint, size): if not os.path.exists(mountpoint): os.makedirs(mountpoint) - cmd = ['mount', '-t', 'tmpfs', - '-o', 'size='+str(size), 'tmpfs', mountpoint] + cmd = ["mount", "-t", "tmpfs", "-o", "size=" + str(size), "tmpfs", mountpoint] cmd = [str(x) for x in cmd] - print " ".join(cmd), "......" + print((" ".join(cmd), "......")) proc = subprocess.Popen(cmd) proc.wait() @@ -181,7 +196,8 @@ def mountTmpfs(mountpoint, size): # def prepare_loop(): - # make_loop_device(config["loop_path"], config["tmpfs_mount_point"], 4096, img_file=None) +# make_loop_device(config["loop_path"], config["tmpfs_mount_point"], 4096, img_file=None) + def partition_disk(dev, part_sizes, padding): """ @@ -195,51 +211,42 @@ def partition_disk(dev, part_sizes, padding): success = False while n_tries > 0: n_tries -= 1 - ret = utils.shcmd("sudo sfdisk {} < /tmp/my.layout".format(dev), - ignore_error = True) + ret = utils.shcmd(f"sudo sfdisk {dev} < /tmp/my.layout", ignore_error=True) if ret == 0: success = True break - print 'parition failed', n_tries, 'left' + print(("parition failed", n_tries, "left")) time.sleep(1) if success == False: raise RuntimeError("Fail when doing sfdisk") - utils.shcmd("sudo partprobe -s {}".format(dev)) + utils.shcmd(f"sudo partprobe -s {dev}") # partition table of /dev/sdb -#unit: sectors +# unit: sectors # -#/dev/sdb1 : start= 4096, size=125829120, Id=a5 -#/dev/sdb2 : start=125833216, size=125829120, Id=83 -#/dev/sdb3 : start= 0, size= 0, Id= 0 -#/dev/sdb4 : start= 0, size= 0, Id= 0 +# /dev/sdb1 : start= 4096, size=125829120, Id=a5 +# /dev/sdb2 : start=125833216, size=125829120, Id=83 +# /dev/sdb3 : start= 0, size= 0, Id= 0 +# /dev/sdb4 : start= 0, size= 0, Id= 0 -def create_layout_file(part_sizes, padding=8*MB): + +def create_layout_file(part_sizes, padding=8 * MB): sector_size = 512 - lines = ["unit: sectors", ''] + lines = ["unit: sectors", ""] # Id is to specify Linux/FreeBSD/swap... line_temp = "/dev/sdb{id} : start= {start}, size={size}, Id=83" - cur_sectors = padding / sector_size # start with 8 sector + cur_sectors = old_div(padding, sector_size) # start with 8 sector for i, partsize in enumerate(part_sizes): - size_in_sector = partsize / sector_size - line = line_temp.format(id=i, start = cur_sectors, - size = size_in_sector) + size_in_sector = old_div(partsize, sector_size) + line = line_temp.format(id=i, start=cur_sectors, size=size_in_sector) lines.append(line) cur_sectors += size_in_sector - with open('/tmp/my.layout', 'w') as f: - f.write('\n'.join(lines)) - f.write('\n') - - - - - - - - + with open("/tmp/my.layout", "w") as f: + f.write("\n".join(lines)) + f.write("\n") diff --git a/wiscsee/workrunner/lbaworkloadgenerator.py b/wiscsee/workrunner/lbaworkloadgenerator.py index a0a4b34..8df08e3 100644 --- a/wiscsee/workrunner/lbaworkloadgenerator.py +++ b/wiscsee/workrunner/lbaworkloadgenerator.py @@ -1,23 +1,22 @@ +from past.utils import old_div import abc import random import config -import workload +from . import workload from wiscsim import hostevent from commons import * from pyreuse.general.zipf import ZipfGenerator -class LBAWorkloadGenerator(object): - __metaclass__ = abc.ABCMeta +class LBAWorkloadGenerator(metaclass=abc.ABCMeta): @abc.abstractmethod def __iter__(self): return -class LBAMultiProcGenerator(object): - __metaclass__ = abc.ABCMeta +class LBAMultiProcGenerator(metaclass=abc.ABCMeta): @abc.abstractmethod def get_iter_list(self): return @@ -26,34 +25,42 @@ def get_iter_list(self): class SampleWorkload(LBAWorkloadGenerator): def __init__(self, conf): self.conf = conf - self.sector_size = self.conf['sector_size'] + self.sector_size = self.conf["sector_size"] def __iter__(self): - yield hostevent.Event(sector_size=self.sector_size, - pid=0, operation=OP_ENABLE_RECORDER, - offset=0, size=0) - - yield hostevent.Event(sector_size=self.sector_size, - pid=0, operation=OP_WRITE, - offset=0, size=3*MB) + yield hostevent.Event( + sector_size=self.sector_size, + pid=0, + operation=OP_ENABLE_RECORDER, + offset=0, + size=0, + ) + + yield hostevent.Event( + sector_size=self.sector_size, + pid=0, + operation=OP_WRITE, + offset=0, + size=3 * MB, + ) class TestWorkload(LBAWorkloadGenerator): def __init__(self, confobj): if not isinstance(confobj, config.Config): - raise TypeError("confobj is not config.Config. It is {}". - format(type(confobj).__name__)) + raise TypeError( + f"confobj is not config.Config. It is {type(confobj).__name__}" + ) self.conf = confobj - self.sector_size = self.conf['sector_size'] - self.ftl_type = self.conf['ftl_type'] - if self.ftl_type == 'dftlext': + self.sector_size = self.conf["sector_size"] + self.ftl_type = self.conf["ftl_type"] + if self.ftl_type == "dftlext": self.over_provisioning = self.conf.over_provisioning - elif self.ftl_type == 'nkftl': - self.over_provisioning = self.conf['nkftl']['provision_ratio'] + elif self.ftl_type == "nkftl": + self.over_provisioning = self.conf["nkftl"]["provision_ratio"] else: - raise RuntimeError("FTL type {} is not supported".format( - self.ftl_type)) + raise RuntimeError(f"FTL type {self.ftl_type} is not supported") def test_random(self): w = OP_WRITE @@ -64,57 +71,66 @@ def test_random(self): events = [] maxpage = 0 - lba_span = int(self.conf.total_num_pages() / self.over_provisioning) + lba_span = int(old_div(self.conf.total_num_pages(), self.over_provisioning)) for i in range(10): op = random.choice(ops) page = int(random.random() * lba_span) if maxpage < page: maxpage = page - events.append( (op, page) ) + events.append((op, page)) for page in range(10): - events.append( (r, page) ) + events.append((r, page)) return events def __iter__(self): - yield hostevent.Event(sector_size = self.sector_size, - pid = 0, operation = OP_ENABLE_RECORDER, - offset = 0, size = 0) + yield hostevent.Event( + sector_size=self.sector_size, + pid=0, + operation=OP_ENABLE_RECORDER, + offset=0, + size=0, + ) events = self.test_random() for op, lpn in events: offset = lpn * self.conf.page_size size = self.conf.page_size - event = hostevent.Event(sector_size = self.sector_size, - pid = 0, operation = op, offset = offset, - size = size) + event = hostevent.Event( + sector_size=self.sector_size, + pid=0, + operation=op, + offset=offset, + size=size, + ) yield event class ExtentTestWorkload(LBAWorkloadGenerator): def __init__(self, confobj): if not isinstance(confobj, config.Config): - raise TypeError("confobj is not config.Config. It is {}". - format(type(confobj).__name__)) + raise TypeError( + f"confobj is not config.Config. It is {type(confobj).__name__}" + ) self.conf = confobj - self.sector_size = self.conf['sector_size'] - self.ftl_type = self.conf['ftl_type'] - if self.ftl_type in ('dftlext', 'dftldes', 'dftldes'): + self.sector_size = self.conf["sector_size"] + self.ftl_type = self.conf["ftl_type"] + if self.ftl_type in ("dftlext", "dftldes", "dftldes"): self.over_provisioning = self.conf.over_provisioning - elif self.ftl_type == 'nkftl2': + elif self.ftl_type == "nkftl2": self.over_provisioning = 1 else: - raise RuntimeError("FTL type {} is not supported".format( - self.ftl_type)) + raise RuntimeError(f"FTL type {self.ftl_type} is not supported") - self.op_count = self.conf['lba_workload_configs']\ - ['ExtentTestWorkload']['op_count'] + self.op_count = self.conf["lba_workload_configs"]["ExtentTestWorkload"][ + "op_count" + ] if isinstance(self.conf, config.ConfigNewFlash): - self.page_size = self.conf['flash_config']['page_size'] + self.page_size = self.conf["flash_config"]["page_size"] else: self.page_size = self.conf.page_size @@ -127,9 +143,9 @@ def test_random(self): events = [] maxpage = 0 - lba_span = int(self.conf.total_num_pages() / self.over_provisioning) - print 'total num pages', self.conf.total_num_pages() - print 'lba_span', lba_span + lba_span = int(old_div(self.conf.total_num_pages(), self.over_provisioning)) + print(("total num pages", self.conf.total_num_pages())) + print(("lba_span", lba_span)) max_access_pages = 16 for i in range(self.op_count): @@ -138,56 +154,65 @@ def test_random(self): npages = random.randint(1, max_access_pages) if maxpage < page: maxpage = page - events.append( (op, page, npages) ) + events.append((op, page, npages)) for page in range(maxpage): - events.append( (r, page, 1) ) + events.append((r, page, 1)) return events def __iter__(self): - yield hostevent.Event(sector_size = self.sector_size, - pid = 0, operation = OP_ENABLE_RECORDER, - offset = 0, size = 0) + yield hostevent.Event( + sector_size=self.sector_size, + pid=0, + operation=OP_ENABLE_RECORDER, + offset=0, + size=0, + ) events = self.test_random() for op, lpn, npages in events: offset = lpn * self.page_size size = self.page_size * npages - event = hostevent.Event(sector_size = self.sector_size, - pid = 0, operation = op, offset = offset, - size = size) + event = hostevent.Event( + sector_size=self.sector_size, + pid=0, + operation=op, + offset=offset, + size=size, + ) yield event + class TestWorkloadFLEX3(LBAWorkloadGenerator): def __init__(self, confobj): if not isinstance(confobj, config.Config): - raise TypeError("confobj is not config.Config. It is {}". - format(type(confobj).__name__)) + raise TypeError( + f"confobj is not config.Config. It is {type(confobj).__name__}" + ) self.conf = confobj - self.sector_size = self.conf['sector_size'] - self.ftl_type = self.conf['ftl_type'] - if self.ftl_type in ('dftlext', 'dftldes', 'dftldes'): + self.sector_size = self.conf["sector_size"] + self.ftl_type = self.conf["ftl_type"] + if self.ftl_type in ("dftlext", "dftldes", "dftldes"): self.over_provisioning = self.conf.over_provisioning - elif self.ftl_type == 'nkftl': - self.over_provisioning = self.conf['nkftl']['provision_ratio'] + elif self.ftl_type == "nkftl": + self.over_provisioning = self.conf["nkftl"]["provision_ratio"] else: - raise RuntimeError("FTL type {} is not supported".format( - self.ftl_type)) - - self.op_count = self.conf['lba_workload_configs']\ - ['TestWorkloadFLEX3']['op_count'] - self.extent_size = self.conf['lba_workload_configs']\ - ['TestWorkloadFLEX3']['extent_size'] - self.ops = self.conf['lba_workload_configs']\ - ['TestWorkloadFLEX3']['ops'] - self.mode = self.conf['lba_workload_configs']\ - ['TestWorkloadFLEX3']['mode'] + raise RuntimeError(f"FTL type {self.ftl_type} is not supported") + + self.op_count = self.conf["lba_workload_configs"]["TestWorkloadFLEX3"][ + "op_count" + ] + self.extent_size = self.conf["lba_workload_configs"]["TestWorkloadFLEX3"][ + "extent_size" + ] + self.ops = self.conf["lba_workload_configs"]["TestWorkloadFLEX3"]["ops"] + self.mode = self.conf["lba_workload_configs"]["TestWorkloadFLEX3"]["mode"] if isinstance(self.conf, config.ConfigNewFlash): - self.page_size = self.conf['flash_config']['page_size'] + self.page_size = self.conf["flash_config"]["page_size"] else: self.page_size = self.conf.page_size @@ -197,75 +222,105 @@ def generate_events(self): events = [] maxpage = 0 - logical_span = int(self.conf.total_num_pages() / self.over_provisioning) - print 'Total num pages', self.conf.total_num_pages(), "pages",\ - self.conf.total_num_pages() * \ - self.conf['flash_config']['page_size'] / MB, "MB" - print "Total blocks", self.conf.n_blocks_per_dev - print "Total logical blocks", logical_span / \ - self.conf['flash_config']['n_pages_per_block'] - print 'Logical_span', logical_span, "pages", logical_span *\ - self.conf['flash_config']['page_size'] / MB, "MB" + logical_span = int(old_div(self.conf.total_num_pages(), self.over_provisioning)) + print( + ( + "Total num pages", + self.conf.total_num_pages(), + "pages", + old_div( + self.conf.total_num_pages() + * self.conf["flash_config"]["page_size"], + MB, + ), + "MB", + ) + ) + print(("Total blocks", self.conf.n_blocks_per_dev)) + print( + ( + "Total logical blocks", + old_div(logical_span, self.conf["flash_config"]["n_pages_per_block"]), + ) + ) + print( + ( + "Logical_span", + logical_span, + "pages", + old_div(logical_span * self.conf["flash_config"]["page_size"], MB), + "MB", + ) + ) for i in range(self.op_count): op = random.choice(ops) - if self.mode == 'random': + if self.mode == "random": page = int(random.random() * (logical_span - self.extent_size)) assert page < logical_span - elif self.mode == 'sequential': + elif self.mode == "sequential": page = i * self.extent_size else: - raise RuntimeError("{} not supported".format(self.mode)) + raise RuntimeError(f"{self.mode} not supported") npages = self.extent_size - events.append( (op, page, npages) ) + events.append((op, page, npages)) return events def __iter__(self): - yield hostevent.Event(sector_size = self.sector_size, - pid = 0, operation = OP_ENABLE_RECORDER, - offset = 0, size = 0) + yield hostevent.Event( + sector_size=self.sector_size, + pid=0, + operation=OP_ENABLE_RECORDER, + offset=0, + size=0, + ) events = self.generate_events() for op, lpn, npages in events: offset = lpn * self.page_size size = self.page_size * npages - event = hostevent.Event(sector_size = self.sector_size, - pid = 0, operation = op, offset = offset, - size = size) + event = hostevent.Event( + sector_size=self.sector_size, + pid=0, + operation=op, + offset=offset, + size=size, + ) yield event class ExtentTestWorkloadFLEX2(LBAWorkloadGenerator): def __init__(self, confobj): if not isinstance(confobj, config.Config): - raise TypeError("confobj is not config.Config. It is {}". - format(type(confobj).__name__)) + raise TypeError( + f"confobj is not config.Config. It is {type(confobj).__name__}" + ) self.conf = confobj - self.sector_size = self.conf['sector_size'] - self.ftl_type = self.conf['ftl_type'] + self.sector_size = self.conf["sector_size"] + self.ftl_type = self.conf["ftl_type"] - if self.ftl_type in ('dftlext', 'dftldes'): + if self.ftl_type in ("dftlext", "dftldes"): self.over_provisioning = self.conf.over_provisioning - elif self.ftl_type == 'nkftl': - self.over_provisioning = self.conf['nkftl']['provision_ratio'] + elif self.ftl_type == "nkftl": + self.over_provisioning = self.conf["nkftl"]["provision_ratio"] else: - raise RuntimeError("FTL type {} is not supported".format( - self.ftl_type)) + raise RuntimeError(f"FTL type {self.ftl_type} is not supported") - self.events = self.conf['lba_workload_configs']\ - ['ExtentTestWorkloadFLEX2']['events'] + self.events = self.conf["lba_workload_configs"]["ExtentTestWorkloadFLEX2"][ + "events" + ] if isinstance(self.conf, config.ConfigNewFlash): - self.page_size = self.conf['flash_config']['page_size'] + self.page_size = self.conf["flash_config"]["page_size"] else: self.page_size = self.conf.page_size def check(self): - lba_span = int(self.conf.total_num_pages() / self.over_provisioning) - print 'total num pages', self.conf.total_num_pages() - print 'lba_span', lba_span + lba_span = int(old_div(self.conf.total_num_pages(), self.over_provisioning)) + print(("total num pages", self.conf.total_num_pages())) + print(("lba_span", lba_span)) ret = [] for event in self.events: @@ -276,72 +331,83 @@ def check(self): return ret def __iter__(self): - yield hostevent.Event(sector_size = self.sector_size, - pid = 0, operation = OP_ENABLE_RECORDER, - offset = 0, size = 0) + yield hostevent.Event( + sector_size=self.sector_size, + pid=0, + operation=OP_ENABLE_RECORDER, + offset=0, + size=0, + ) events = self.check() for op, lpn, npages in events: offset = lpn * self.page_size size = self.page_size * npages - event = hostevent.Event(sector_size = self.sector_size, - pid = 0, operation = op, offset = offset, - size = size) + event = hostevent.Event( + sector_size=self.sector_size, + pid=0, + operation=op, + offset=offset, + size=size, + ) yield event class AccessesWithDist(LBAWorkloadGenerator): def __init__(self, conf): self.conf = conf - self.sector_size = self.conf['sector_size'] - - self.distribution = self.conf['AccessesWithDist']['lba_access_dist'] - self.traffic_size = self.conf['AccessesWithDist']['traffic_size'] - self.chunk_size = self.conf['AccessesWithDist']['chunk_size'] - self.space_size = self.conf['AccessesWithDist']['space_size'] - self.skew_factor = self.conf['AccessesWithDist']['skew_factor'] - self.zipf_alpha = self.conf['AccessesWithDist']['zipf_alpha'] - self.lbabytes = self.conf['dev_size_mb'] * MB + self.sector_size = self.conf["sector_size"] + self.distribution = self.conf["AccessesWithDist"]["lba_access_dist"] + self.traffic_size = self.conf["AccessesWithDist"]["traffic_size"] + self.chunk_size = self.conf["AccessesWithDist"]["chunk_size"] + self.space_size = self.conf["AccessesWithDist"]["space_size"] + self.skew_factor = self.conf["AccessesWithDist"]["skew_factor"] + self.zipf_alpha = self.conf["AccessesWithDist"]["zipf_alpha"] + self.lbabytes = self.conf["dev_size_mb"] * MB def __iter__(self): - yield hostevent.Event(sector_size=self.sector_size, - pid=0, operation=OP_ENABLE_RECORDER, - offset=0, size=0) + yield hostevent.Event( + sector_size=self.sector_size, + pid=0, + operation=OP_ENABLE_RECORDER, + offset=0, + size=0, + ) # yield hostevent.Event(sector_size=self.sector_size, - # pid=0, operation=OP_WRITE, - # offset=0, size=3*MB) + # pid=0, operation=OP_WRITE, + # offset=0, size=3*MB) - if self.distribution == 'uniform': + if self.distribution == "uniform": for req in self.uniform_events(): yield req - elif self.distribution == 'hotcold': + elif self.distribution == "hotcold": for req in self.hot_cold_space_event(): yield req - elif self.distribution == 'zipf': + elif self.distribution == "zipf": for req in self.zipf_events(): yield req else: - raise NotImplementedError('distribution {} not implemented'.format( - self.distribution)) + raise NotImplementedError( + f"distribution {self.distribution} not implemented" + ) def uniform_events(self): chunk_size = self.chunk_size traffic_size = self.traffic_size - n_chunks_in_traffic = traffic_size / chunk_size - n_chunks_in_space = self.space_size / chunk_size + n_chunks_in_traffic = old_div(traffic_size, chunk_size) + n_chunks_in_space = old_div(self.space_size, chunk_size) for i in range(n_chunks_in_traffic): chunk_id = random.randint(0, n_chunks_in_space - 1) yield self.get_write_event(chunk_id) - def hot_cold_space_event(self): """ first half cold, second half hot @@ -349,9 +415,9 @@ def hot_cold_space_event(self): chunk_size = self.chunk_size traffic_size = self.traffic_size - n_chunks_in_traffic = traffic_size / chunk_size - n_chunks_in_space = self.space_size / chunk_size - n_chunks_in_half_space = int(n_chunks_in_space / 2) + n_chunks_in_traffic = old_div(traffic_size, chunk_size) + n_chunks_in_space = old_div(self.space_size, chunk_size) + n_chunks_in_half_space = int(old_div(n_chunks_in_space, 2)) n_chunks_written = 0 @@ -382,23 +448,28 @@ def zipf_events(self): chunk_size = self.chunk_size traffic_size = self.traffic_size - n_chunks_in_traffic = traffic_size / chunk_size - n_chunks_in_space = self.space_size / chunk_size + n_chunks_in_traffic = old_div(traffic_size, chunk_size) + n_chunks_in_space = old_div(self.space_size, chunk_size) zipfgen = ZipfGenerator(n_chunks_in_space, self.zipf_alpha) for i in range(n_chunks_in_traffic): - chunk_id = zipfgen.next() + chunk_id = next(zipfgen) yield self.get_write_event(chunk_id) def get_write_event(self, chunk_id): offset = chunk_id * self.chunk_size size = self.chunk_size - return hostevent.Event(sector_size=self.sector_size, - pid=0, operation=OP_WRITE, offset=offset, size=size) + return hostevent.Event( + sector_size=self.sector_size, + pid=0, + operation=OP_WRITE, + offset=offset, + size=size, + ) -class BarrierGen(object): +class BarrierGen: def __init__(self, n_ncq_slots): self.n_ncq_slots = n_ncq_slots @@ -413,19 +484,18 @@ def barrier_events(self): class BlktraceEvents(LBAWorkloadGenerator): def __init__(self, confobj): if not isinstance(confobj, config.Config): - raise TypeError("confobj is not config.Config. It is {}". - format(type(confobj).__name__)) + raise TypeError( + f"confobj is not config.Config. It is {type(confobj).__name__}" + ) self.conf = confobj - self.mkfs_event_path = self.conf['lba_workload_configs']\ - ['mkfs_event_path'] - self.ftlsim_event_path = self.conf['lba_workload_configs']\ - ['ftlsim_event_path'] + self.mkfs_event_path = self.conf["lba_workload_configs"]["mkfs_event_path"] + self.ftlsim_event_path = self.conf["lba_workload_configs"]["ftlsim_event_path"] - self.stop_on_bytes = self.conf['stop_sim_on_bytes'] + self.stop_on_bytes = self.conf["stop_sim_on_bytes"] - if str(self.stop_on_bytes).lower() in ('inf', 'infinity', 'infinit'): - self.stop_on_bytes = float('inf') + if str(self.stop_on_bytes).lower() in ("inf", "infinity", "infinit"): + self.stop_on_bytes = float("inf") def __iter__(self): barriergen = BarrierGen(self.conf.ssd_ncq_depth()) @@ -444,16 +514,14 @@ def __iter__(self): for event in self.gc_event(): yield event - for req in barriergen.barrier_events(): - yield req + yield from barriergen.barrier_events() yield hostevent.ControlEvent(operation=OP_REC_BW) def prepfs_events(self): prepfs_line_iter = hostevent.FileLineIterator(self.mkfs_event_path) event_prepfs_iter = hostevent.EventIterator(self.conf, prepfs_line_iter) - for event in event_prepfs_iter: - yield event + yield from event_prepfs_iter def target_workload_events(self): # special event indicates the start of workload @@ -461,8 +529,9 @@ def target_workload_events(self): yield hostevent.ControlEvent(operation=OP_ENABLE_RECORDER) for req in barriergen.barrier_events(): yield req - yield hostevent.ControlEvent(operation=OP_REC_TIMESTAMP, - arg1='interest_workload_start') + yield hostevent.ControlEvent( + operation=OP_REC_TIMESTAMP, arg1="interest_workload_start" + ) workload_line_iter = hostevent.FileLineIterator(self.ftlsim_event_path) event_workload_iter = hostevent.EventIterator(self.conf, workload_line_iter) @@ -471,27 +540,25 @@ def target_workload_events(self): for event in event_workload_iter: yield event - if event.operation in [OP_READ, OP_WRITE] and event.action == 'D': + if event.operation in [OP_READ, OP_WRITE] and event.action == "D": total_rw_bytes += event.size if total_rw_bytes >= self.stop_on_bytes: - print 'break! stop on ', self.stop_on_bytes/MB + print(("break! stop on ", old_div(self.stop_on_bytes, MB))) break for req in barriergen.barrier_events(): yield req - yield hostevent.ControlEvent(operation=OP_REC_TIMESTAMP, - arg1='interest_workload_end') + yield hostevent.ControlEvent( + operation=OP_REC_TIMESTAMP, arg1="interest_workload_end" + ) def gc_event(self): barriergen = BarrierGen(self.conf.ssd_ncq_depth()) - if self.conf['do_gc_after_workload'] is True: - for req in barriergen.barrier_events(): - yield req - yield hostevent.ControlEvent(operation=OP_REC_TIMESTAMP, - arg1='gc_start_timestamp') + if self.conf["do_gc_after_workload"] is True: + yield from barriergen.barrier_events() + yield hostevent.ControlEvent( + operation=OP_REC_TIMESTAMP, arg1="gc_start_timestamp" + ) yield hostevent.ControlEvent(operation=OP_CLEAN) - - - diff --git a/wiscsee/workrunner/multiwriters.py b/wiscsee/workrunner/multiwriters.py index 5e328f9..ca123ae 100644 --- a/wiscsee/workrunner/multiwriters.py +++ b/wiscsee/workrunner/multiwriters.py @@ -1,3 +1,4 @@ +from past.utils import old_div import subprocess import pprint @@ -5,16 +6,17 @@ MB = 2**20 GB = 2**30 -translator = { 'file_size': '-f', - 'write_size': '-w', - 'n_writes': '-n', - 'pattern': '-p', - 'fsync': '-y', - 'sync': '-s', - 'file_path': '-l', - 'tag': '-t', - 'markerfile': '-m' - } +translator = { + "file_size": "-f", + "write_size": "-w", + "n_writes": "-n", + "pattern": "-p", + "fsync": "-y", + "sync": "-s", + "file_path": "-l", + "tag": "-t", + "markerfile": "-m", +} def parse_player_runtime_out(lines): @@ -27,7 +29,7 @@ def parse_player_runtime_out(lines): return d -class MultiWriters(object): +class MultiWriters: def __init__(self, player_path, parameters): """ parameters is a list of dictionaries @@ -47,8 +49,10 @@ def __init__(self, player_path, parameters): """ args = [] for para in parameters: - arg = [player_path, ] - for k, v in para.items(): + arg = [ + player_path, + ] + for k, v in list(para.items()): arg.append(translator[k]) arg.append(str(v)) args.append(arg) @@ -60,8 +64,8 @@ def run(self): procs = [] for args in self.args_table: - print ' '.join(args) - p = subprocess.Popen(args, stdout = subprocess.PIPE) + print(" ".join(args)) + p = subprocess.Popen(args, stdout=subprocess.PIPE) procs.append(p) for p in procs: @@ -70,12 +74,11 @@ def run(self): results = [] for p in procs: if p.returncode != 0: - raise RuntimeError("multiwriter process fails. PID={}".format( - p.pid)) + raise RuntimeError(f"multiwriter process fails. PID={p.pid}") - lines = p.communicate()[0].split('\n') + lines = p.communicate()[0].split("\n") d = parse_player_runtime_out(lines) - d['pid.python'] = p.pid + d["pid.python"] = p.pid results.append(d) # pprint.pprint( results ) @@ -84,30 +87,31 @@ def run(self): def main(): parameters = [ - { 'file_size': 256 * MB, - 'write_size': 64 * KB, - 'n_writes': 4 * 256 * MB / (64 * KB), - 'pattern': 'random', - 'fsync': 1, - 'sync': 0, - 'file_path': '/mnt/fsonloop/file01', - 'tag': 'mytag001' - }, - { 'file_size': 256 * MB, - 'write_size': 64 * KB, - 'n_writes': 4 * 256 * MB / (64 * KB), - 'pattern': 'random', - 'fsync': 1, - 'sync': 0, - 'file_path': '/mnt/fsonloop/file01', - 'tag': 'mytag002' - } + { + "file_size": 256 * MB, + "write_size": 64 * KB, + "n_writes": old_div(4 * 256 * MB, (64 * KB)), + "pattern": "random", + "fsync": 1, + "sync": 0, + "file_path": "/mnt/fsonloop/file01", + "tag": "mytag001", + }, + { + "file_size": 256 * MB, + "write_size": 64 * KB, + "n_writes": old_div(4 * 256 * MB, (64 * KB)), + "pattern": "random", + "fsync": 1, + "sync": 0, + "file_path": "/mnt/fsonloop/file01", + "tag": "mytag002", + }, ] - mw = MultiWriters('./player-runtime', parameters) - pprint.pprint( mw.run() ) + mw = MultiWriters("./player-runtime", parameters) + pprint.pprint(mw.run()) -if __name__ == '__main__': +if __name__ == "__main__": main() - diff --git a/wiscsee/workrunner/nonblockingreader.py b/wiscsee/workrunner/nonblockingreader.py index 7e700d8..5a5c236 100644 --- a/wiscsee/workrunner/nonblockingreader.py +++ b/wiscsee/workrunner/nonblockingreader.py @@ -1,18 +1,19 @@ -import sys, threading, Queue +import sys, threading, queue as Queue + def enqueue_lines(f, line_queue): - for line in iter(f.readline, b''): + for line in iter(f.readline, b""): line_queue.put(line) -class NonBlockingReader(object): + +class NonBlockingReader: def __init__(self, file_path): """Note that the thread is started once the instance is created""" self.file_path = file_path - self.f = open(file_path, 'r') + self.f = open(file_path) self.q = Queue.Queue() - self.t = threading.Thread(target=enqueue_lines, - args=(self.f, self.q)) - self.t.daemon = True # thread dies with the program + self.t = threading.Thread(target=enqueue_lines, args=(self.f, self.q)) + self.t.daemon = True # thread dies with the program self.t.start() def readline(self): @@ -25,11 +26,9 @@ def readline(self): return line -if __name__ == '__main__': +if __name__ == "__main__": nb_reader = NonBlockingReader("/sys/kernel/debug/tracing/trace_pipe") while True: line = nb_reader.readline() - print line, - - + print(line, end=" ") diff --git a/wiscsee/workrunner/wlrunner.py b/wiscsee/workrunner/wlrunner.py index e36f141..196a52e 100644 --- a/wiscsee/workrunner/wlrunner.py +++ b/wiscsee/workrunner/wlrunner.py @@ -6,17 +6,17 @@ from pyreuse.sysutils import blocktrace from pyreuse.fsutils.ext4dumpextents import get_extents_of_dir import config -import cpuhandler -import filesystem -import fshelper +from . import cpuhandler +from . import filesystem +from . import fshelper from wiscsim import hostevent from utilities import utils -import workload +from . import workload from commons import * -class BarrierGen(object): +class BarrierGen: def __init__(self, n_ncq_slots): self.n_ncq_slots = n_ncq_slots @@ -28,146 +28,145 @@ def barrier_events(self): yield hostevent.ControlEvent(operation=OP_NOOP) -class WorkloadRunner(object): +class WorkloadRunner: def __init__(self, confobj): if not isinstance(confobj, config.Config): - raise TypeError("confobj is not of type class config.Config". - format(type(confobj).__name__())) + raise TypeError("confobj is not of type class config.Config") self.conf = confobj - if self.conf.device_type == 'loop': + if self.conf.device_type == "loop": # we don't pad loop - self.conf['dev_padding'] = 0 + self.conf["dev_padding"] = 0 # blktracer for making file system self.blktracer_prepfs = blocktrace.BlockTraceManager( - dev = self.conf['device_path'], - event_file_column_names = self.conf['event_file_column_names'], - resultpath = self.conf.get_blkparse_result_path_mkfs(), - to_ftlsim_path = self.conf.get_ftlsim_events_output_path_mkfs(), - sector_size = self.conf['sector_size'], - padding_bytes = self.conf['dev_padding'], - do_sort = self.conf['sort_block_trace'] - ) + dev=self.conf["device_path"], + event_file_column_names=self.conf["event_file_column_names"], + resultpath=self.conf.get_blkparse_result_path_mkfs(), + to_ftlsim_path=self.conf.get_ftlsim_events_output_path_mkfs(), + sector_size=self.conf["sector_size"], + padding_bytes=self.conf["dev_padding"], + do_sort=self.conf["sort_block_trace"], + ) # blktracer for running workload self.blktracer = blocktrace.BlockTraceManager( - dev = self.conf['device_path'], - event_file_column_names = self.conf['event_file_column_names'], - resultpath = self.conf.get_blkparse_result_path(), - to_ftlsim_path = self.conf.get_ftlsim_events_output_path(), - sector_size = self.conf['sector_size'], - padding_bytes = self.conf['dev_padding'], - do_sort = self.conf['sort_block_trace'] - ) - - self.aging_workload = eval("workload.{wlclass}(confobj = self.conf, " \ + dev=self.conf["device_path"], + event_file_column_names=self.conf["event_file_column_names"], + resultpath=self.conf.get_blkparse_result_path(), + to_ftlsim_path=self.conf.get_ftlsim_events_output_path(), + sector_size=self.conf["sector_size"], + padding_bytes=self.conf["dev_padding"], + do_sort=self.conf["sort_block_trace"], + ) + + self.aging_workload = eval( + "workload.{wlclass}(confobj = self.conf, " "workload_conf_key = '{wlconf_key}')".format( - wlclass = self.conf["age_workload_class"], - wlconf_key = self.conf["aging_config_key"] - ) #format - ) #eval + wlclass=self.conf["age_workload_class"], + wlconf_key=self.conf["aging_config_key"], + ) # format + ) # eval # create workload object - workload_str = "workload.{wlclass}(confobj = self.conf, " \ + workload_str = ( + "workload.{wlclass}(confobj = self.conf, " "workload_conf_key = '{wlconf_key}')".format( - wlclass = self.conf["workload_class"], - wlconf_key = self.conf["workload_conf_key"] - ) #format + wlclass=self.conf["workload_class"], + wlconf_key=self.conf["workload_conf_key"], + ) + ) # format self.workload = eval(workload_str) def prepare_device(self): - if self.conf.device_type == 'loop': + if self.conf.device_type == "loop": self.loopdev = filesystem.LoopDevice( - dev_path = self.conf['device_path'], - tmpfs_mount_point = self.conf['tmpfs_mount_point'], - size_mb = self.conf['dev_size_mb']) + dev_path=self.conf["device_path"], + tmpfs_mount_point=self.conf["tmpfs_mount_point"], + size_mb=self.conf["dev_size_mb"], + ) self.loopdev.create() - self.conf['dev_padding'] = 0 + self.conf["dev_padding"] = 0 - elif self.conf.device_type == 'real': + elif self.conf.device_type == "real": # umount file system if it is mounted - if fshelper.isMounted(self.conf['fs_mount_point']): - utils.shcmd( - "sudo umount {}".format(self.conf['fs_mount_point'])) + if fshelper.isMounted(self.conf["fs_mount_point"]): + utils.shcmd("sudo umount {}".format(self.conf["fs_mount_point"])) # partition the dev - base_dev_path = self.conf['device_path'].rstrip('0123456789') + base_dev_path = self.conf["device_path"].rstrip("0123456789") - mo = re.search(r'\d+$', self.conf['device_path']) + mo = re.search(r"\d+$", self.conf["device_path"]) if mo == None: - raise RuntimeError("You have to specify a partition, not " - "an entire disk: {}".format( - self.conf['device_path'])) + raise RuntimeError( + "You have to specify a partition, not an entire disk: {}".format( + self.conf["device_path"] + ) + ) dev_id = int(mo.group()) if dev_id == 0: raise RuntimeError( - "Disk partition ID (a part of device_path) starts with 1. " - "Example: /dev/sdc1.") + "Disk partition ID (a part of device_path) starts with 1. " + "Example: /dev/sdc1." + ) if dev_id > 4: raise RuntimeError( - "Only 4 partitions allowed. We will format the whole disk. " - "You probably should just specify the first " - "partition of your disk in device_path." - ) + "Only 4 partitions allowed. We will format the whole disk. " + "You probably should just specify the first " + "partition of your disk in device_path." + ) # if dev_id = 3, we will have: # [0 0 0] # sdc 1 2 3 part_sizes = [0 for i in range(dev_id)] - size = self.conf['dev_size_mb'] * 2**20 + size = self.conf["dev_size_mb"] * 2**20 part_sizes[dev_id - 1] = size - fshelper.partition_disk(base_dev_path, part_sizes, - self.conf['dev_padding'] - ) + fshelper.partition_disk(base_dev_path, part_sizes, self.conf["dev_padding"]) def build_fs(self): # create file system object, it is not physically created - fs = self.conf['filesystem'] + fs = self.conf["filesystem"] if fs == None: return - if fs == 'ext4': + if fs == "ext4": fsclass = filesystem.Ext4 - elif fs == 'f2fs': + elif fs == "f2fs": fsclass = filesystem.F2fs - elif fs == 'btrfs': + elif fs == "btrfs": fsclass = filesystem.Btrfs - elif fs == 'xfs': + elif fs == "xfs": fsclass = filesystem.Xfs else: - raise RuntimeError("{} is not a valid file system type"\ - .format(fs)) - fs_obj = fsclass(device = self.conf['device_path'], - mount_point = self.conf['fs_mount_point']) + raise RuntimeError(f"{fs} is not a valid file system type") + fs_obj = fsclass( + device=self.conf["device_path"], mount_point=self.conf["fs_mount_point"] + ) # Making and mounting file system try: - mk_opt_dic = self.conf[self.conf['filesystem']].get('make_opts', None) + mk_opt_dic = self.conf[self.conf["filesystem"]].get("make_opts", None) except KeyError: mk_opt_dic = None - fs_obj.make(opt_dic = mk_opt_dic) - fs_obj.mount(opt_list = - self.conf['mnt_opts'][ self.conf['filesystem'] ]) - utils.shcmd('sync') + fs_obj.make(opt_dic=mk_opt_dic) + fs_obj.mount(opt_list=self.conf["mnt_opts"][self.conf["filesystem"]]) + utils.shcmd("sync") # F2FS specific - if self.conf['filesystem'] == 'f2fs': - for opt_name, value in \ - self.conf['f2fs'].get('sysfs', {}).items(): + if self.conf["filesystem"] == "f2fs": + for opt_name, value in list(self.conf["f2fs"].get("sysfs", {}).items()): fs_obj.sysfs_setup(opt_name, value) def __set_linux_ncq_depth(self): - if self.conf.device_type == 'real': + if self.conf.device_type == "real": device_name = self.conf.get_device_name_no_num() - utils.set_linux_ncq_depth(device_name, - self.conf['linux_ncq_depth']) + utils.set_linux_ncq_depth(device_name, self.conf["linux_ncq_depth"]) def __set_linux_io_scheduler(self): - if self.conf.device_type == 'real': + if self.conf.device_type == "real": device_name = self.conf.get_device_name_no_num() - utils.set_linux_io_scheduler(device_name, - self.conf['linux_io_scheduler']) + utils.set_linux_io_scheduler(device_name, self.conf["linux_io_scheduler"]) def __set_linux_environment(self): self.__set_linux_ncq_depth() @@ -176,20 +175,20 @@ def __set_linux_environment(self): def run(self): self.__set_linux_environment() - if self.conf['enable_blktrace'] == True: + if self.conf["enable_blktrace"] == True: return self.run_with_blktrace() else: return self.run_without_blktrace() def run_without_blktrace(self): - cpuhandler.set_cpus(self.conf['n_online_cpus']) + cpuhandler.set_cpus(self.conf["n_online_cpus"]) self.prepare_device() self.build_fs() - print '----------------------------------------------------' - print '---------Running Aging Workload_-------------------' - print '----------------------------------------------------' + print("----------------------------------------------------") + print("---------Running Aging Workload_-------------------") + print("----------------------------------------------------") self.aging_workload.run() utils.drop_caches() @@ -201,7 +200,7 @@ def run_without_blktrace(self): end_time = datetime.datetime.now() app_duration = end_time - start_time - print 'Application duration >>>>>>>>>', app_duration.total_seconds() + print(("Application duration >>>>>>>>>", app_duration.total_seconds())) self.write_app_duration(app_duration.total_seconds()) self._post_target_workload() @@ -211,30 +210,32 @@ def run_without_blktrace(self): def run_with_blktrace(self): try: # Set number of CPUs - cpuhandler.set_cpus(self.conf['n_online_cpus']) + cpuhandler.set_cpus(self.conf["n_online_cpus"]) self.prepare_device() # strat blktrace # This is only for making and mounting file system, because we # want to separate them with workloads. - if self.conf['trace_issue_and_complete'] is True: - trace_filter=['issue', 'complete'] + if self.conf["trace_issue_and_complete"] is True: + trace_filter = ["issue", "complete"] else: - trace_filter=['issue'] + trace_filter = ["issue"] - self.blktracer_prepfs.start_tracing_and_collecting(trace_filter=trace_filter) + self.blktracer_prepfs.start_tracing_and_collecting( + trace_filter=trace_filter + ) time.sleep(1) while self.blktracer_prepfs.proc == None: - print 'Waiting for blktrace to start.....' + print("Waiting for blktrace to start.....") time.sleep(0.5) self.build_fs() # Age the file system - print '----------------------------------------------------' - print '---------Running Aging Workload-------------------' - print '----------------------------------------------------' + print("----------------------------------------------------") + print("---------Running Aging Workload-------------------") + print("----------------------------------------------------") self.aging_workload.run() utils.drop_caches() @@ -247,25 +248,25 @@ def run_with_blktrace(self): time.sleep(2) while self.blktracer.proc == None: - print 'Waiting for blktrace to start.....' + print("Waiting for blktrace to start.....") time.sleep(0.5) - print 'Running workload ..................' + print("Running workload ..................") self._pre_target_workload() - print '----------------------------------------------------' - print '---------Running TARGET workload-------------------' - print '----------------------------------------------------' + print("----------------------------------------------------") + print("---------Running TARGET workload-------------------") + print("----------------------------------------------------") start_time = datetime.datetime.now() self.workload.run() end_time = datetime.datetime.now() app_duration = end_time - start_time - print 'Application duration >>>>>>>>>', app_duration.total_seconds() + print(("Application duration >>>>>>>>>", app_duration.total_seconds())) self.write_app_duration(app_duration.total_seconds()) self._post_target_workload() - time.sleep(1) # has to sleep here so the blktrace gets all the data + time.sleep(1) # has to sleep here so the blktrace gets all the data except Exception: raise @@ -281,8 +282,8 @@ def run_with_blktrace(self): self.blktracer.stop_tracing_and_collecting() def write_app_duration(self, secs): - path = os.path.join(self.conf['result_dir'], 'app_duration.txt') - with open(path, 'w') as f: + path = os.path.join(self.conf["result_dir"], "app_duration.txt") + with open(path, "w") as f: f.write(str(secs)) def remove_raw_trace(self): @@ -293,32 +294,38 @@ def _pre_target_workload(self): pass def _post_target_workload(self): - if self.conf.get('do_fstrim', False) is True: - cmd = "fstrim {}".format(self.conf['fs_mount_point']) + if self.conf.get("do_fstrim", False) is True: + cmd = "fstrim {}".format(self.conf["fs_mount_point"]) utils.shcmd(cmd) - if self.conf['filesystem'] == 'f2fs' and self.conf['f2fs_gc_after_workload'] is True: + if ( + self.conf["filesystem"] == "f2fs" + and self.conf["f2fs_gc_after_workload"] is True + ): time.sleep(1) utils.drop_caches() - utils.invoke_f2fs_gc(self.conf['fs_mount_point'], 1, -1) + utils.invoke_f2fs_gc(self.conf["fs_mount_point"], 1, -1) - if self.conf['filesystem'] == 'ext4' and self.conf['dump_ext4_after_workload'] is True: - utils.shcmd('sync') + if ( + self.conf["filesystem"] == "ext4" + and self.conf["dump_ext4_after_workload"] is True + ): + utils.shcmd("sync") self.dumpe2fs() # self.dump_extents() def dumpe2fs(self): - dumppath = os.path.join(self.conf['result_dir'], 'dumpe2fs.out') - utils.shcmd("dumpe2fs {} > {}".format( - self.conf['device_path'], dumppath)) + dumppath = os.path.join(self.conf["result_dir"], "dumpe2fs.out") + utils.shcmd("dumpe2fs {} > {}".format(self.conf["device_path"], dumppath)) def dump_extents(self): - dumppath = os.path.join(self.conf['result_dir'], 'extents.json') - extent_path = dumppath + '.table' + dumppath = os.path.join(self.conf["result_dir"], "extents.json") + extent_path = dumppath + ".table" - extents_list = get_extents_of_dir(dirpath=self.conf['fs_mount_point'], - dev_path=self.conf['device_path']) - d = {'extents': extents_list} + extents_list = get_extents_of_dir( + dirpath=self.conf["fs_mount_point"], dev_path=self.conf["device_path"] + ) + d = {"extents": extents_list} utils.dump_json(d, dumppath) utils.table_to_file(extents_list, extent_path, width=0) @@ -340,17 +347,16 @@ def get_event_iterator(self): for event in self.gc_event(): yield event - for req in barriergen.barrier_events(): - yield req + yield from barriergen.barrier_events() yield hostevent.ControlEvent(operation=OP_REC_BW) def prepfs_events(self): prepfs_line_iter = hostevent.FileLineIterator( - self.conf.get_ftlsim_events_output_path_mkfs()) + self.conf.get_ftlsim_events_output_path_mkfs() + ) event_prepfs_iter = hostevent.EventIterator(self.conf, prepfs_line_iter) - for event in event_prepfs_iter: - yield event + yield from event_prepfs_iter def target_workload_events(self): # special event indicates the start of workload @@ -358,36 +364,29 @@ def target_workload_events(self): yield hostevent.ControlEvent(operation=OP_ENABLE_RECORDER) for req in barriergen.barrier_events(): yield req - yield hostevent.ControlEvent(operation=OP_REC_TIMESTAMP, - arg1='interest_workload_start') + yield hostevent.ControlEvent( + operation=OP_REC_TIMESTAMP, arg1="interest_workload_start" + ) workload_line_iter = hostevent.FileLineIterator( - self.conf.get_ftlsim_events_output_path()) + self.conf.get_ftlsim_events_output_path() + ) event_workload_iter = hostevent.EventIterator(self.conf, workload_line_iter) - for event in event_workload_iter: - yield event + yield from event_workload_iter for req in barriergen.barrier_events(): yield req - yield hostevent.ControlEvent(operation=OP_REC_TIMESTAMP, - arg1='interest_workload_end') + yield hostevent.ControlEvent( + operation=OP_REC_TIMESTAMP, arg1="interest_workload_end" + ) def gc_event(self): barriergen = BarrierGen(self.conf.ssd_ncq_depth()) - if self.conf['do_gc_after_workload'] is True: - for req in barriergen.barrier_events(): - yield req - yield hostevent.ControlEvent(operation=OP_REC_TIMESTAMP, - arg1='gc_start_timestamp') + if self.conf["do_gc_after_workload"] is True: + yield from barriergen.barrier_events() + yield hostevent.ControlEvent( + operation=OP_REC_TIMESTAMP, arg1="gc_start_timestamp" + ) yield hostevent.ControlEvent(operation=OP_CLEAN) - - - - - - - - - diff --git a/wiscsee/workrunner/workload.py b/wiscsee/workrunner/workload.py index 942f8e7..61b7023 100644 --- a/wiscsee/workrunner/workload.py +++ b/wiscsee/workrunner/workload.py @@ -1,3 +1,4 @@ +from past.utils import old_div import collections import os import pprint @@ -8,14 +9,15 @@ from commons import * import config -import multiwriters +from . import multiwriters from utilities import utils from pyreuse.helpers import * from pyreuse.apputils.parseleveldboutput import parse_file -class Workload(object): - def __init__(self, confobj, workload_conf_key = None): + +class Workload: + def __init__(self, confobj, workload_conf_key=None): """ workload_conf is part of confobj. But we may need to run multiple workloads with different configurations in our @@ -27,11 +29,10 @@ def __init__(self, confobj, workload_conf_key = None): one Workload should use this new __init__() with two parameters. """ if not isinstance(confobj, config.Config): - raise TypeError("confobj is not of type class config.Config". - format(type(confobj).__name__)) + raise TypeError("confobj is not of type class config.Config") self.conf = confobj - if workload_conf_key != None and workload_conf_key != 'None': + if workload_conf_key != None and workload_conf_key != "None": self.workload_conf = confobj[workload_conf_key] def run(self): @@ -40,12 +41,14 @@ def run(self): def stop(self): raise NotImplementedError + class NoOp(Workload): """ This is a workload class that does nothing. It may be used to skip the file system aging stage. To skip aging workload, set conf['age_workload_class'] = "NoOp" """ + def run(self): pass @@ -54,16 +57,16 @@ def stop(self): class SimpleRandReadWrite(Workload): - def __init__(self, confobj, workload_conf_key = None): - super(SimpleRandReadWrite, self).__init__(confobj, workload_conf_key) + def __init__(self, confobj, workload_conf_key=None): + super().__init__(confobj, workload_conf_key) def run(self): mnt = self.conf["fs_mount_point"] datafile = os.path.join(mnt, "datafile") region = 2 * MB chunksize = 64 * KB - n_chunks = region / chunksize - chunkids = range(n_chunks) + n_chunks = old_div(region, chunksize) + chunkids = list(range(n_chunks)) buf = "a" * chunksize f = open(datafile, "w+") @@ -88,17 +91,15 @@ def stop(self): # class LinuxDD(Workload): - # def __init__(self, confobj, workload_conf_key = None): - # super(LinuxDD, self).__init__(confobj, workload_conf_key) - - # def run(self): - # mnt = self.conf["fs_mount_point"] - # cmd = "dd if=/dev/zero of={}/datafile bs=64k count=128".format(mnt) - # print cmd - # subprocess.call(cmd, shell=True) - # subprocess.call("sync") - - # def stop(self): - # pass - - +# def __init__(self, confobj, workload_conf_key = None): +# super(LinuxDD, self).__init__(confobj, workload_conf_key) + +# def run(self): +# mnt = self.conf["fs_mount_point"] +# cmd = "dd if=/dev/zero of={}/datafile bs=64k count=128".format(mnt) +# print cmd +# subprocess.call(cmd, shell=True) +# subprocess.call("sync") + +# def stop(self): +# pass From a2d7f316f18a66764481b553511dd79a58b0edcf Mon Sep 17 00:00:00 2001 From: "js0.site" Date: Sat, 24 Jan 2026 06:27:28 +0800 Subject: [PATCH 2/9] =?UTF-8?q?=F0=9F=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bench_selected.py | 96 ++++++++---- wiscsee/leaftl_scripts/run_ftl | 233 ++++++++++++++++++++++------- wiscsee/wiscsim/learnedftl.py | 14 +- wiscsee/wiscsim/workload_parser.py | 34 +++-- 4 files changed, 270 insertions(+), 107 deletions(-) diff --git a/bench_selected.py b/bench_selected.py index bfb0b16..6029797 100755 --- a/bench_selected.py +++ b/bench_selected.py @@ -6,13 +6,14 @@ import glob import json + def run_and_stream(cmd, env): """Runs a command and streams output to stdout while capturing for parsing.""" full_output = [] # Make sure we use the same python interpreter for child processes if cmd[0] == sys.executable: - pass # already set - + pass # already set + try: process = subprocess.Popen( cmd, @@ -20,19 +21,20 @@ def run_and_stream(cmd, env): stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, - bufsize=1 + bufsize=1, ) - + for line in process.stdout: - print(line, end='', flush=True) + print(line, end="", flush=True) full_output.append(line) - + process.wait() return "".join(full_output) except Exception as e: print(f"\nError running command: {e}") return None + def main(): # Set up directory script_dir = os.path.dirname(os.path.abspath(__file__)) @@ -52,68 +54,98 @@ def main(): # Traces directory data_dir = os.path.join(script_dir, "data") run_ftl_script = os.path.join("wiscsee", "leaftl_scripts", "run_ftl") - + # Scan for all .bin files generated by init.js trace_files = sorted(glob.glob(os.path.join(data_dir, "*.bin"))) - + if not trace_files: print(f"No .bin files found in {data_dir}") return - + print("\nStarting Benchmarks...\n") print("Trace, DFTL (MB), SFTL (MB), LeaFTL (MB)") - + results = [] - + for trace_path in trace_files: trace_name = os.path.basename(trace_path) print(f"\n>>> Processing {trace_name}") - + # 1. Run LearnedFTL (provides LeaFTL and DFTL sizes) - cmd_lea = [sys.executable, run_ftl_script, "-f", "learnedftl", "-t", trace_path, "-sl", "0"] + cmd_lea = [ + sys.executable, + run_ftl_script, + "-f", + "learnedftl", + "-t", + trace_path, + "-sl", + "0", + ] output_lea = run_and_stream(cmd_lea, env) - + leaftl_mb = 0.0 dftl_mb = 0.0 sftl_mb = 0.0 - + if output_lea: - lea_match = re.search(r"estimated learnedftl memory footprint: (\d+) B", output_lea) + # Match both normal and partial stats on crash + lea_match = re.search( + r"estimated learnedftl memory footprint: (\d+) B", output_lea + ) if lea_match: leaftl_mb = float(lea_match.group(1)) / (1024 * 1024) - - dftl_match = re.search(r"estimated dftl memory footprint: (\d+) B", output_lea) + + dftl_match = re.search( + r"estimated dftl memory footprint: (\d+) B", output_lea + ) if dftl_match: dftl_mb = float(dftl_match.group(1)) / (1024 * 1024) # 2. Run SFTL (provides SFTL size) - cmd_sftl = [sys.executable, run_ftl_script, "-f", "sftl", "-t", trace_path, "-sl", "0"] + cmd_sftl = [ + sys.executable, + run_ftl_script, + "-f", + "sftl", + "-t", + trace_path, + "-sl", + "0", + ] output_sftl = run_and_stream(cmd_sftl, env) - + if output_sftl: - sftl_match = re.search(r"estimated learnedftl memory footprint: (\d+) B", output_sftl) + sftl_match = re.search( + r"estimated learnedftl memory footprint: (\d+) B", output_sftl + ) if sftl_match: sftl_mb = float(sftl_match.group(1)) / (1024 * 1024) - - summary_line = f"SUMMARY: {trace_name}, {dftl_mb:.4f}, {sftl_mb:.4f}, {leaftl_mb:.4f}" + + summary_line = ( + f"SUMMARY: {trace_name}, {dftl_mb:.4f}, {sftl_mb:.4f}, {leaftl_mb:.4f}" + ) print(summary_line) - - results.append({ - "trace": trace_name, - "dftl_mb": round(dftl_mb, 4), - "sftl_mb": round(sftl_mb, 4), - "leaftl_mb": round(leaftl_mb, 4) - }) - + + results.append( + { + "trace": trace_name, + "dftl_mb": round(dftl_mb, 4), + "sftl_mb": round(sftl_mb, 4), + "leaftl_mb": round(leaftl_mb, 4), + } + ) + sys.stdout.flush() # Save to JSON json_path = "bench_results.json" with open(json_path, "w") as f: json.dump(results, f, indent=4) - + print(f"\nFinal results saved to {json_path}") + if __name__ == "__main__": main() diff --git a/wiscsee/leaftl_scripts/run_ftl b/wiscsee/leaftl_scripts/run_ftl index 6be49ef..aa0eebd 100755 --- a/wiscsee/leaftl_scripts/run_ftl +++ b/wiscsee/leaftl_scripts/run_ftl @@ -3,13 +3,14 @@ import unittest import collections import shutil import os -#import matplotlib + +# import matplotlib import argparse # from scipy import percentile -#matplotlib.use('Agg') -#import matplotlib.pyplot as plt -#import statsmodels.distributions +# matplotlib.use('Agg') +# import matplotlib.pyplot as plt +# import statsmodels.distributions import config from workflow import * @@ -28,115 +29,237 @@ KB = 1024 MB = 1024**2 GB = 1024**3 + def split_lpns(offset, size, page_size): page_size = float(page_size) - lpns = [lpn for lpn in range(int(math.floor(offset/page_size)), int(math.ceil((offset+size)/page_size)))] + lpns = [ + lpn + for lpn in range( + int(math.floor(offset / page_size)), + int(math.ceil((offset + size) / page_size)), + ) + ] return lpns + def create_config(ftl_type="dftldes"): if ftl_type == "dftldes" or ftl_type == "learnedftl" or ftl_type == "sftl": conf = wiscsim.dftldes.Config() - conf['ftl_type'] = "learnedftl" - conf['internal_ftl_type'] = ftl_type + conf["ftl_type"] = "learnedftl" + conf["internal_ftl_type"] = ftl_type else: raise NotImplementedError # ssd config - conf['flash_config']['n_pages_per_block'] = 256 - conf['flash_config']['n_blocks_per_plane'] = 2048 - conf['flash_config']['n_planes_per_chip'] = 64 - conf['flash_config']['n_chips_per_package'] = 1 - conf['flash_config']['n_packages_per_channel'] = 1 - conf['flash_config']['n_channels_per_dev'] = 8 + conf["flash_config"]["n_pages_per_block"] = 256 + conf["flash_config"]["n_blocks_per_plane"] = 2048 + conf["flash_config"]["n_planes_per_chip"] = 64 + conf["flash_config"]["n_chips_per_package"] = 1 + conf["flash_config"]["n_packages_per_channel"] = 1 + conf["flash_config"]["n_channels_per_dev"] = 8 # set ftl - conf['do_not_check_gc_setting'] = True + conf["do_not_check_gc_setting"] = True conf.GC_high_threshold_ratio = 0.96 conf.GC_low_threshold_ratio = 0.5 - conf['enable_simulation'] = True + conf["enable_simulation"] = True - utils.set_exp_metadata(conf, save_data = False, - expname = 'run_expname', - subexpname = 'run_subexpname') + utils.set_exp_metadata( + conf, save_data=False, expname="run_expname", subexpname="run_subexpname" + ) - conf['simulator_class'] = 'SimulatorDESNew' + conf["simulator_class"] = "SimulatorDESNew" utils.runtime_update(conf) return conf -class RunFTL(): - def __init__(self, ftl, trace, start_lineno, lineno, output, write_only, ncq_depth, page_size, *args, **xargs): - assert(ftl in ["learnedftl", "hybridftl", "dftldes", "sftl"]) +class RunFTL: + def __init__( + self, + ftl, + trace, + start_lineno, + lineno, + output, + write_only, + ncq_depth, + page_size, + *args, + **xargs, + ): + assert ftl in ["learnedftl", "hybridftl", "dftldes", "sftl"] self.trace = trace self.start_lineno = start_lineno self.lineno = lineno self.write_only = bool(int(write_only)) self.conf = create_config(ftl) - self.conf['SSDFramework']['ncq_depth'] = ncq_depth - self.conf['flash_config']['page_size'] = page_size + self.conf["SSDFramework"]["ncq_depth"] = ncq_depth + self.conf["flash_config"]["page_size"] = page_size self.func = "self.run_%s()" % "learnedftl" - + if output: recorder = True - xargs['result_dir'] = output + xargs["result_dir"] = output else: recorder = False - self.events = parse_events(self.trace, self.conf.page_size, recorder=recorder, start_lineno = start_lineno, lineno=self.lineno, write_only=self.write_only) + self.events = parse_events( + self.trace, + self.conf.page_size, + recorder=recorder, + start_lineno=start_lineno, + lineno=self.lineno, + write_only=self.write_only, + ) self.conf.update(xargs) def run(self): eval(self.func) - + def statistics(): pass def run_learnedftl(self): - log_msg("ftl = %s, gamma = %.4f, cache_size = %.2f MB, lineno = %d" % (self.conf['internal_ftl_type'], self.conf['gamma'], self.conf['cache_size'] / MB, self.lineno)) + log_msg( + "ftl = %s, gamma = %.4f, cache_size = %.2f MB, lineno = %d" + % ( + self.conf["internal_ftl_type"], + self.conf["gamma"], + self.conf["cache_size"] / MB, + self.lineno, + ) + ) wf = Workflow(self.conf) - sim = wf.run_simulator(self.events) - mapping_table = sim.ssd.ftl.metadata.mapping_table - reference_mapping_table = sim.ssd.ftl.metadata.reference_mapping_table + self.sim = wf.run_simulator(self.events) + mapping_table = self.sim.ssd.ftl.metadata.mapping_table + reference_mapping_table = self.sim.ssd.ftl.metadata.reference_mapping_table # print(mapping_table.runs[-5:]) # print(mapping_table.lookup_range(797846, 797887)) # mapping_table.compact(promote=True) - - log_msg(sim.ssd.ftl.hist) + log_msg(self.sim.ssd.ftl.hist) log_msg("max # of levels: %d" % mapping_table.levels) dist = mapping_table.dist_levels if len(dist) > 0: - avg, std, percentile = np.average(dist), np.std(dist), np.percentile(dist, 99) - log_msg("average # of levels: %.2f, variation: %.2f, 99: %.2f" % (avg, std, percentile)) + avg, std, percentile = ( + np.average(dist), + np.std(dist), + np.percentile(dist, 99), + ) + log_msg( + "average # of levels: %.2f, variation: %.2f, 99: %.2f" + % (avg, std, percentile) + ) log_msg("# of segments: %d" % len(mapping_table.segments)) - log_msg("# of consecutive segments: %d" % len(list(filter(lambda seg : seg.consecutive, mapping_table.segments)))) + log_msg( + "# of consecutive segments: %d" + % len(list(filter(lambda seg: seg.consecutive, mapping_table.segments))) + ) log_msg("estimated learnedftl memory footprint: %d B" % mapping_table.memory) - log_msg("estimated dftl memory footprint: %d B" % (len(reference_mapping_table.mapping_table)*8)) + log_msg( + "estimated dftl memory footprint: %d B" + % (len(reference_mapping_table.mapping_table) * 8) + ) + + def print_current_stats(self): + try: + mapping_table = self.sim.ssd.ftl.metadata.mapping_table + reference_mapping_table = self.sim.ssd.ftl.metadata.reference_mapping_table + print("\n[PARTIAL STATS ON CRASH]") + print("estimated learnedftl memory footprint: %d B" % mapping_table.memory) + print( + "estimated dftl memory footprint: %d B" + % (len(reference_mapping_table.mapping_table) * 8) + ) + except: + pass -if __name__ == '__main__': + +if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument('-f', '--ftl', dest="ftl", required=True, help="FTL to run") - parser.add_argument('-t', '--trace', dest="trace", help="Trace path") - parser.add_argument('-l', '--lineno', dest="lineno", default=10**9, help="Line number", type=int) - parser.add_argument('-sl', '--start-line', dest="start_lineno", default=0, help="Start line number", type=int) - parser.add_argument('-c', '--cache', dest="cache_size", default=0, help="Cache Size in MB", type=float) - parser.add_argument('-mc', '--mapping-cache', dest="mapping_cache_bytes", default=10, help="Mapping table cache size in MB", type=float) - parser.add_argument('-g', '--gamma', dest="gamma", default=1e-4, help="Error bound of learnedftl", type=float) - parser.add_argument('-d', '--dry', dest="dry_run", action='store_true', default=False, help="Perform dry-run") - parser.add_argument('-o', '--output', dest="output", default='', help="Output directory") - parser.add_argument('-wo', '--write-only', dest="write_only", default=0, help="Write events only") - parser.add_argument('-q', '--ncq-depth', dest="ncq_depth", default=1, help="NCQ Depth", type=int) - parser.add_argument('-p', '--page-size', dest="page_size", default=4096, help="Page size", type=int) + parser.add_argument("-f", "--ftl", dest="ftl", required=True, help="FTL to run") + parser.add_argument("-t", "--trace", dest="trace", help="Trace path") + parser.add_argument( + "-l", "--lineno", dest="lineno", default=10**9, help="Line number", type=int + ) + parser.add_argument( + "-sl", + "--start-line", + dest="start_lineno", + default=0, + help="Start line number", + type=int, + ) + parser.add_argument( + "-c", + "--cache", + dest="cache_size", + default=0, + help="Cache Size in MB", + type=float, + ) + parser.add_argument( + "-mc", + "--mapping-cache", + dest="mapping_cache_bytes", + default=10, + help="Mapping table cache size in MB", + type=float, + ) + parser.add_argument( + "-g", + "--gamma", + dest="gamma", + default=1e-4, + help="Error bound of learnedftl", + type=float, + ) + parser.add_argument( + "-d", + "--dry", + dest="dry_run", + action="store_true", + default=False, + help="Perform dry-run", + ) + parser.add_argument( + "-o", "--output", dest="output", default="", help="Output directory" + ) + parser.add_argument( + "-wo", "--write-only", dest="write_only", default=0, help="Write events only" + ) + parser.add_argument( + "-q", "--ncq-depth", dest="ncq_depth", default=1, help="NCQ Depth", type=int + ) + parser.add_argument( + "-p", "--page-size", dest="page_size", default=4096, help="Page size", type=int + ) args = parser.parse_args() - experiment = RunFTL(args.ftl, args.trace, args.start_lineno, args.lineno, args.output, args.write_only, args.ncq_depth, args.page_size, gamma=args.gamma, cache_size=int(args.cache_size*MB), dry_run=args.dry_run, mapping_cache_bytes=int(args.mapping_cache_bytes*MB)) - experiment.run() - - + experiment = RunFTL( + args.ftl, + args.trace, + args.start_lineno, + args.lineno, + args.output, + args.write_only, + args.ncq_depth, + args.page_size, + gamma=args.gamma, + cache_size=int(args.cache_size * MB), + dry_run=args.dry_run, + mapping_cache_bytes=int(args.mapping_cache_bytes * MB), + ) + try: + experiment.run() + except IndexError as e: + if "pop from empty list" in str(e): + experiment.print_current_stats() + raise e diff --git a/wiscsee/wiscsim/learnedftl.py b/wiscsee/wiscsim/learnedftl.py index 40b5b8f..f42d51a 100644 --- a/wiscsee/wiscsim/learnedftl.py +++ b/wiscsee/wiscsim/learnedftl.py @@ -793,11 +793,13 @@ def __init__(self, conf): self.timestamp_table = {} self.cur_timestamp = 0 self.counter = defaultdict(lambda: 0) - self.free_block_list = [ - self.conf.n_blocks_per_channel * channel + block - for block in range(self.conf.n_blocks_per_channel) - for channel in range(self.conf.n_channels_per_dev) - ] + self.free_block_list = deque( + [ + self.conf.n_blocks_per_channel * channel + block + for block in range(self.conf.n_blocks_per_channel) + for channel in range(self.conf.n_channels_per_dev) + ] + ) def _incr_timestamp(self): """ @@ -819,7 +821,7 @@ def get_num_valid_pages(self, blocknum): # FIXME improve efficiency; now round-robin # bottleneck def next_free_block(self, wear_level=False): - free_block = self.free_block_list.pop(0) + free_block = self.free_block_list.popleft() return free_block # for block in range(self.conf.n_blocks_per_channel): # for channel in range(self.conf.n_channels_per_dev): diff --git a/wiscsee/wiscsim/workload_parser.py b/wiscsee/wiscsim/workload_parser.py index 1caacef..e2113c1 100644 --- a/wiscsee/wiscsim/workload_parser.py +++ b/wiscsee/wiscsim/workload_parser.py @@ -76,40 +76,44 @@ def parse_binary_events(filename, page_size, recorder=True): SECTOR_SIZE = 512 # page_size passed in, typically 4096 - with open(filename, 'rb') as f: + with open(filename, "rb") as f: timestamp = 0 while True: # Read OP (1 byte) op_bytes = f.read(1) if not op_bytes: break - op = struct.unpack('B', op_bytes)[0] + op = struct.unpack("B", op_bytes)[0] # Read LBA (4 bytes) lba_bytes = f.read(4) if not lba_bytes: break - lba = struct.unpack(' Date: Sat, 24 Jan 2026 07:26:09 +0800 Subject: [PATCH 3/9] =?UTF-8?q?=F0=9F=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- wiscsee/leaftl_scripts/run_ftl | 43 +++++++++++++ wiscsee/wiscsim/controller.py | 4 ++ wiscsee/wiscsim/learnedftl.py | 112 +++++++++++++++++++++++---------- 3 files changed, 125 insertions(+), 34 deletions(-) diff --git a/wiscsee/leaftl_scripts/run_ftl b/wiscsee/leaftl_scripts/run_ftl index aa0eebd..d0dbe7f 100755 --- a/wiscsee/leaftl_scripts/run_ftl +++ b/wiscsee/leaftl_scripts/run_ftl @@ -3,6 +3,7 @@ import unittest import collections import shutil import os +import json # import matplotlib import argparse @@ -101,6 +102,48 @@ class RunFTL: self.conf["SSDFramework"]["ncq_depth"] = ncq_depth self.conf["flash_config"]["page_size"] = page_size + # Set capacity based on trace's JSON file if it exists + if self.trace: + json_path = self.trace.rsplit(".", 1)[0] + ".json" + if os.path.exists(json_path): + try: + with open(json_path, "r") as f: + trace_metadata = json.load(f) + if "max_lba" in trace_metadata: + max_lba = int(trace_metadata["max_lba"]) + # LearnedFTL traces usually have LBA as page indexes + # We add some over-provisioning (e.g. 25%) + total_pages = int((max_lba + 1) * 1.25) + n_pages_per_block = self.conf["flash_config"]["n_pages_per_block"] + total_blocks = ( + total_pages + n_pages_per_block - 1 + ) // n_pages_per_block + + # Total blocks = n_blocks_per_plane * channels * planes * chips * packages + layout_factor = ( + self.conf["flash_config"]["n_planes_per_chip"] + * self.conf["flash_config"]["n_chips_per_package"] + * self.conf["flash_config"]["n_packages_per_channel"] + * self.conf["flash_config"]["n_channels_per_dev"] + ) + n_blocks_per_plane = ( + total_blocks + layout_factor - 1 + ) // layout_factor + + # Minimum blocks per plane safety + n_blocks_per_plane = max(n_blocks_per_plane, 2) + + self.conf["flash_config"]["n_blocks_per_plane"] = n_blocks_per_plane + log_msg( + "Set capacity based on {}: max_lba={}, total_blocks={}".format( + json_path, + max_lba, + n_blocks_per_plane * layout_factor, + ) + ) + except Exception as e: + log_msg("Failed to read trace metadata from {}: {}".format(json_path, e)) + self.func = "self.run_%s()" % "learnedftl" if output: diff --git a/wiscsee/wiscsim/controller.py b/wiscsee/wiscsim/controller.py index 7d89bae..b99e689 100644 --- a/wiscsee/wiscsim/controller.py +++ b/wiscsee/wiscsim/controller.py @@ -192,6 +192,10 @@ def get_flash_requests_for_ppns(self, page_start, page_count, op): def physical_to_machine_page(self, page_no): addr = FlashAddress() + # Support logically large address space (infinite capacity mode) + # by wrapping around physical resources. + page_no = page_no % self.n_pages_per_dev + # page_hierarchy has [channel, package, ..., block] # location has [channel, package, ..., block, page] for i, count in enumerate(self.page_hierarchy): diff --git a/wiscsee/wiscsim/learnedftl.py b/wiscsee/wiscsim/learnedftl.py index f42d51a..7f80cce 100644 --- a/wiscsee/wiscsim/learnedftl.py +++ b/wiscsee/wiscsim/learnedftl.py @@ -613,7 +613,7 @@ def post_processing(self): def clean(self, forced=False, merge=True): self.pre_written_bytes_gc = self.written_bytes erased_pbns = [] - validate_pages = [] + valid_ppns = [] num_valid = sum( self.metadata.bvc.counter[block] for block in self.metadata.bvc.counter @@ -625,7 +625,6 @@ def clean(self, forced=False, merge=True): if self.metadata.bvc.counter[block] > 0 ) ) - # print(num_valid / num_all) if num_all == 0: return if old_div(num_valid, num_all) > 0.9: @@ -633,26 +632,48 @@ def clean(self, forced=False, merge=True): for block in self.metadata.bvc.counter: num_validate_pages = self.metadata.bvc.counter[block] - if num_validate_pages / 256.0 <= 0.1: + if num_validate_pages > 0 and num_validate_pages / float(self.conf.n_pages_per_block) <= 0.1: erased_pbns.append(block) - # print(block) - validate_pages += self.metadata.pvb.get_valid_pages(block) + valid_ppns += self.metadata.pvb.get_valid_pages(block) + + if not erased_pbns: + # Greedy GC: If no blocks under threshold, pick blocks with minimum valid pages + # especially if we are low on space or forced + if forced or len(self.metadata.bvc.free_block_list) < 4: + # Sort blocks by valid page count + victim_candidates = [] + for block in list(self.metadata.bvc.counter.keys()): + if self.metadata.bvc.counter[block] < self.conf.n_pages_per_block: + victim_candidates.append((self.metadata.bvc.counter[block], block)) + + if victim_candidates: + victim_candidates.sort() + # Pick top 4 best victims or fewer if not available + for i in range(min(4, len(victim_candidates))): + _, block = victim_candidates[i] + erased_pbns.append(block) + valid_ppns += self.metadata.pvb.get_valid_pages(block) + + if not erased_pbns: + return - # print(validate_pages) - count = 0 - all_ppns_to_write = [] - for lpn in validate_pages: - if count % 256 == 0: - next_free_block = self.metadata.bvc.next_free_block() - self.metadata.pvb.validate_block(next_free_block) - next_free_ppn = self.conf.n_pages_per_block * next_free_block - count = 1 - else: - next_free_ppn += 1 - count += 1 - all_ppns_to_write.append(next_free_ppn) + lpns_to_move = [] + for ppn in valid_ppns: + lpn = self.metadata.ppn_to_lpn_map.get(ppn) + if lpn is not None: + lpns_to_move.append(lpn) - # print(all_ppns_to_write) + read_procs = [] + write_procs = [] + if lpns_to_move: + mappings, pages_to_read_meta, pages_to_write = self.metadata.update(lpns_to_move) + + for ppn in valid_ppns: + read_procs.append(self.env.process(self._read_ppns([ppn]))) + for ppn in pages_to_read_meta: + read_procs.append(self.env.process(self._read_ppns([ppn]))) + for ppn in pages_to_write: + write_procs.append(self.env.process(self._write_ppns([ppn]))) erase_procs = [] for erased_pbn in erased_pbns: @@ -664,23 +685,23 @@ def clean(self, forced=False, merge=True): ) ] - write_procs = [] - for ppn in all_ppns_to_write: - p = self.env.process(self._write_ppns([ppn])) - write_procs.append(p) - start = self.env.now - yield simpy.AllOf(self.env, erase_procs) - erase_finished = self.env.now + yield simpy.AllOf(self.env, read_procs) yield simpy.AllOf(self.env, write_procs) - write_finished = self.env.now - print( - ( - len(validate_pages), - erase_finished - start, - write_finished - erase_finished, + yield simpy.AllOf(self.env, erase_procs) + + for erased_pbn in erased_pbns: + self.metadata.erase_block(erased_pbn) + + end = self.env.now + if erased_pbns: + print( + ( + f"GC finished: moved {len(lpns_to_move)} pages, " + f"erased {len(erased_pbns)} blocks, " + f"time: {end - start}" + ) ) - ) def is_wear_leveling_needed(self): factor, diff = self.block_pool.get_wear_status() @@ -693,6 +714,8 @@ def level_wear(self): pass def is_cleaning_needed(self): + if len(self.metadata.bvc.free_block_list) < 4: + return True if self.written_bytes > self.pre_written_bytes_gc + self.gc_interval: return True return False @@ -731,6 +754,13 @@ def __init__(self, conf, bvc): self.bitmap.setall(0) self.bvc = bvc + def expand(self, blocknum): + total_pages_needed = (blocknum + 1) * self.conf.n_pages_per_block + if len(self.bitmap) < total_pages_needed: + extension = bitarray.bitarray(total_pages_needed - len(self.bitmap)) + extension.setall(0) + self.bitmap.extend(extension) + def validate_page(self, pagenum): self.bitmap[pagenum] = self.VALID self.bvc.counter[pagenum // self.conf.n_pages_per_block] += 1 @@ -800,6 +830,8 @@ def __init__(self, conf): for channel in range(self.conf.n_channels_per_dev) ] ) + self.max_physical_block = self.conf.n_blocks_per_dev + self.metadata = None # Should be set by Metadata def _incr_timestamp(self): """ @@ -821,6 +853,13 @@ def get_num_valid_pages(self, blocknum): # FIXME improve efficiency; now round-robin # bottleneck def next_free_block(self, wear_level=False): + if not self.free_block_list: + # Dynamic expansion fallback + free_block = self.max_physical_block + self.max_physical_block += 1 + if self.metadata: + self.metadata.pvb.expand(free_block) + return free_block free_block = self.free_block_list.popleft() return free_block # for block in range(self.conf.n_blocks_per_channel): @@ -829,7 +868,7 @@ def next_free_block(self, wear_level=False): # if self.get_num_valid_pages(pbn) == 0: # yield pbn - def gc_block(self, blocknum): + def erase_block(self, blocknum): self.counter[blocknum] = 0 self.free_block_list.append(blocknum) @@ -924,6 +963,7 @@ def __init__(self, confobj, counter): self.reference_mapping_table = PFTL() # flash block -> last invalidation time and num of valid pages self.bvc = BlockValidityCounter(confobj) + self.bvc.metadata = self # Key metadata structures self.pvb = PageValidityBitmap(confobj, self.bvc) # ppn -> lpn mapping stored in OOB @@ -938,6 +978,7 @@ def __init__(self, confobj, counter): # counters self.levels = defaultdict(int) + self.ppn_to_lpn_map = {} ############# Flash read related ############ @@ -1081,6 +1122,7 @@ def update_block(self, extents): # update reference mapping table for lpn, ppn in entries: self.reference_mapping_table.set(lpn, ppn) + self.ppn_to_lpn_map[ppn] = lpn # update oob # store the lpn of each ppn within [ppn - gamma - 1, ppn + gamma + 1] @@ -1104,6 +1146,8 @@ def erase_block(self, flash_block): self.oob.set_oob(ppn, None) # if you try to erase translation block here, it may fail, # but it is expected. + if ppn in self.ppn_to_lpn_map: + del self.ppn_to_lpn_map[ppn] del self.bvc.timestamp_table[ppn] except KeyError: pass From 35c22156b6f1336c772d649352a58498a3197156 Mon Sep 17 00:00:00 2001 From: "js0.site" Date: Sat, 24 Jan 2026 07:37:11 +0800 Subject: [PATCH 4/9] . --- wiscsee/leaftl_scripts/run_ftl | 1 + wiscsee/wiscsim/learnedftl.py | 71 ++++++++++++++++++---------------- 2 files changed, 38 insertions(+), 34 deletions(-) diff --git a/wiscsee/leaftl_scripts/run_ftl b/wiscsee/leaftl_scripts/run_ftl index d0dbe7f..23af471 100755 --- a/wiscsee/leaftl_scripts/run_ftl +++ b/wiscsee/leaftl_scripts/run_ftl @@ -179,6 +179,7 @@ class RunFTL: self.lineno, ) ) + self.conf["total_events"] = len(self.events) wf = Workflow(self.conf) self.sim = wf.run_simulator(self.events) mapping_table = self.sim.ssd.ftl.metadata.mapping_table diff --git a/wiscsee/wiscsim/learnedftl.py b/wiscsee/wiscsim/learnedftl.py index 7f80cce..edd7ddd 100644 --- a/wiscsee/wiscsim/learnedftl.py +++ b/wiscsee/wiscsim/learnedftl.py @@ -74,9 +74,9 @@ def __init__(self, confobj, recorderobj, flashobj, simpy_env, des_flash, ncq): self.pre_written_bytes = 0 self.pre_discarded_bytes = 0 self.pre_read_bytes = 0 - self.display_interval = 1000 * MB - self.compaction_interval = 100 * MB - self.promotion_interval = 500 * MB + self.display_interval = 500 * MB + self.compaction_interval = 200 * MB + self.promotion_interval = 1000 * MB self.gc_interval = 1000 * MB self.rw_events = 0 # self.buffer = WriteBuffer(16*self.conf.n_pages_per_block, 16*self.conf.n_pages_per_block, filtering=7.0/8.0) @@ -93,6 +93,9 @@ def __init__(self, confobj, recorderobj, flashobj, simpy_env, des_flash, ncq): self.waf = {"request": 0, "actual": 0} self.raf = {"request": 0, "actual": 0} self.enable_recording = False + + self.total_events = self.conf.get("total_events", 0) + self.processed_events = 0 def recorder_enabled(self, enable=True): self.enable_recording = enable @@ -196,11 +199,18 @@ def display_msg(self, mode): display_bytes = self.written_bytes avg_lookup = 0 - if float(sum(self.metadata.levels.values())) != 0: + total_lookups = sum(self.metadata.levels.values()) + if total_lookups != 0: avg_lookup = sum( int(k) * int(v) for k, v in list(self.metadata.levels.items()) - ) / float(sum(self.metadata.levels.values())) + ) / float(total_lookups) + progress_str = "" + if self.total_events > 0: + pct = (self.processed_events / self.total_events) * 100 + progress_str = "Progress: {:.2f}% ".format(pct) + log_msg( + progress_str, "Event", self.rw_events, f"{mode} (MB)", @@ -218,7 +228,7 @@ def display_msg(self, mode): "Latency per page: {:.2f}us".format( old_div( (np.sum(self.write_latencies) + np.sum(self.read_latencies)), - (self.waf["request"] + self.raf["request"]), + max(1, self.waf["request"] + self.raf["request"]), ) ), ) @@ -389,6 +399,7 @@ def lba_write(self, lpn, data=None): # yield self.env.process(self.garbage_collector.clean()) def write_ext(self, extent, data=None): + self.processed_events += 1 req_size = extent.lpn_count * self.conf.page_size self.recorder.add_to_general_accumulater("traffic", "write", req_size) self.written_bytes += req_size @@ -400,7 +411,7 @@ def write_ext(self, extent, data=None): self.display_msg("Write") self.pre_written_bytes = self.written_bytes - yield self.env.timeout(COMPACTION_DELAY) + # yield self.env.timeout(COMPACTION_DELAY) # roots = objgraph.get_leaking_objects() # objgraph.show_most_common_types() # objgraph.show_growth() @@ -539,19 +550,16 @@ def _sub_ext_data(self, data, extent, sub_ext): return sub_data def _write_ppns(self, ppns): - """ - The ppns in mappings is obtained from loggroup.next_ppns() - """ - # flash controller + # In fast mode, skip SimPy scheduling overhead + if not self.enable_recording: + return (0, 0) yield self.env.process(self.des_flash.rw_ppns(ppns, "write", tag="Unknown")) - return (0, 0) def _read_ppns(self, ppns): - """ - The ppns in mappings is obtained from loggroup.next_ppns() - """ - # flash controller + # In fast mode, skip SimPy scheduling overhead + if not self.enable_recording: + return yield self.env.process(self.des_flash.rw_ppns(ppns, "read", tag="Unknown")) def _update_log_mappings(self, mappings): @@ -572,23 +580,24 @@ def lba_discard(self, lpn): yield self.env.process(self._discard_logical_block(Extent(lpn, 1))) def discard_ext(self, extent): + self.processed_events += 1 req_size = extent.lpn_count * self.conf.page_size self.recorder.add_to_general_accumulater("traffic", "discard", req_size) self.discarded_bytes += req_size + if self.discarded_bytes > self.pre_discarded_bytes + self.display_interval: + pct = 0 + if self.total_events > 0: + pct = (self.processed_events / self.total_events) * 100 + log_msg( + "Progress: {:.2f}% ".format(pct), "Discarded (MB)", old_div(self.pre_discarded_bytes, MB), - "discarding", - round(float(req_size) / MB, 2), ) sys.stdout.flush() self.pre_discarded_bytes = self.discarded_bytes - self.recorder.add_to_general_accumulater( - "traffic", "discard", extent.lpn_count * self.conf.page_size - ) - extents = split_ext(self.conf.n_pages_per_block, extent) for logical_block_ext in extents: yield self.env.process(self._discard_logical_block(logical_block_ext)) @@ -2131,24 +2140,18 @@ def dist_levels(self): class PFTL: def __init__(self): - # store update history for verification purpose - self.mapping_table = defaultdict(list) + # store only latest mapping for performance + self.mapping_table = {} def set(self, lpn, ppn): - self.mapping_table[lpn].append(ppn) + self.mapping_table[lpn] = ppn def get(self, lpn): - # force check; since we are using defaultdict we don't want to create empty entry - if lpn not in self.mapping_table: - return None - ppns = self.mapping_table[lpn] - if len(ppns) > 0: - return ppns[-1] + return self.mapping_table.get(lpn) def get_all(self, lpn): - if lpn not in self.mapping_table: - return None - return self.mapping_table[lpn] + ppn = self.mapping_table.get(lpn) + return [ppn] if ppn is not None else None @property def memory(self): From f225401a114495ccb034cb1c1d386dce7b389ef6 Mon Sep 17 00:00:00 2001 From: "js0.site" Date: Sat, 24 Jan 2026 07:40:44 +0800 Subject: [PATCH 5/9] Opt: bypass SimPy async for faster read/write --- wiscsee/wiscsim/learnedftl.py | 315 +++++++++------------------------- 1 file changed, 79 insertions(+), 236 deletions(-) diff --git a/wiscsee/wiscsim/learnedftl.py b/wiscsee/wiscsim/learnedftl.py index edd7ddd..5a32f33 100644 --- a/wiscsee/wiscsim/learnedftl.py +++ b/wiscsee/wiscsim/learnedftl.py @@ -247,149 +247,62 @@ def display_msg(self, mode): # log_msg("Mapping Table Read Miss Ratio: %.2f" % (self.counter['mapping_table_read_miss'] / float(self.counter['mapping_table_read_miss'] + self.counter['mapping_table_read_hit']))) def read_ext(self, extent): - should_print = False + self.processed_events += 1 req_size = extent.lpn_count * self.conf.page_size self.recorder.add_to_general_accumulater("traffic", "read", req_size) self.read_bytes += req_size self.rw_events += 1 + if self.read_bytes > self.pre_read_bytes + self.display_interval: self.display_msg("Read") self.pre_read_bytes = self.read_bytes - extents = split_ext(extent) - start_time = self.env.now # <----- start - - op_id = self.recorder.get_unique_num() - # if op_id == 279353: - # should_print = True - procs = [] - total_pages_to_read = [] - total_pages_to_write = [] - requested_read = 0.0 - - if should_print: - log_msg(start_time) + start_time = self.env.now + requested_read = 0 + actual_read_count = 0 + + # Local cache for hot objects + ref_table = self.metadata.reference_mapping_table + lpn_to_ppn_func = self.metadata.lpn_to_ppn + rw_cache = self.rw_cache + is_rec = self.enable_recording - for ext in extents: - # nothing to lookup if no lpn is written - if not self.metadata.reference_mapping_table.get(ext.lpn_start): + for lpn in extent.lpn_iter(): + if not ref_table.get(lpn): self.counter["RAW"] += 1 continue - - assert ext.lpn_count == 1 + requested_read += 1 - # cachehit, writeback, lpn_previous = self.datacache.read(ext.lpn_start) - - # cachehit = self.datacache.read(ext.lpn_start) # or self.buffer.read(ext.lpn_start) - cachehit, writeback, ppn = self.rw_cache.read(ext.lpn_start) - + cachehit, writeback, ppn = rw_cache.read(lpn) + if not cachehit: - procs += [self.env.process(self.read_logical_block(ext, should_print))] + target_ppn, pages_to_write, pages_to_read = lpn_to_ppn_func(lpn) + if target_ppn: + if target_ppn not in pages_to_read: + pages_to_read.append(target_ppn) + self.hist[len(pages_to_read)] += 1 + + for r_ppn in pages_to_read: + self._read_ppns_sync([r_ppn]) + actual_read_count += 1 + for w_ppn in pages_to_write: + self._write_ppns_sync([w_ppn]) + else: + self._read_ppns_sync([0]) else: self.hist[0] += 1 - yield self.env.timeout(CACHE_HIT) - # yield self.env.timeout(delay=100) - # if writeback: - # additional_lpns_to_evict = self.datacache.evict_extra(0) - # yield self.env.process(self.write_ext(Extent(lpn_previous, 1), cached=False)) - # yield extra writes - + if is_rec: + yield self.env.timeout(CACHE_HIT) + if writeback: - yield self.env.process(self._write_ppns([ppn])) - total_pages_to_write.append(ppn) - - # self.env.exit(ext_data) - if should_print: - log_msg(requested_read) - log_msg(self.env.active_process) - - ret = yield simpy.AllOf(self.env, procs) - - total_pages_to_read += [ - page for pages in list(ret.values()) for page in pages[1] - ] - total_pages_to_write += [ - page for pages in list(ret.values()) for page in pages[0] - ] - - lpns_to_read = float(len(total_pages_to_read)) - - end_time = self.env.now + self._write_ppns_sync([ppn]) - if self.enable_recording: + if is_rec: + end_time = self.env.now if requested_read > 0: - self.read_latencies += [ - (end_time - start_time) / 1000.0 - ] # [(end_time - start_time)/(1000.0*requested_read)]*int(requested_read) - + self.read_latencies.append((end_time - start_time) / 1000.0) self.raf["request"] += requested_read - self.raf["actual"] += lpns_to_read - self.waf["actual"] += len(total_pages_to_write) - - write_timeline( - self.conf, - self.recorder, - op_id=op_id, - op="read_ext", - arg=extent.lpn_count, - start_time=start_time, - end_time=end_time, - ) - - # if lpns_to_read > len(extents) and len(extents) >= 6: - # log_msg(lpns_to_read, len(extents), requested_read, len(pages_to_write), ((end_time - start_time)/(1000.0*requested_read)), end_time - start_time) - # log_msg([page // self.conf.n_pages_per_channel for page in pages_to_read]) - - # self.env.exit(lpns_to_read) - - def read_logical_block(self, extent, should_print=False): - assert extent.lpn_count == 1 - - # replace the following lines with a nice interface - lpn = extent.lpn_start - read_ppns = [] - - # if self.conf["dry_run"]: - # self.hist[len(read_ppns)] += 1 - # read_ppn = self.metadata.reference_mapping_table.get(lpn) - # yield self.env.process(self.des_flash.rw_ppns([read_ppn], 'read', tag = "Unknown")) - # return - - ppn, pages_to_write, pages_to_read = self.metadata.lpn_to_ppn(lpn) - - if ppn: - # if accurate mapping entry - if ppn not in pages_to_read: - pages_to_read.append(ppn) - - self.hist[len(pages_to_read)] += 1 - # if len(read_ppns) >= 2: - # self.datacache.set_priority(extent.lpn_start, priority=True) - # read_ppns = read_ppns[-1:] - block_id = old_div(ppn, self.conf.n_pages_per_block) - - # content = self.flash.page_read(ppn, cat = TAG_FORGROUND) - # contents.append(content) - procs = [] - for read_ppn in pages_to_read: - if should_print: - log_msg(read_ppn, self.env.now) - yield self.env.process(self._read_ppns([read_ppn])) - - for write_ppn in pages_to_write: - if should_print: - log_msg(write_ppn, self.env.now) - yield self.env.process(self._write_ppns([write_ppn])) - - # yield simpy.AllOf(self.env, procs) - if should_print: - log_msg("Hi", pages_to_read, self.env.now) - - # else: - # # yield self.env.timeout(delay=self.conf['flash_config']['t_R']) - # yield self.env.process( - # self.des_flash.rw_ppns([0], 'read', tag = "Unknown")) - return (pages_to_write, pages_to_read) + self.raf["actual"] += actual_read_count def lba_write(self, lpn, data=None): yield self.env.process( @@ -405,129 +318,50 @@ def write_ext(self, extent, data=None): self.written_bytes += req_size self.rw_events += 1 + is_rec = self.enable_recording + if self.written_bytes > self.pre_written_bytes + self.display_interval: self.metadata.mapping_table.compact() self.metadata.mapping_table.promote() self.display_msg("Write") self.pre_written_bytes = self.written_bytes - - # yield self.env.timeout(COMPACTION_DELAY) - # roots = objgraph.get_leaking_objects() - # objgraph.show_most_common_types() - # objgraph.show_growth() - - # print(sys.getsizeof(self.metadata)) - # print(sys.getsizeof(self.metadata.mapping_table)) - - # print(len(pickle.dumps(self.datacache))) - # print(len(pickle.dumps(self.metadata))) - # print(len(pickle.dumps(self.metadata.mapping_table))) - - # from pympler import asizeof - # print(asizeof.asizeof(self.metadata)) - # print(asizeof.asizeof(self.metadata.reference_mapping_table)) - # print(asizeof.asizeof(self.metadata.pvb)) - # print(asizeof.asizeof(self.metadata.bvc)) - # print(asizeof.asizeof(self.metadata.oob)) - # print(asizeof.asizeof(self.metadata.levels)) - # print(asizeof.asizeof(self.metadata.mapping_table.frames)) - # print("") - # # print(vars(self.metadata.mapping_table)) - # print(asizeof.asizeof(self.metadata.mapping_table.frames.keys())) - # print(asizeof.asizeof(self.metadata.mapping_table.frames.values())) - # print(asizeof.asizeof(self.metadata.mapping_table.GTD)) - # print(asizeof.asizeof(self.metadata.mapping_table.memory_counter)) - # print(asizeof.asizeof(self.metadata.mapping_table.frame_on_flash)) - # print(asizeof.asizeof(self.metadata.mapping_table.dirty)) - - # print(asizeof.asizeof(self.metadata.mapping_table.frames.values()[0].runs)) - # print(asizeof.asizeof(self.metadata.mapping_table.frames.values()[0].segments)) - - # log_msg('Mapping Table Compaction Begins') - - # if self.written_bytes % self.compaction_interval == 0: + self.metadata.mapping_table.compact() - - # if self.written_bytes % self.promotion_interval == 0: self.metadata.mapping_table.promote() - # log_msg('Mapping Table Compaction Ends') - extents = split_ext(extent) - start_time = self.env.now # <----- start - op_id = self.recorder.get_unique_num() - write_procs = [] - total_pages_to_write = [] - for ext in extents: - # if cached: - # assert(ext.lpn_count == 1) - # writeback, lpn_previous = self.datacache.write(ext.lpn_start) - # if writeback: - # additional_lpns_to_evict = self.datacache.evict_extra(0) - # yield self.env.process(self.write_ext(Extent(lpn_previous, 1), cached=False)) - # yield extra writes - # else: - # self.datacache.resize(4*MB - self.metadata.mapping_table.memory) - # self.datacache.resize(4*MB - self.metadata.reference_mapping_table.memory) - # for lpn in exts: - # p = self.env.process(self.write_single_page(mappings[lpn])) - # write_procs.append(p) - # print(ext.lpn_start) - # if self.rw_events > 10: - # exit() + start_time = self.env.now + total_pages_to_write_count = 0 + + # Local optimization + rw_cache = self.rw_cache + meta_update = self.metadata.update - # TODO: should we modify data cache here? Jinghan: Done - # FIXME: check if there is any bug here - # self.datacache.read(ext.lpn_start) - writeback, ppn = self.rw_cache.write(ext.lpn_start) + for lpn in extent.lpn_iter(): + writeback, ppn = rw_cache.write(lpn) if writeback: - p = self.env.process(self._write_ppns([ppn])) - write_procs.append(p) - total_pages_to_write.append(ppn) - - # self.datacache.invalidate(ext) - if self.rw_cache.should_assign_page(): - exts = self.rw_cache.flush_unassigned() - mappings, pages_to_read, pages_to_write = self.metadata.update(exts) - self.rw_cache.update_assigned(mappings) - total_pages_to_write.extend(pages_to_write) - # if len(pages_to_read) != 0 or len(pages_to_write) != 0: - # print(pages_to_read, pages_to_write) - - # self.buffer.write(ext.lpn_start) - # self.datacache.invalidate(ext) - # if self.buffer.should_flush(): - # exts = self.buffer.flush() - # self.counter += len(exts) - # mappings = self.metadata.update(exts) - for ppn in pages_to_write: - p = self.env.process(self._write_ppns([ppn])) - write_procs.append(p) - - for ppn in pages_to_read: - p = self.env.process(self._read_ppns([ppn])) - write_procs.append(p) - - if not writeback: - yield self.env.timeout(CACHE_HIT) - - yield simpy.AllOf(self.env, write_procs) - - end_time = self.env.now # <----- end + self._write_ppns_sync([ppn]) + total_pages_to_write_count += 1 - if self.enable_recording: + if rw_cache.should_assign_page(): + lpa_list = rw_cache.flush_unassigned() + mappings, pages_to_read, pages_to_write = meta_update(lpa_list) + rw_cache.update_assigned(mappings) + + for p_ppn in pages_to_write: + self._write_ppns_sync([p_ppn]) + total_pages_to_write_count += 1 + for p_ppn in pages_to_read: + self._read_ppns_sync([p_ppn]) + + if is_rec: + if not writeback: + yield self.env.timeout(CACHE_HIT) + + if is_rec: + end_time = self.env.now self.write_latencies.append((end_time - start_time) / 1000.0) self.waf["request"] += extent.lpn_count - self.waf["actual"] += len(total_pages_to_write) - - write_timeline( - self.conf, - self.recorder, - op_id=op_id, - op="write_ext", - arg=extent.lpn_count, - start_time=start_time, - end_time=end_time, - ) + self.waf["actual"] += total_pages_to_write_count def _block_iter_of_extent(self, extent): block_start, _ = self.conf.page_to_block_off(extent.lpn_start) @@ -576,8 +410,17 @@ def _remap_oob(self, new_mappings): old_ppn = None self.oob.remap(lpn=lpn, old_ppn=old_ppn, new_ppn=new_ppn) - def lba_discard(self, lpn): - yield self.env.process(self._discard_logical_block(Extent(lpn, 1))) + def _write_ppns_sync(self, ppns): + # Full bypass of SimPy for speed + if not self.enable_recording: + return + self.env.process(self.des_flash.rw_ppns(ppns, "write", tag="Unknown")) + + def _read_ppns_sync(self, ppns): + # Full bypass of SimPy for speed + if not self.enable_recording: + return + self.env.process(self.des_flash.rw_ppns(ppns, "read", tag="Unknown")) def discard_ext(self, extent): self.processed_events += 1 From ae6323f4e893e7063b11b925d9431307f31fabae Mon Sep 17 00:00:00 2001 From: "js0.site" Date: Sat, 24 Jan 2026 07:42:25 +0800 Subject: [PATCH 6/9] Opt: handle zero-capacity cache and chunked bitmap expansion --- wiscsee/wiscsim/datacache.py | 4 ++++ wiscsee/wiscsim/learnedftl.py | 7 +++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/wiscsee/wiscsim/datacache.py b/wiscsee/wiscsim/datacache.py index a02645e..149293e 100755 --- a/wiscsee/wiscsim/datacache.py +++ b/wiscsee/wiscsim/datacache.py @@ -432,6 +432,8 @@ def method2cache(self, method): return MRUCache() def read(self, lpn): + if self.capacity <= 0: + return False, False, None writeback, wb_ppn = False, None if lpn not in self.cache: self.counter["read misses"] += 1 @@ -456,6 +458,8 @@ def read(self, lpn): return hit, writeback, wb_ppn def write(self, lpn): + if self.capacity <= 0: + return False, None writeback, wb_ppn = False, None if lpn not in self.cache: self.counter["write misses"] += 1 diff --git a/wiscsee/wiscsim/learnedftl.py b/wiscsee/wiscsim/learnedftl.py index 5a32f33..b79ca31 100644 --- a/wiscsee/wiscsim/learnedftl.py +++ b/wiscsee/wiscsim/learnedftl.py @@ -608,8 +608,11 @@ def __init__(self, conf, bvc): def expand(self, blocknum): total_pages_needed = (blocknum + 1) * self.conf.n_pages_per_block - if len(self.bitmap) < total_pages_needed: - extension = bitarray.bitarray(total_pages_needed - len(self.bitmap)) + current_pages = len(self.bitmap) + if current_pages < total_pages_needed: + # Chunked expansion: at least 1024 blocks or double current size + expand_to = max(total_pages_needed, current_pages * 2, 1024 * self.conf.n_pages_per_block) + extension = bitarray.bitarray(expand_to - current_pages) extension.setall(0) self.bitmap.extend(extension) From 0d2d62e258596c0351038ece5770e36499adfdf3 Mon Sep 17 00:00:00 2001 From: "js0.site" Date: Sat, 24 Jan 2026 07:51:14 +0800 Subject: [PATCH 7/9] Opt: disable warm-up for performance --- wiscsee/wiscsim/workload_parser.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/wiscsee/wiscsim/workload_parser.py b/wiscsee/wiscsim/workload_parser.py index e2113c1..dd7593a 100644 --- a/wiscsee/wiscsim/workload_parser.py +++ b/wiscsee/wiscsim/workload_parser.py @@ -269,16 +269,9 @@ def parse_events( if write_only: continue op = OP_READ - should_warm_up = False - for lpn in split_lpns(offset, size, page_size): - if lpn not in exist_lpns: - should_warm_up = True - exist_lpns[lpn] = None - if should_warm_up: - warm_up_writes += [ - Event(512, 0, OP_WRITE, offset, size, timestamp=0) - ] - num_writes += len(split_lpns(offset, size, page_size)) + # Disabled warm-up for performance + # for lpn in split_lpns(offset, size, page_size): + # exist_lpns[lpn] = None elif mode in ["Write", "W", 1, "w", "WS", "WM"]: op = OP_WRITE for lpn in split_lpns(offset, size, page_size): From 700342e0c39fe382c09dd2ab2f0e10145883ce10 Mon Sep 17 00:00:00 2001 From: "js0.site" Date: Sat, 24 Jan 2026 09:20:40 +0800 Subject: [PATCH 8/9] . --- bench_selected.py | 35 +++++++++++++++++++++++++--------- wiscsee/leaftl_scripts/run_ftl | 11 +++++++---- 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/bench_selected.py b/bench_selected.py index 6029797..1ae4530 100755 --- a/bench_selected.py +++ b/bench_selected.py @@ -54,6 +54,11 @@ def main(): # Traces directory data_dir = os.path.join(script_dir, "data") run_ftl_script = os.path.join("wiscsee", "leaftl_scripts", "run_ftl") + + # Results directory + json_dir = os.path.join(script_dir, "json") + if not os.path.exists(json_dir): + os.makedirs(json_dir) # Scan for all .bin files generated by init.js trace_files = sorted(glob.glob(os.path.join(data_dir, "*.bin"))) @@ -69,6 +74,13 @@ def main(): for trace_path in trace_files: trace_name = os.path.basename(trace_path) + trace_base = os.path.splitext(trace_name)[0] + result_file = os.path.join(json_dir, f"{trace_base}.json") + + if os.path.exists(result_file): + print(f"\n>>> Skipping {trace_name} (result already exists in {result_file})") + continue + print(f"\n>>> Processing {trace_name}") # 1. Run LearnedFTL (provides LeaFTL and DFTL sizes) @@ -122,19 +134,24 @@ def main(): if sftl_match: sftl_mb = float(sftl_match.group(1)) / (1024 * 1024) + ratio = dftl_mb / leaftl_mb if leaftl_mb > 0 else 0 summary_line = ( - f"SUMMARY: {trace_name}, {dftl_mb:.4f}, {sftl_mb:.4f}, {leaftl_mb:.4f}" + f"SUMMARY: {trace_name}, DFTL: {dftl_mb:.4f} MB, SFTL: {sftl_mb:.4f} MB, LeaFTL: {leaftl_mb:.4f} MB, Ratio(DFTL/LeaFTL): {ratio:.2f}x" ) print(summary_line) - results.append( - { - "trace": trace_name, - "dftl_mb": round(dftl_mb, 4), - "sftl_mb": round(sftl_mb, 4), - "leaftl_mb": round(leaftl_mb, 4), - } - ) + res_obj = { + "trace": trace_name, + "dftl_mb": round(dftl_mb, 4), + "sftl_mb": round(sftl_mb, 4), + "leaftl_mb": round(leaftl_mb, 4), + "ratio": round(ratio, 2) + } + results.append(res_obj) + + # Save individual result immediately + with open(result_file, "w") as f: + json.dump(res_obj, f, indent=4) sys.stdout.flush() diff --git a/wiscsee/leaftl_scripts/run_ftl b/wiscsee/leaftl_scripts/run_ftl index 23af471..9341ae1 100755 --- a/wiscsee/leaftl_scripts/run_ftl +++ b/wiscsee/leaftl_scripts/run_ftl @@ -207,10 +207,13 @@ class RunFTL: % len(list(filter(lambda seg: seg.consecutive, mapping_table.segments))) ) log_msg("estimated learnedftl memory footprint: %d B" % mapping_table.memory) - log_msg( - "estimated dftl memory footprint: %d B" - % (len(reference_mapping_table.mapping_table) * 8) - ) + dftl_memory = len(reference_mapping_table.mapping_table) * 8 + log_msg("estimated dftl memory footprint: %d B" % dftl_memory) + if mapping_table.memory > 0: + log_msg("Compression Ratio: %.2fx (Savings: %.2f%%)" % ( + dftl_memory / mapping_table.memory, + (1 - (mapping_table.memory / dftl_memory)) * 100 if dftl_memory > 0 else 0 + )) def print_current_stats(self): try: From b917770fb05de02eecb2ab93fa32709a4954cb54 Mon Sep 17 00:00:00 2001 From: "js0.site" Date: Sat, 24 Jan 2026 17:32:58 +0800 Subject: [PATCH 9/9] . --- bench_selected.py | 18 ++--- full_fast_replay.py | 147 ++++++++++++++++++++++++++++++++++ json/hm_1.json | 7 ++ wiscsee/wiscsim/learnedftl.py | 77 ++++++++++-------- 4 files changed, 208 insertions(+), 41 deletions(-) create mode 100755 full_fast_replay.py create mode 100644 json/hm_1.json diff --git a/bench_selected.py b/bench_selected.py index 1ae4530..07701a3 100755 --- a/bench_selected.py +++ b/bench_selected.py @@ -54,7 +54,7 @@ def main(): # Traces directory data_dir = os.path.join(script_dir, "data") run_ftl_script = os.path.join("wiscsee", "leaftl_scripts", "run_ftl") - + # Results directory json_dir = os.path.join(script_dir, "json") if not os.path.exists(json_dir): @@ -76,11 +76,13 @@ def main(): trace_name = os.path.basename(trace_path) trace_base = os.path.splitext(trace_name)[0] result_file = os.path.join(json_dir, f"{trace_base}.json") - + if os.path.exists(result_file): - print(f"\n>>> Skipping {trace_name} (result already exists in {result_file})") + print( + f"\n>>> Skipping {trace_name} (result already exists in {result_file})" + ) continue - + print(f"\n>>> Processing {trace_name}") # 1. Run LearnedFTL (provides LeaFTL and DFTL sizes) @@ -135,9 +137,7 @@ def main(): sftl_mb = float(sftl_match.group(1)) / (1024 * 1024) ratio = dftl_mb / leaftl_mb if leaftl_mb > 0 else 0 - summary_line = ( - f"SUMMARY: {trace_name}, DFTL: {dftl_mb:.4f} MB, SFTL: {sftl_mb:.4f} MB, LeaFTL: {leaftl_mb:.4f} MB, Ratio(DFTL/LeaFTL): {ratio:.2f}x" - ) + summary_line = f"SUMMARY: {trace_name}, DFTL: {dftl_mb:.4f} MB, SFTL: {sftl_mb:.4f} MB, LeaFTL: {leaftl_mb:.4f} MB, Ratio(DFTL/LeaFTL): {ratio:.2f}x" print(summary_line) res_obj = { @@ -145,10 +145,10 @@ def main(): "dftl_mb": round(dftl_mb, 4), "sftl_mb": round(sftl_mb, 4), "leaftl_mb": round(leaftl_mb, 4), - "ratio": round(ratio, 2) + "ratio": round(ratio, 2), } results.append(res_obj) - + # Save individual result immediately with open(result_file, "w") as f: json.dump(res_obj, f, indent=4) diff --git a/full_fast_replay.py b/full_fast_replay.py new file mode 100755 index 0000000..aac74cd --- /dev/null +++ b/full_fast_replay.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +import os +import sys +import struct +import json +import glob +import time +from collections import defaultdict +import numpy as np + +# Add current dir to path to find wiscsee +current_dir = os.path.dirname(os.path.abspath(__file__)) +wiscsee_path = os.path.join(current_dir, "wiscsee") +sys.path.append(wiscsee_path) + +from wiscsim.learnedftl import FlashMetadata +import config + +class FastConfig(config.ConfigNCQFTL): + def __init__(self, internal_type="learnedftl"): + super().__init__() + self["flash_config"]["page_size"] = 4096 + self["flash_config"]["n_pages_per_block"] = 128 + self["flash_config"]["n_blocks_per_plane"] = 1000000 + self["gamma"] = 0.0001 + self["mapping_cache_bytes"] = 2**40 # Large cache for analysis + self["internal_ftl_type"] = internal_type + self["ftl_type"] = "learnedftl" + +def fast_replay(trace_path): + trace_file_name = os.path.basename(trace_path) + print(f"\n[Replay] {trace_file_name}") + + # 1. Initialize metadata for both FTL types + # DFTL size is calculated from unique LPNs + meta_lea = FlashMetadata(FastConfig("learnedftl"), defaultdict(float)) + meta_sftl = FlashMetadata(FastConfig("sftl"), defaultdict(float)) + + # Binary op codes + OP_GET, OP_SET, OP_MGET, OP_MSET, OP_RM, OP_MRM = 0x00, 0x01, 0x02, 0x03, 0x04, 0x05 + + total_ops = 0 + write_pages = 0 + start_time = time.time() + last_print = start_time + + MB_UNIT = 1024 * 1024 + + try: + with open(trace_path, "rb") as f: + while True: + op_buf = f.read(1) + if not op_buf: break + op = struct.unpack("B", op_buf)[0] + + lba_buf = f.read(4) + if not lba_buf: break + lba = struct.unpack(" 3.0: + pct = (total_ops / 600000) * 100 # Rough progress for large traces + print(f" Processed {total_ops} ops ({write_pages} total pages written)...", end="\r") + last_print = now + + except Exception as e: + print(f"\n Error: {e}") + + # 2. Final memory calculations + lea_bytes = meta_lea.mapping_table.memory + sftl_bytes = meta_sftl.mapping_table.memory + unique_lpns = len(meta_lea.reference_mapping_table.mapping_table) + dftl_bytes = unique_lpns * 8 + + duration = time.time() - start_time + print(f"\n Finished in {duration:.2f}s. Mapping Sizes (MB):") + print(f" DFTL: {dftl_bytes/MB_UNIT:.4f}") + print(f" SFTL: {sftl_bytes/MB_UNIT:.4f}") + print(f" LeaFTL: {lea_bytes/MB_UNIT:.4f}") + + return { + "trace": trace_file_name, + "dftl_mb": round(dftl_bytes / MB_UNIT, 4), + "sftl_mb": round(sftl_bytes / MB_UNIT, 4), + "leaftl_mb": round(lea_bytes / MB_UNIT, 4) + } + +def main(): + data_dir = os.path.join(current_dir, "data") + json_dir = os.path.join(current_dir, "json_fast_replay") + if not os.path.exists(json_dir): os.makedirs(json_dir) + + traces = sorted(glob.glob(os.path.join(data_dir, "*.bin"))) + + results = [] + for trace in traces: + res = fast_replay(trace) + results.append(res) + + # Save to JSON + trace_base = os.path.splitext(os.path.basename(trace))[0] + with open(os.path.join(json_dir, f"{trace_base}.json"), "w") as f: + json.dump(res, f, indent=4) + + print("\n" + "="*60) + print(f"{'Trace':<15} | {'DFTL (MB)':<10} | {'SFTL (MB)':<10} | {'LeaFTL (MB)':<10}") + print("-" * 60) + for r in results: + print(f"{r['trace']:<15} | {r['dftl_mb']:<10.4f} | {r['sftl_mb']:<10.4f} | {r['leaftl_mb']:<10.4f}") + print("="*60) + +if __name__ == "__main__": + main() diff --git a/json/hm_1.json b/json/hm_1.json new file mode 100644 index 0000000..912eb9f --- /dev/null +++ b/json/hm_1.json @@ -0,0 +1,7 @@ +{ + "trace": "hm_1.bin", + "dftl_mb": 0.0755, + "sftl_mb": 0.0484, + "leaftl_mb": 0.0475, + "ratio": 1.59 +} \ No newline at end of file diff --git a/wiscsee/wiscsim/learnedftl.py b/wiscsee/wiscsim/learnedftl.py index b79ca31..205b1c0 100644 --- a/wiscsee/wiscsim/learnedftl.py +++ b/wiscsee/wiscsim/learnedftl.py @@ -93,7 +93,7 @@ def __init__(self, confobj, recorderobj, flashobj, simpy_env, des_flash, ncq): self.waf = {"request": 0, "actual": 0} self.raf = {"request": 0, "actual": 0} self.enable_recording = False - + self.total_events = self.conf.get("total_events", 0) self.processed_events = 0 @@ -207,7 +207,7 @@ def display_msg(self, mode): progress_str = "" if self.total_events > 0: pct = (self.processed_events / self.total_events) * 100 - progress_str = "Progress: {:.2f}% ".format(pct) + progress_str = f"Progress: {pct:.2f}% " log_msg( progress_str, @@ -260,7 +260,7 @@ def read_ext(self, extent): start_time = self.env.now requested_read = 0 actual_read_count = 0 - + # Local cache for hot objects ref_table = self.metadata.reference_mapping_table lpn_to_ppn_func = self.metadata.lpn_to_ppn @@ -271,17 +271,17 @@ def read_ext(self, extent): if not ref_table.get(lpn): self.counter["RAW"] += 1 continue - + requested_read += 1 cachehit, writeback, ppn = rw_cache.read(lpn) - + if not cachehit: target_ppn, pages_to_write, pages_to_read = lpn_to_ppn_func(lpn) if target_ppn: if target_ppn not in pages_to_read: pages_to_read.append(target_ppn) self.hist[len(pages_to_read)] += 1 - + for r_ppn in pages_to_read: self._read_ppns_sync([r_ppn]) actual_read_count += 1 @@ -293,7 +293,7 @@ def read_ext(self, extent): self.hist[0] += 1 if is_rec: yield self.env.timeout(CACHE_HIT) - + if writeback: self._write_ppns_sync([ppn]) @@ -325,13 +325,13 @@ def write_ext(self, extent, data=None): self.metadata.mapping_table.promote() self.display_msg("Write") self.pre_written_bytes = self.written_bytes - + self.metadata.mapping_table.compact() self.metadata.mapping_table.promote() start_time = self.env.now total_pages_to_write_count = 0 - + # Local optimization rw_cache = self.rw_cache meta_update = self.metadata.update @@ -346,7 +346,7 @@ def write_ext(self, extent, data=None): lpa_list = rw_cache.flush_unassigned() mappings, pages_to_read, pages_to_write = meta_update(lpa_list) rw_cache.update_assigned(mappings) - + for p_ppn in pages_to_write: self._write_ppns_sync([p_ppn]) total_pages_to_write_count += 1 @@ -432,9 +432,9 @@ def discard_ext(self, extent): pct = 0 if self.total_events > 0: pct = (self.processed_events / self.total_events) * 100 - + log_msg( - "Progress: {:.2f}% ".format(pct), + f"Progress: {pct:.2f}% ", "Discarded (MB)", old_div(self.pre_discarded_bytes, MB), ) @@ -484,7 +484,10 @@ def clean(self, forced=False, merge=True): for block in self.metadata.bvc.counter: num_validate_pages = self.metadata.bvc.counter[block] - if num_validate_pages > 0 and num_validate_pages / float(self.conf.n_pages_per_block) <= 0.1: + if ( + num_validate_pages > 0 + and num_validate_pages / float(self.conf.n_pages_per_block) <= 0.1 + ): erased_pbns.append(block) valid_ppns += self.metadata.pvb.get_valid_pages(block) @@ -496,8 +499,10 @@ def clean(self, forced=False, merge=True): victim_candidates = [] for block in list(self.metadata.bvc.counter.keys()): if self.metadata.bvc.counter[block] < self.conf.n_pages_per_block: - victim_candidates.append((self.metadata.bvc.counter[block], block)) - + victim_candidates.append( + (self.metadata.bvc.counter[block], block) + ) + if victim_candidates: victim_candidates.sort() # Pick top 4 best victims or fewer if not available @@ -518,8 +523,10 @@ def clean(self, forced=False, merge=True): read_procs = [] write_procs = [] if lpns_to_move: - mappings, pages_to_read_meta, pages_to_write = self.metadata.update(lpns_to_move) - + mappings, pages_to_read_meta, pages_to_write = self.metadata.update( + lpns_to_move + ) + for ppn in valid_ppns: read_procs.append(self.env.process(self._read_ppns([ppn]))) for ppn in pages_to_read_meta: @@ -541,18 +548,16 @@ def clean(self, forced=False, merge=True): yield simpy.AllOf(self.env, read_procs) yield simpy.AllOf(self.env, write_procs) yield simpy.AllOf(self.env, erase_procs) - + for erased_pbn in erased_pbns: self.metadata.erase_block(erased_pbn) end = self.env.now if erased_pbns: print( - ( - f"GC finished: moved {len(lpns_to_move)} pages, " - f"erased {len(erased_pbns)} blocks, " - f"time: {end - start}" - ) + f"GC finished: moved {len(lpns_to_move)} pages, " + f"erased {len(erased_pbns)} blocks, " + f"time: {end - start}" ) def is_wear_leveling_needed(self): @@ -611,7 +616,11 @@ def expand(self, blocknum): current_pages = len(self.bitmap) if current_pages < total_pages_needed: # Chunked expansion: at least 1024 blocks or double current size - expand_to = max(total_pages_needed, current_pages * 2, 1024 * self.conf.n_pages_per_block) + expand_to = max( + total_pages_needed, + current_pages * 2, + 1024 * self.conf.n_pages_per_block, + ) extension = bitarray.bitarray(expand_to - current_pages) extension.setall(0) self.bitmap.extend(extension) @@ -686,7 +695,7 @@ def __init__(self, conf): ] ) self.max_physical_block = self.conf.n_blocks_per_dev - self.metadata = None # Should be set by Metadata + self.metadata = None # Should be set by Metadata def _incr_timestamp(self): """ @@ -1162,10 +1171,10 @@ def is_valid(self, x): def get_y(self, x, check=True): if not check or self.is_valid(x): - predict = int(round(x * self.k + self.b)) - # lowbound = self.blocknum * Segment.PAGE_PER_BLOCK - # upbound = (self.blocknum + 1) * Segment.PAGE_PER_BLOCK - 1 - # return max(min(predict, upbound), lowbound) + val = x * self.k + self.b + if not np.isfinite(val): + return -1 + predict = int(round(val)) return predict return None @@ -1396,10 +1405,14 @@ def build_segment(self): 1, self.s0[1] - self.s0[0], self.s0[0], self.s0[0], points=self.points ) elif self.state == PLR.READY: - avg_slope = np.float16((self.rho_lower.k + self.rho_upper.k) / 2.0) - # avg_slope = (self.rho_lower.k + self.rho_upper.k) / 2.0 - # rec_k = round(1.0/avg_slope) + avg_slope = (self.rho_lower.k + self.rho_upper.k) / 2.0 intercept = -self.sint[0] * avg_slope + self.sint[1] + + if not np.isfinite(avg_slope) or not np.isfinite(intercept): + # Fallback to simple mapping if regression fails + avg_slope = 1.0 + intercept = self.s0[1] - self.s0[0] + seg = Segment( avg_slope, intercept, self.s0[0], self.s1[0], points=self.points )