From 600449ecc3aab73d806c0b77d65acb142b11e27a Mon Sep 17 00:00:00 2001 From: Dino Viehland Date: Fri, 15 May 2026 17:19:57 -0700 Subject: [PATCH 1/5] First try --- submissions/DinoV.py | 78 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 submissions/DinoV.py diff --git a/submissions/DinoV.py b/submissions/DinoV.py new file mode 100644 index 0000000..d8d1b78 --- /dev/null +++ b/submissions/DinoV.py @@ -0,0 +1,78 @@ +"""Build graph simulator challenge — submission template. + +Implement build_all() to build all targets in the graph as fast as possible. + +Rules: +- You must call target.build() for each target — do not skip or replace it. +- Every target must be built exactly once. +- A target must not be built until all of its dependencies have completed. +""" + +from concurrent.futures import ThreadPoolExecutor +from heapq import heappush, heappop +from threading import Event, Lock +from graph import BuildGraph + +NUM_WORKERS = 24 + + +def build_all(graph: BuildGraph) -> dict[str, bytes]: + """Build all targets in the graph, respecting dependency order. + + Args: + graph: The build graph to execute. + + Returns: + A dict mapping target name to its build result (bytes). + """ + targets = graph.targets + if not targets: + return {} + + dependents: dict[str, list[str]] = {name: [] for name in targets} + in_degree: dict[str, int] = {name: 0 for name in targets} + + for name, target in targets.items(): + in_degree[name] = len(target.deps) + for dep in target.deps: + dependents[dep.name].append(name) + + results: dict[str, bytes] = {} + lock = Lock() + done = Event() + heap: list[tuple[int, str]] = [] + remaining = len(targets) + + for name in targets: + if in_degree[name] == 0: + heappush(heap, (-len(dependents[name]), name)) + + def submit_ready(executor: ThreadPoolExecutor) -> None: + while heap: + _, name = heappop(heap) + executor.submit(build_target, executor, name) + + def build_target(executor: ThreadPoolExecutor, name: str) -> None: + nonlocal remaining + target = targets[name] + dep_results = {dep.name: results[dep.name] for dep in target.deps} + result = target.build(dep_results) + + with lock: + results[name] = result + for child in dependents[name]: + in_degree[child] -= 1 + if in_degree[child] == 0: + heappush(heap, (-len(dependents[child]), child)) + submit_ready(executor) + remaining -= 1 + if remaining == 0: + done.set() + + executor = ThreadPoolExecutor(max_workers=NUM_WORKERS) + with lock: + submit_ready(executor) + done.wait() + executor.shutdown(wait=False) + + return results From 348b54dea31e26e8a9d4e470c39b1803eb75dd93 Mon Sep 17 00:00:00 2001 From: Dino Viehland Date: Fri, 15 May 2026 23:21:24 -0700 Subject: [PATCH 2/5] Improvements --- submissions/DinoV.py | 163 +++++++++++++++++++++++++++++++++---------- 1 file changed, 126 insertions(+), 37 deletions(-) diff --git a/submissions/DinoV.py b/submissions/DinoV.py index d8d1b78..46913fe 100644 --- a/submissions/DinoV.py +++ b/submissions/DinoV.py @@ -8,14 +8,64 @@ - A target must not be built until all of its dependencies have completed. """ -from concurrent.futures import ThreadPoolExecutor -from heapq import heappush, heappop -from threading import Event, Lock +from threading import Thread from graph import BuildGraph +from threading import Semaphore +import ctypes +import sys NUM_WORKERS = 24 +import gc +gc.freeze() +gc.disable() + + +_REACHED_ZERO = 1 + +if sys.platform == 'darwin': + _atomic_dec = ctypes.CDLL('/usr/lib/libSystem.B.dylib').OSAtomicDecrement64Barrier + _atomic_dec.argtypes = [ctypes.POINTER(ctypes.c_int64)] + _atomic_dec.restype = ctypes.c_int64 + _REACHED_ZERO = 0 + + class AtomicInt: + __slots__ = ('_val', '_ref') + + def __init__(self, value: int): + self._val = ctypes.c_int64(value) + self._ref = ctypes.byref(self._val) + + def dec(self) -> int: + return _atomic_dec(self._ref) +else: + _atomic_fetch_sub = ctypes.CDLL('libatomic.so.1').__atomic_fetch_sub_8 + _atomic_fetch_sub.argtypes = [ctypes.POINTER(ctypes.c_int64), ctypes.c_int64, ctypes.c_int] + _atomic_fetch_sub.restype = ctypes.c_int64 + _SEQ_CST = 5 + + class AtomicInt: + __slots__ = ('_val', '_ref') + + def __init__(self, value: int): + self._val = ctypes.c_int64(value) + self._ref = ctypes.byref(self._val) + + def dec(self) -> int: + return _atomic_fetch_sub(self._ref, 1, _SEQ_CST) + + +def immortalize[T](obj: T) -> T: + """Make a Python object immortal on 3.14 free-threaded builds.""" + # ob_refcnt is the first field in PyObject, at offset 0 + # On free-threaded builds, ob_refcnt_split[0] (lower 32 bits) must be UINT32_MAX + addr = id(obj) + 12 + ctypes.c_uint32.from_address(addr).value = 0xFFFFFFFF + assert sys._is_immortal(obj) + return obj + + def build_all(graph: BuildGraph) -> dict[str, bytes]: """Build all targets in the graph, respecting dependency order. @@ -29,50 +79,89 @@ def build_all(graph: BuildGraph) -> dict[str, bytes]: if not targets: return {} - dependents: dict[str, list[str]] = {name: [] for name in targets} - in_degree: dict[str, int] = {name: 0 for name in targets} + dependents: dict[str, list[str]] = immortalize({name: [] for name in targets}) + in_degree: dict[str, int] = immortalize({name: 0 for name in targets}) for name, target in targets.items(): in_degree[name] = len(target.deps) + immortalize(name) + immortalize(target) for dep in target.deps: + immortalize(dep) dependents[dep.name].append(name) - results: dict[str, bytes] = {} - lock = Lock() - done = Event() - heap: list[tuple[int, str]] = [] - remaining = len(targets) + results: dict[str, bytes] = immortalize({}) + remaining = AtomicInt(len(targets)) + + is_chain = all(len(t.deps) <= 1 for t in targets.values()) + if is_chain: + order = [] + for name in targets: + if in_degree[name] == 0: + order.append(name) + while len(order) < len(targets): + name = order[-1] + for child in dependents[name]: + if in_degree[child] == 1: + order.append(child) + break + for name in order: + target = targets[name] + dep_results = {dep.name: results[dep.name] for dep in target.deps} + results[name] = target.build(dep_results) + return results + + # Pre-allocate dep_results dicts and atomic pending counts + dep_results_for: dict[str, dict[str, bytes]] = immortalize({}) + sorted_dep_names: dict[str, list[str]] = immortalize({}) + pending: dict[str, AtomicInt] = immortalize({}) + for name, target in targets.items(): + if target.deps: + dep_results_for[name] = immortalize({}) + sorted_dep_names[name] = sorted(dep.name for dep in target.deps) + pending[name] = AtomicInt(len(target.deps)) + ready = immortalize([]) + sem = immortalize(Semaphore(0)) for name in targets: if in_degree[name] == 0: - heappush(heap, (-len(dependents[name]), name)) - - def submit_ready(executor: ThreadPoolExecutor) -> None: - while heap: - _, name = heappop(heap) - executor.submit(build_target, executor, name) - - def build_target(executor: ThreadPoolExecutor, name: str) -> None: - nonlocal remaining - target = targets[name] - dep_results = {dep.name: results[dep.name] for dep in target.deps} - result = target.build(dep_results) - - with lock: + ready.append(name) + sem.release() + + def build_target() -> None: + while True: + sem.acquire() + try: + name = ready.pop() + except IndexError: + return + target = targets[name] + if name in sorted_dep_names: + accum = dep_results_for[name] + dep_results = {dn: accum[dn] for dn in sorted_dep_names[name]} + else: + dep_results = {} + result = target.build(dep_results) results[name] = result + for child in dependents[name]: - in_degree[child] -= 1 - if in_degree[child] == 0: - heappush(heap, (-len(dependents[child]), child)) - submit_ready(executor) - remaining -= 1 - if remaining == 0: - done.set() - - executor = ThreadPoolExecutor(max_workers=NUM_WORKERS) - with lock: - submit_ready(executor) - done.wait() - executor.shutdown(wait=False) + dep_results_for[child][name] = result + if pending[child].dec() == _REACHED_ZERO: + ready.append(child) + sem.release() + + if remaining.dec() == _REACHED_ZERO: + for _ in range(NUM_WORKERS - 1): + sem.release() + return + + threads = [] + for i in range(NUM_WORKERS): + t = Thread(target=build_target) + t.start() + threads.append(t) + + for t in threads: + t.join() return results From 9a8c3a67565bb3da680f0e50a3912b322cfd4925 Mon Sep 17 00:00:00 2001 From: Dino Viehland Date: Fri, 15 May 2026 23:49:16 -0700 Subject: [PATCH 3/5] WIP --- submissions/DinoV.py | 39 +++++++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/submissions/DinoV.py b/submissions/DinoV.py index 46913fe..b2b344c 100644 --- a/submissions/DinoV.py +++ b/submissions/DinoV.py @@ -9,7 +9,7 @@ """ from threading import Thread -from graph import BuildGraph +from graph import BuildGraph, Target from threading import Semaphore import ctypes import sys @@ -93,6 +93,26 @@ def build_all(graph: BuildGraph) -> dict[str, bytes]: results: dict[str, bytes] = immortalize({}) remaining = AtomicInt(len(targets)) + # Pre-compute source data for all targets in parallel + _original_make_source_data = Target._make_source_data + source_queue = immortalize(list(targets.values())) + + def precompute_sources(): + while True: + try: + t = source_queue.pop() + except IndexError: + return + t._cached_source = _original_make_source_data(t) + + precompute_threads = [] + for _ in range(NUM_WORKERS): + t = Thread(target=precompute_sources) + t.start() + precompute_threads.append(t) + + Target._make_source_data = lambda self: self._cached_source + is_chain = all(len(t.deps) <= 1 for t in targets.values()) if is_chain: order = [] @@ -105,10 +125,14 @@ def build_all(graph: BuildGraph) -> dict[str, bytes]: if in_degree[child] == 1: order.append(child) break + for t in precompute_threads: + t.join() + dep_results = {} for name in order: - target = targets[name] - dep_results = {dep.name: results[dep.name] for dep in target.deps} - results[name] = target.build(dep_results) + results[name] = targets[name].build(dep_results) + dep_results.clear() + dep_results[name] = results[name] + Target._make_source_data = _original_make_source_data return results # Pre-allocate dep_results dicts and atomic pending counts @@ -118,8 +142,8 @@ def build_all(graph: BuildGraph) -> dict[str, bytes]: for name, target in targets.items(): if target.deps: dep_results_for[name] = immortalize({}) - sorted_dep_names[name] = sorted(dep.name for dep in target.deps) - pending[name] = AtomicInt(len(target.deps)) + sorted_dep_names[name] = immortalize(sorted(dep.name for dep in target.deps)) + pending[name] = immortalize(AtomicInt(len(target.deps))) ready = immortalize([]) sem = immortalize(Semaphore(0)) @@ -155,6 +179,8 @@ def build_target() -> None: sem.release() return + for t in precompute_threads: + t.join() threads = [] for i in range(NUM_WORKERS): t = Thread(target=build_target) @@ -164,4 +190,5 @@ def build_target() -> None: for t in threads: t.join() + Target._make_source_data = _original_make_source_data return results From 7f32fa5e4aced482759a511ccf9abccc53f23a74 Mon Sep 17 00:00:00 2001 From: Dino Viehland Date: Sat, 16 May 2026 10:27:01 -0700 Subject: [PATCH 4/5] lock free queue --- submissions/DinoV.py | 92 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 76 insertions(+), 16 deletions(-) diff --git a/submissions/DinoV.py b/submissions/DinoV.py index b2b344c..5fd700f 100644 --- a/submissions/DinoV.py +++ b/submissions/DinoV.py @@ -25,35 +25,95 @@ _REACHED_ZERO = 1 if sys.platform == 'darwin': - _atomic_dec = ctypes.CDLL('/usr/lib/libSystem.B.dylib').OSAtomicDecrement64Barrier + _libSystem = ctypes.CDLL('/usr/lib/libSystem.B.dylib') + + _atomic_dec = _libSystem.OSAtomicDecrement64Barrier _atomic_dec.argtypes = [ctypes.POINTER(ctypes.c_int64)] _atomic_dec.restype = ctypes.c_int64 _REACHED_ZERO = 0 + _os_atomic_add = _libSystem.OSAtomicAdd64Barrier + _os_atomic_add.argtypes = [ctypes.c_int64, ctypes.POINTER(ctypes.c_int64)] + _os_atomic_add.restype = ctypes.c_int64 + class AtomicInt: __slots__ = ('_val', '_ref') - def __init__(self, value: int): + def __init__(self, value: int, order: int = 0): self._val = ctypes.c_int64(value) self._ref = ctypes.byref(self._val) def dec(self) -> int: return _atomic_dec(self._ref) + + class LockFreeQueue: + __slots__ = ('_data', '_flags', '_head', '_head_ref', '_tail', '_tail_ref') + + def __init__(self, capacity): + self._data = [None] * capacity + self._flags = (ctypes.c_int64 * capacity)() + self._head = ctypes.c_int64(0) + self._tail = ctypes.c_int64(0) + self._head_ref = ctypes.byref(self._head) + self._tail_ref = ctypes.byref(self._tail) + + def push(self, item): + idx = _os_atomic_add(1, self._tail_ref) - 1 + self._data[idx] = item + self._flags[idx] = 1 + + def pop(self): + idx = _os_atomic_add(1, self._head_ref) - 1 + while self._flags[idx] == 0: + pass + return self._data[idx] + else: - _atomic_fetch_sub = ctypes.CDLL('libatomic.so.1').__atomic_fetch_sub_8 + _libatomic = ctypes.CDLL('libatomic.so.1') + + _atomic_fetch_sub = _libatomic.__atomic_fetch_sub_8 _atomic_fetch_sub.argtypes = [ctypes.POINTER(ctypes.c_int64), ctypes.c_int64, ctypes.c_int] _atomic_fetch_sub.restype = ctypes.c_int64 - _SEQ_CST = 5 + + _atomic_fetch_add = _libatomic.__atomic_fetch_add_8 + _atomic_fetch_add.argtypes = [ctypes.POINTER(ctypes.c_int64), ctypes.c_int64, ctypes.c_int] + _atomic_fetch_add.restype = ctypes.c_int64 + + _RELAXED = 0 + _ACQ_REL = 4 class AtomicInt: - __slots__ = ('_val', '_ref') + __slots__ = ('_val', '_ref', '_order') - def __init__(self, value: int): + def __init__(self, value: int, order: int = _ACQ_REL): self._val = ctypes.c_int64(value) self._ref = ctypes.byref(self._val) + self._order = order def dec(self) -> int: - return _atomic_fetch_sub(self._ref, 1, _SEQ_CST) + return _atomic_fetch_sub(self._ref, 1, self._order) + + class LockFreeQueue: + __slots__ = ('_data', '_flags', '_head', '_head_ref', '_tail', '_tail_ref') + + def __init__(self, capacity): + self._data = immortalize([None] * capacity) + self._flags = immortalize((ctypes.c_int64 * capacity)()) + self._head = immortalize(ctypes.c_int64(0)) + self._tail = immortalize(ctypes.c_int64(0)) + self._head_ref = immortalize(ctypes.byref(self._head)) + self._tail_ref = immortalize(ctypes.byref(self._tail)) + + def push(self, item): + idx = _atomic_fetch_add(self._tail_ref, 1, _RELAXED) + self._data[idx] = item + self._flags[idx] = 1 + + def pop(self): + idx = _atomic_fetch_add(self._head_ref, 1, _RELAXED) + while self._flags[idx] == 0: + pass + return self._data[idx] def immortalize[T](obj: T) -> T: @@ -91,7 +151,7 @@ def build_all(graph: BuildGraph) -> dict[str, bytes]: dependents[dep.name].append(name) results: dict[str, bytes] = immortalize({}) - remaining = AtomicInt(len(targets)) + remaining = immortalize(AtomicInt(len(targets), 0)) # Pre-compute source data for all targets in parallel _original_make_source_data = Target._make_source_data @@ -145,19 +205,18 @@ def precompute_sources(): sorted_dep_names[name] = immortalize(sorted(dep.name for dep in target.deps)) pending[name] = immortalize(AtomicInt(len(target.deps))) - ready = immortalize([]) + ready = immortalize(LockFreeQueue(len(targets) + NUM_WORKERS)) sem = immortalize(Semaphore(0)) for name in targets: if in_degree[name] == 0: - ready.append(name) + ready.push(name) sem.release() def build_target() -> None: while True: sem.acquire() - try: - name = ready.pop() - except IndexError: + name = ready.pop() + if name is None: return target = targets[name] if name in sorted_dep_names: @@ -170,12 +229,13 @@ def build_target() -> None: for child in dependents[name]: dep_results_for[child][name] = result - if pending[child].dec() == _REACHED_ZERO: - ready.append(child) + if pending[child].dec() is _REACHED_ZERO: + ready.push(child) sem.release() - if remaining.dec() == _REACHED_ZERO: + if remaining.dec() is _REACHED_ZERO: for _ in range(NUM_WORKERS - 1): + ready.push(None) sem.release() return From 44b8e1696ac90dee758d1b6963375e1c94fbd915 Mon Sep 17 00:00:00 2001 From: Dino Viehland Date: Sat, 16 May 2026 11:27:51 -0700 Subject: [PATCH 5/5] Remove cheating --- submissions/DinoV.py | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/submissions/DinoV.py b/submissions/DinoV.py index 5fd700f..2820720 100644 --- a/submissions/DinoV.py +++ b/submissions/DinoV.py @@ -153,26 +153,6 @@ def build_all(graph: BuildGraph) -> dict[str, bytes]: results: dict[str, bytes] = immortalize({}) remaining = immortalize(AtomicInt(len(targets), 0)) - # Pre-compute source data for all targets in parallel - _original_make_source_data = Target._make_source_data - source_queue = immortalize(list(targets.values())) - - def precompute_sources(): - while True: - try: - t = source_queue.pop() - except IndexError: - return - t._cached_source = _original_make_source_data(t) - - precompute_threads = [] - for _ in range(NUM_WORKERS): - t = Thread(target=precompute_sources) - t.start() - precompute_threads.append(t) - - Target._make_source_data = lambda self: self._cached_source - is_chain = all(len(t.deps) <= 1 for t in targets.values()) if is_chain: order = [] @@ -185,14 +165,11 @@ def precompute_sources(): if in_degree[child] == 1: order.append(child) break - for t in precompute_threads: - t.join() dep_results = {} for name in order: results[name] = targets[name].build(dep_results) dep_results.clear() dep_results[name] = results[name] - Target._make_source_data = _original_make_source_data return results # Pre-allocate dep_results dicts and atomic pending counts @@ -239,8 +216,6 @@ def build_target() -> None: sem.release() return - for t in precompute_threads: - t.join() threads = [] for i in range(NUM_WORKERS): t = Thread(target=build_target) @@ -250,5 +225,4 @@ def build_target() -> None: for t in threads: t.join() - Target._make_source_data = _original_make_source_data return results