diff --git a/src/realm/inst_impl.cc b/src/realm/inst_impl.cc index d3469ff3c9..30aa876662 100644 --- a/src/realm/inst_impl.cc +++ b/src/realm/inst_impl.cc @@ -38,11 +38,14 @@ namespace Realm { void RegionInstanceImpl::DeferredCreate::defer(RegionInstanceImpl *_inst, MemoryImpl *_mem, - bool _need_alloc_result, Event wait_on) + bool _need_alloc_result, + unsigned _release_seqid_cap, + Event wait_on) { inst = _inst; mem = _mem; need_alloc_result = _need_alloc_result; + release_seqid_cap = _release_seqid_cap; EventImpl::add_waiter(wait_on, this); } diff --git a/src/realm/inst_impl.h b/src/realm/inst_impl.h index b1f41c9f3c..59e855fd24 100644 --- a/src/realm/inst_impl.h +++ b/src/realm/inst_impl.h @@ -61,15 +61,28 @@ namespace Realm { class DeferredCreate : public EventWaiter { public: void defer(RegionInstanceImpl *_inst, MemoryImpl *_mem, bool _need_alloc_result, - Event wait_on); + unsigned _release_seqid_cap, Event wait_on); virtual void event_triggered(bool poisoned, TimeLimit work_until); virtual void print(std::ostream &os) const; virtual Event get_finish_event(void) const; + // snapshot of the memory's pending-release sequence id taken when the + // deferred creation was requested - the eventual allocation attempt + // is only funded by releases with seqids no newer than this, since a + // release requested afterwards may depend on our creation event and + // funding from it can deadlock. NOTE: this ordering argument is only + // airtight for creations issued on the memory's owner node - a remote + // creation publishes its creation event at the creator before the + // owner takes this snapshot, so a release requested in that window + // can slip under the cap (known gap, see + // tla/allocation/FUTURE-VERIFICATION.md "multi-node create ordering") + unsigned get_release_seqid_cap(void) const { return release_seqid_cap; } + protected: RegionInstanceImpl *inst; MemoryImpl *mem; bool need_alloc_result; + unsigned release_seqid_cap = 0; }; DeferredCreate deferred_create; diff --git a/src/realm/mem_impl.cc b/src/realm/mem_impl.cc index 4707f2c0aa..831daa85ce 100644 --- a/src/realm/mem_impl.cc +++ b/src/realm/mem_impl.cc @@ -162,9 +162,12 @@ namespace Realm { inst, need_alloc_result, false /*!alloc_poisoned*/, TimeLimit::responsive()); } } else { - // defer allocation attempt + // defer allocation attempt (the release seqid cap is only meaningful + // for memories that support deferred allocation - see + // LocalManagedMemory::allocate_storage_deferrable) inst->metadata.inst_offset = RegionInstanceImpl::INSTOFFSET_DELAYEDALLOC; - inst->deferred_create.defer(inst, this, need_alloc_result, precondition); + inst->deferred_create.defer(inst, this, need_alloc_result, + 0 /*release_seqid_cap - unused*/, precondition); result = ALLOC_DEFERRED /*asynchronous notification*/; } @@ -710,9 +713,20 @@ namespace Realm { // attempt allocation below } } else { - // defer allocation attempt + // defer allocation attempt - snapshot the release seqid so the + // eventual attempt only considers funding from deletions requested + // before this creation: once we hand out the creation event below, + // any newly requested deletion may (transitively) depend on it, and + // planning the allocation out of such a deletion's space would + // deadlock + unsigned release_seqid_cap; + { + AutoLock<> al(allocator_mutex); + release_seqid_cap = cur_release_seqid; + } inst->metadata.inst_offset = RegionInstanceImpl::INSTOFFSET_DELAYEDALLOC; - inst->deferred_create.defer(inst, this, need_alloc_result, precondition); + inst->deferred_create.defer(inst, this, need_alloc_result, release_seqid_cap, + precondition); return ALLOC_DEFERRED /*asynchronous notification*/; } @@ -731,9 +745,11 @@ namespace Realm { // normal allocation from our managed pool AutoLock<> al(allocator_mutex); + // the precondition was already triggered at request time, so every + // pending release predates this request and may fund it result = attempt_deferrable_allocation(inst, inst->metadata.layout->bytes_used, inst->metadata.layout->alignment_reqd, - inst_offset); + inst_offset, cur_release_seqid); } // if we needed an alloc result, send deferred responses too @@ -747,63 +763,123 @@ namespace Realm { // for internal use by allocation routines - must be called with // allocator_mutex held! MemoryImpl::AllocationResult LocalManagedMemory::attempt_deferrable_allocation( - RegionInstanceImpl *inst, size_t bytes, size_t alignment, size_t &inst_offset) + RegionInstanceImpl *inst, size_t bytes, size_t alignment, size_t &inst_offset, + unsigned release_seqid_cap) { +#ifdef DEBUG_REALM + // ready releases are swept whenever the allocation queue drains, so an + // empty queue implies nothing in the release list is ready + if(pending_allocs.empty()) + for(std::deque::const_iterator it = pending_releases.begin(); + it != pending_releases.end(); ++it) + assert(!it->is_ready); +#endif + // as long as there aren't any pending allocations, we can attempt to // satisfy the allocation based on the current state if(pending_allocs.empty()) { bool ok = current_allocator.allocate(inst->me, bytes, alignment, inst_offset); - if(ok) { + if(ok) return ALLOC_INSTANT_SUCCESS; - } else { - // doesn't currently fit - are there any pending deletes that - // might allow it to fit in the future? - // also, check that deferred allocations are even permitted - if(pending_releases.empty() || !Config::deferred_instance_allocation) { - // nope - this allocation can't succeed based on what we know - // right now - return ALLOC_INSTANT_FAILURE; - } else { - // build the future state based on those deletes and try again - future_allocator = current_allocator; - for(std::deque::iterator it = pending_releases.begin(); - it != pending_releases.end(); ++it) { - // shouldn't have any ready ones here - assert(!it->is_ready); - // due to network delays, it's possible for multiple - // deallocations of the same instance to be in our list, - // so ignore failures to deallocate from the future state - // (this is conservative, because it can only cause a - // false-failure of a future allocation) - it->release(future_allocator, true /*missing ok*/); - } - bool ok = future_allocator.allocate(inst->me, bytes, alignment, inst_offset); - if(ok) { - pending_allocs.emplace_back( - PendingAlloc(inst, bytes, alignment, cur_release_seqid)); - // now that we have pending allocs, we need release_allocator - // to be valid - release_allocator = current_allocator; - return ALLOC_DEFERRED; - } else { - return ALLOC_INSTANT_FAILURE; /*immediate notification*/ - // NOTE: future_allocator becomes invalid - } - } + // doesn't currently fit - are there any pending deletes that + // might allow it to fit in the future? + // also, check that deferred allocations are even permitted + if(pending_releases.empty() || !Config::deferred_instance_allocation) { + // nope - this allocation can't succeed based on what we know + // right now + return ALLOC_INSTANT_FAILURE; } } else { - // with other pending allocs, we can only tentatively allocate based - // on future state - bool ok = future_allocator.allocate(inst->me, bytes, alignment, inst_offset); - if(ok) { - pending_allocs.emplace_back( - PendingAlloc(inst, bytes, alignment, cur_release_seqid)); - return ALLOC_DEFERRED; - } else { - return ALLOC_INSTANT_FAILURE; /*immediate notification*/ + // newer allocations may not be funded by older releases than any + // already-queued allocation - this keeps last_release_seqid + // non-decreasing along 'pending_allocs', which the drain and replay + // logic depend on (and an inversion between two deferred creations + // is exactly the shape that would rebuild a funding cycle) + if(release_seqid_cap < pending_allocs.back().last_release_seqid) + return ALLOC_INSTANT_FAILURE; + + // fast path: a request whose precondition had already triggered + // (cap == cur_release_seqid) can be tested directly against the + // maintained future state - future_allocator is kept in canonical + // (seqid watermark) order and a fresh allocation's watermark + // position is its end, so the direct test is equivalent to the + // canonical replay below (see tla/allocation/bugs/BLUEPRINT-REVIEW.md + // section 1.8) + if(release_seqid_cap == cur_release_seqid) { + bool ok = future_allocator.allocate(inst->me, bytes, alignment, inst_offset); + if(ok) { + pending_allocs.emplace_back( + PendingAlloc(inst, bytes, alignment, release_seqid_cap)); + return ALLOC_DEFERRED; + } else { + return ALLOC_INSTANT_FAILURE; /*immediate notification*/ + } } } + + // test whether the releases requested no later than our snapshot (for + // owner-node creations, exactly the ones that cannot depend on our + // creation event - remote creations have a known snapshot-window gap, + // see tla/allocation/FUTURE-VERIFICATION.md "multi-node create + // ordering"), together with the already-queued allocations, make room + // for this request: replay them onto a scratch copy of the current + // state in canonical (seqid watermark) order + RangeAllocator test_allocator = current_allocator; + std::deque::iterator rel_it = pending_releases.begin(); + for(std::deque::iterator a_it = pending_allocs.begin(); + a_it != pending_allocs.end(); ++a_it) { + // releases up to this allocation's watermark are applied first + // (watermarks are <= our cap by the monotonicity check above) + while((rel_it != pending_releases.end()) && + (rel_it->seqid <= a_it->last_release_seqid)) { + // due to network delays, it's possible for multiple deallocations + // of the same instance to be in our list, so ignore failures to + // deallocate from the replayed state (this is conservative, + // because it can only cause a false-failure of the test) + rel_it->release(test_allocator, true /*missing ok*/); + ++rel_it; + } + size_t offset = 0; + bool placed = + test_allocator.allocate(a_it->inst->me, a_it->bytes, a_it->alignment, offset); + // every queued allocation was admitted against exactly this replayed + // state, and completions apply the same operations in the same + // order, so the placement must succeed + assert(placed); + if(!placed) + return ALLOC_INSTANT_FAILURE; /*conservative in release builds*/ + } + // remaining releases within our own funding bound + while((rel_it != pending_releases.end()) && (rel_it->seqid <= release_seqid_cap)) { + rel_it->release(test_allocator, true /*missing ok*/); + ++rel_it; + } + + if(!test_allocator.allocate(inst->me, bytes, alignment, inst_offset)) { + // even the releases we may legally wait for don't make room - + // fail immediately (and honestly) rather than risk planning a + // funding cycle + // NOTE: future_allocator is deliberately left untouched here + return ALLOC_INSTANT_FAILURE; /*immediate notification*/ + } + + // accepted - queue it up with the cap as its watermark, and extend the + // test state into the new future state: everything past our funding + // bound applies after us in canonical order (no queued allocation can + // follow us with a smaller watermark) + pending_allocs.emplace_back(PendingAlloc(inst, bytes, alignment, release_seqid_cap)); + while(rel_it != pending_releases.end()) { + rel_it->release(test_allocator, true /*missing ok*/); + ++rel_it; + } + future_allocator.swap(test_allocator); + if(pending_allocs.size() == 1) { + // now that we have pending allocs, we need release_allocator + // to be valid + release_allocator = current_allocator; + } + return ALLOC_DEFERRED; } // release storage associated with an instance @@ -1133,9 +1209,13 @@ namespace Realm { result = ALLOC_INSTANT_FAILURE; } } else { - result = attempt_deferrable_allocation(inst, inst->metadata.layout->bytes_used, - inst->metadata.layout->alignment_reqd, - inst_offset); + // fund only from releases that were already requested when this + // creation was requested (the snapshot taken at deferral time) - + // releases requested in between may depend on our creation event + result = attempt_deferrable_allocation( + inst, inst->metadata.layout->bytes_used, + inst->metadata.layout->alignment_reqd, inst_offset, + inst->deferred_create.get_release_seqid_cap()); } } @@ -1481,6 +1561,13 @@ namespace Realm { } } + // ready releases must never outlive the allocation queue - if it has + // drained (on either the poisoned or unpoisoned path above), apply + // and retire any that remain + if(pending_allocs.empty()) { + sweep_ready_releases(deferred_dealloc_notifies); + } + #ifdef DEBUG_DEFERRED_ALLOCATIONS log_defalloc.print() << "deferred redistrict done: m=" << me << " inst=" << old_inst->me @@ -1593,6 +1680,74 @@ namespace Realm { } } } + + // any allocations whose watermark exceeds every surviving release's + // seqid were not reconsidered by the loop above - continue the + // replay from where it stopped (NOT from the beginning: earlier + // entries are already placed in the future state, and re-placing + // one would double-allocate its tag) so that every remaining + // allocation is either re-placed in the rebuilt future state or + // failed + while(it2 != pending_allocs.end()) { + size_t offset; + bool ok = + future_allocator.allocate(it2->inst->me, it2->bytes, it2->alignment, offset); + if(ok) { + ++it2; + } else { + // this should only happen if we've seen the poisoned release + assert(found); + + failed_allocs.push_back(it2->inst); + it2 = pending_allocs.erase(it2); + } + } + + if(!pending_allocs.empty()) { + // release_allocator was rebuilt from the current state above - + // restore the ready releases that survive the removal + // (release state = current state + ready releases) + for(std::deque::iterator it3 = pending_releases.begin(); + it3 != pending_releases.end(); ++it3) { + if(it3->is_ready) { + it3->release(release_allocator); + } + } + } + // if the replay failed the last pending allocation(s), the caller's + // sweep of ready releases will run once we return (see + // sweep_ready_releases call sites) + } + } + + // applies (in list order) and erases any ready pending releases - called + // whenever 'pending_allocs' has drained. ready entries only ever come + // into existence while allocations are queued (their application to the + // current state was deferred to keep the planned future consistent), so + // once the queue drains they must not linger: their tags are still in + // current_allocator and their dealloc notifications are still deferred + void LocalManagedMemory::sweep_ready_releases( + std::vector &deferred_dealloc_notifies) + { + assert(pending_allocs.empty()); + std::deque::iterator it = pending_releases.begin(); + while(it != pending_releases.end()) { + if(it->is_ready) { + // a redistrict entry carves its children out of the parent's range + // here - the children were already notified of their offsets when + // the entry became ready, and those offsets provably match this + // split: split_range carves children sequentially from the parent + // range's own start, and allocated ranges never move, so the + // parent interval (and hence every child offset) is identical in + // the promise-time future/release state and the current state + it->release(current_allocator); + if(it->deferred_dealloc_notify) { + deferred_dealloc_notifies.push_back(it->inst); + } + it = pending_releases.erase(it); + } else { + ++it; + } } } @@ -1755,6 +1910,13 @@ namespace Realm { remove_pending_release(inst, failed_allocs); } + // ready releases must never outlive the allocation queue - if it has + // drained (on either the poisoned or unpoisoned path above), apply + // and retire any that remain + if(pending_allocs.empty()) { + sweep_ready_releases(deferred_dealloc_notifies); + } + #ifdef DEBUG_DEFERRED_ALLOCATIONS log_defalloc.print() << "deferred destruction done: m=" << me << " inst=" << inst->me diff --git a/src/realm/mem_impl.h b/src/realm/mem_impl.h index 2195778161..4c5a97a81f 100644 --- a/src/realm/mem_impl.h +++ b/src/realm/mem_impl.h @@ -376,8 +376,25 @@ namespace Realm { protected: // for internal use by allocation routines - must be called with // allocator_mutex held! + // 'release_seqid_cap' bounds which pending releases may fund a deferred + // admission: only releases with seqid <= cap (i.e. requested no later + // than the creation request) are considered - a release requested after + // the creation request may depend on the instance's creation event, and + // planning the allocation out of its space can deadlock AllocationResult attempt_deferrable_allocation(RegionInstanceImpl *inst, size_t bytes, - size_t alignment, size_t &inst_offset); + size_t alignment, size_t &inst_offset, + unsigned release_seqid_cap); + + // applies (in list order) and erases any ready pending releases - called + // whenever 'pending_allocs' has drained so that ready releases (and the + // deferred dealloc notifications they hold) never outlive the + // allocations they were queued behind. Erased entries with + // deferred_dealloc_notify set are appended to + // 'deferred_dealloc_notifies' so the caller can fire their + // notify_deallocation() once the allocator_mutex has been released. + // must be called with allocator_mutex held! + void + sweep_ready_releases(std::vector &deferred_dealloc_notifies); // attempts to satisfy pending allocations based on reordering releases to // move the ready ones first - assumes 'release_allocator' has been @@ -400,9 +417,17 @@ namespace Realm { // current: always valid - tracks all completed allocations and all // releases that can be applied without risking deadlock // future: valid if pending_allocs exist - tracks heap state including - // all pending allocs and releases + // all pending allocs and releases, applied in canonical + // (seqid watermark) order: for each release in seqid + // order, the release is applied and then any pending + // allocs whose last_release_seqid equals that seqid are + // placed // release: valid if pending_allocs exist - models heap state with // completed allocations and any ready releases + // + // invariant: ready entries never outlive the allocation queue - whenever + // 'pending_allocs' drains, any ready 'pending_releases' entries are + // swept into current_allocator (see sweep_ready_releases) // Pick which kind of range allocator we want to use using RangeAllocator = BasicRangeAllocator; @@ -413,6 +438,11 @@ namespace Realm { struct PendingAlloc { RegionInstanceImpl *inst; size_t bytes, alignment; + // the newest pending release this allocation may be funded by: the + // release seqid snapshot taken when the creation was REQUESTED (not + // when its precondition triggered) - releases requested later may + // depend on this instance's creation event. non-decreasing along + // 'pending_allocs' (enforced at admission) unsigned last_release_seqid; PendingAlloc(RegionInstanceImpl *_inst, size_t _bytes, size_t _align, unsigned _release_seqid); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index f451e976b7..62b21c9856 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -378,6 +378,8 @@ add_integration_test(rsrv_acquire_poisoned "${REALM_TEST_DIR}/rsrv_acquire_poiso add_integration_test(refcount_image_test "${REALM_TEST_DIR}/refcount_image_test.cc") set(inst_chain_redistrict_ARGS -i 2) add_integration_test(inst_chain_redistrict "${REALM_TEST_DIR}/inst_chain_redistrict.cc") +set(deferred_allocs_ARGS -ll:csize 16) +add_integration_test(deferred_allocs "${REALM_TEST_DIR}/deferred_allocs.cc") add_integration_test(refcount_preimage_test "${REALM_TEST_DIR}/refcount_preimage_test.cc") add_integration_test( test_profiling "${REALM_TEST_DIR}/test_profiling.cc" diff --git a/tests/deferred_allocs.cc b/tests/deferred_allocs.cc index 2293a06837..b2ce01339f 100644 --- a/tests/deferred_allocs.cc +++ b/tests/deferred_allocs.cc @@ -63,6 +63,7 @@ struct TestConfig { int buckets_max; bool all_memories; bool check_alloc_result; + int fix_tests; // bitmask selecting deferred-alloc fix regression tests }; struct InstanceInfo { @@ -70,6 +71,9 @@ struct InstanceInfo { Event create_event; bool alloc_result; UserEvent destroy_event; + UserEvent create_precond; + UserEvent alloc_result_event; + bool alloc_result_pending; enum State { ALLOC_PENDING, @@ -85,6 +89,9 @@ struct InstanceInfo { , create_event(_create_event) , alloc_result(_alloc_result) , destroy_event(UserEvent::NO_USER_EVENT) + , create_precond(UserEvent::NO_USER_EVENT) + , alloc_result_event(UserEvent::NO_USER_EVENT) + , alloc_result_pending(false) , state(ALLOC_PENDING) {} }; @@ -180,8 +187,6 @@ void directed_test_memory(const TestConfig &config, Memory m, Processor p, InstanceInfo &ii = insts[amt]; assert(ii.state == InstanceInfo::ALLOC_PENDING); log_app.debug() << "success #" << amt << " inst=" << ii.inst; - if(config.check_alloc_result) - assert(ii.alloc_result); bool poisoned = false; // normal apps should not call external_wait, but we do it here to // detect hangs more easily and we know nothing else wants to run @@ -199,7 +204,22 @@ void directed_test_memory(const TestConfig &config, Memory m, Processor p, << " test=" << name << " (" << testdesc << ")"; abort(); } - ii.state = InstanceInfo::ALLOCED; + // a create with a deferred precondition gets its alloc result no + // earlier than the precondition trigger - resolve it now + if(ii.alloc_result_pending) { + alarm(10); + bool rpoisoned = false; + ii.alloc_result_event.wait_faultaware(rpoisoned); + alarm(0); + ii.alloc_result = !rpoisoned; + ii.alloc_result_pending = false; + } + if(config.check_alloc_result) + assert(ii.alloc_result); + // a destroy may already have been requested while the create was + // still pending + ii.state = (ii.destroy_event.exists() ? InstanceInfo::DEST_PENDING + : InstanceInfo::ALLOCED); break; } @@ -209,8 +229,6 @@ void directed_test_memory(const TestConfig &config, Memory m, Processor p, InstanceInfo &ii = insts[amt]; assert(ii.state == InstanceInfo::ALLOC_PENDING); log_app.debug() << "failed #" << amt << " inst=" << ii.inst; - if(config.check_alloc_result) - assert(ii.alloc_result == (cmd == 'u')); bool poisoned = false; // normal apps should not call external_wait, but we do it here to // detect hangs more easily and we know nothing else wants to run @@ -228,6 +246,18 @@ void directed_test_memory(const TestConfig &config, Memory m, Processor p, << " test=" << name << " (" << testdesc << ")"; abort(); } + // a create with a deferred precondition gets its alloc result no + // earlier than the precondition trigger - resolve it now + if(ii.alloc_result_pending) { + alarm(10); + bool rpoisoned = false; + ii.alloc_result_event.wait_faultaware(rpoisoned); + alarm(0); + ii.alloc_result = !rpoisoned; + ii.alloc_result_pending = false; + } + if(config.check_alloc_result) + assert(ii.alloc_result == (cmd == 'u')); ii.state = InstanceInfo::ALLOC_FAILED; ii.inst.destroy(); break; @@ -254,15 +284,93 @@ void directed_test_memory(const TestConfig &config, Memory m, Processor p, case 'd': // destroy { InstanceInfo &ii = insts[amt]; - assert(ii.state == InstanceInfo::ALLOCED); + // a deferred destroy may legally be requested while the creation is + // still pending - it stays ALLOC_PENDING until the creation outcome + // is observed ('s' then moves it to DEST_PENDING) + assert((ii.state == InstanceInfo::ALLOCED) || + ((ii.state == InstanceInfo::ALLOC_PENDING) && !ii.destroy_event.exists())); ii.destroy_event = UserEvent::create_user_event(); log_app.debug() << "destroy #" << amt << " inst=" << ii.inst << " event=" << ii.destroy_event; ii.inst.destroy(ii.destroy_event); + if(ii.state == InstanceInfo::ALLOCED) + ii.state = InstanceInfo::DEST_PENDING; + break; + } + + case 'p': // allocate with an untriggered user-event precondition + { + size_t idx = insts.size(); + Rect<1> rect(1, amt * bucket_size); + + // we need a profiling request set that ignores failures + ProfilingRequestSet prs; + prs.add_request(Processor::NO_PROC, 0 /*ignore*/) + .add_measurement(); + UserEvent alloc_result_event = UserEvent::NO_USER_EVENT; + if(config.check_alloc_result) { + alloc_result_event = UserEvent::create_user_event(); + prs.add_request(p, ALLOC_RESULT_TASK, &alloc_result_event, + sizeof(alloc_result_event)) + .add_measurement(); + } + + UserEvent precond = UserEvent::create_user_event(); + RegionInstance inst; + Event e = RegionInstance::create_instance(inst, m, rect, field_sizes, 0 /*SOA*/, + prs, precond); + + // the alloc result cannot arrive before the precondition triggers, so + // defer checking it until the outcome ('s'/'f'/'u') is tested + InstanceInfo ii(inst, e, true /*tbd*/); + ii.create_precond = precond; + ii.alloc_result_event = alloc_result_event; + ii.alloc_result_pending = config.check_alloc_result; + insts.push_back(ii); + log_app.debug() << "alloc #" << idx << ": size=" << amt << " inst=" << inst + << " ready=" << e << " precond=" << precond; + + break; + } + + case 'g': // trigger a pending creation's precondition + { + InstanceInfo &ii = insts[amt]; + assert(ii.state == InstanceInfo::ALLOC_PENDING); + assert(ii.create_precond.exists()); + log_app.debug() << "go #" << amt << " inst=" << ii.inst + << " precond=" << ii.create_precond; + ii.create_precond.trigger(); + break; + } + + case 'D': // destroy, preconditioned on another instance's creation event + { + assert(*pos == ','); + pos++; + int amt2 = strtol(pos, (char **)&pos, 10); + InstanceInfo &ii = insts[amt]; + assert(ii.state == InstanceInfo::ALLOCED); + log_app.debug() << "destroy #" << amt << " inst=" << ii.inst << " precond=create(#" + << amt2 << ")=" << insts[amt2].create_event; + ii.destroy_event = UserEvent::NO_USER_EVENT; + ii.inst.destroy(insts[amt2].create_event); ii.state = InstanceInfo::DEST_PENDING; break; } + case 'z': // a cross-preconditioned destroy was dropped (its precondition + // poisoned) - the instance is still alive, so fix up the + // test's bookkeeping to match + { + InstanceInfo &ii = insts[amt]; + assert(ii.state == InstanceInfo::DEST_PENDING); + assert(!ii.destroy_event.exists()); + log_app.debug() << "dropped destroy #" << amt << " inst=" << ii.inst; + ii.state = InstanceInfo::ALLOCED; + break; + } + case 'i': // destroy (instant) { InstanceInfo &ii = insts[amt]; @@ -322,7 +430,10 @@ void directed_test_memory(const TestConfig &config, Memory m, Processor p, case InstanceInfo::DEST_PENDING: { - it->destroy_event.trigger(); + // cross-preconditioned destroys ('D') have no user event to trigger - + // tests must resolve those themselves before finishing + if(it->destroy_event.exists()) + it->destroy_event.trigger(); break; } @@ -604,6 +715,57 @@ void top_level_task(const void *args, size_t arglen, const void *userdata, size_ " i0 s4" // 4123 ); + // ---- regression tests for the deferred-allocation fix bundle ---- + // (request-time capped admission, stranded-ready sweep, trailing + // replay in remove_pending_release) + + // #1 fills memory; #1 is created on an untriggered user event, and + // #0's destruction legally depends on #1's creation (the + // copy-then-free migration idiom). Unfixed Realm funds #1 from #0's + // pending release - a release that can only happen after #1 exists - + // and hangs forever ('f1' detects the hang via its bounded wait). + // Fixed Realm caps #1's funding at its request time, so #1 fails + // honestly, the dependent destroy is dropped (poisoned), and #0 + // remains usable. + if((config.fix_tests & 1) != 0) + directed_test_memory(config, m, p, "capped admission vs migration cycle", + "1 a1 s0 p1 D0,1 g1 f1 z0 i0"); + + // GC-ripple pattern: the funding destroy is requested after the + // preconditioned create but is APPLIED before the create's + // precondition triggers, so the capped admission must still succeed + // via the current heap state. The probe alloc/destroy (#2) proves + // #0's release reached the heap before #1's precondition fires. + if((config.fix_tests & 2) != 0) + directed_test_memory(config, m, p, "gc ripple still succeeds", + "1 a1 s0 p1 d0 t0 a1 s2 i2 g1 s1"); + + // Stranded-ready recipe (BUG-6): #0+#1 fill memory; #2 defers against + // #0's pending release; #2's own destroy is requested while pending; + // #1's instant destroy fails release-reordering and is pushed back + // READY; #0's trigger then drains the queue, placing #2 and leaving + // the ready entry stranded behind #2's non-ready one. In unfixed + // DEBUG builds the next deferral-needing create (#3) fires + // assert(!it->is_ready) in the future rebuild; fixed Realm sweeps + // the stranded entry when the pending-alloc queue empties, and #3 + // proceeds normally. All ordering here is enforced by user events + // and bounded waits, so the interleaving is deterministic. + if((config.fix_tests & 4) != 0) + directed_test_memory(config, m, p, "stranded ready release sweep", + "3 a2 s0 a1 s1 d0 a2 d2 i1 t0 s2 a2 n3 t2 s3"); + + // Trigger-inversion (monotone-cap guard + trailing replay): two + // preconditioned creates whose user events fire in inverted order. + // #2 (requested after #0's dependent destroy) legally defers against + // it; #1 (requested before) then triggers with an older cap and must + // fail instantly - unfixed Realm admits it and deadlocks ('f1' + // catches the hang). The poisoned cascade then removes #0's + // destroy, and the trailing replay must fail #2 cleanly ('u2') + // instead of stranding it forever. + if((config.fix_tests & 8) != 0) + directed_test_memory(config, m, p, "monotone cap guard vs inversion", + "3 a3 s0 p1 D0,1 p2 g2 n2 g1 f1 u2 z0 i0"); + #ifdef REALM_REORDER_DEFERRED_ALLOCATIONS directed_test_memory(config, m, p, "out of order success", "3 a1 s0 a2 s1 d0 d1 a2 a1 t1 s3 t0 s2"); @@ -651,6 +813,7 @@ int main(int argc, const char **argv) config.buckets_max = 4; config.all_memories = false; config.check_alloc_result = true; + config.fix_tests = 15; CommandLineParser clp; clp.add_option_int("-seed", config.seed); @@ -660,6 +823,7 @@ int main(int argc, const char **argv) clp.add_option_int("-min", config.buckets_min); clp.add_option_int("-max", config.buckets_max); clp.add_option_bool("-all", config.all_memories); + clp.add_option_int("-fixtests", config.fix_tests); bool ok = clp.parse_command_line(argc, argv); assert(ok); diff --git a/tla/allocation/.gitignore b/tla/allocation/.gitignore new file mode 100644 index 0000000000..4da98f04dd --- /dev/null +++ b/tla/allocation/.gitignore @@ -0,0 +1,3 @@ +states/ +jtmp/ +slurm-*.out diff --git a/tla/allocation/Big.cfg b/tla/allocation/Big.cfg new file mode 100644 index 0000000000..6d50c2da27 --- /dev/null +++ b/tla/allocation/Big.cfg @@ -0,0 +1,44 @@ +\* Big: open hunt at the largest tractable scale. 5 instances, H=6, mixed +\* sizes (2,1,2,1,3), user poison on, full invariant battery. +\* TLC flags: -deadlock (invariant hunt). +\* Expected: same known violations as Safety/Poison4 appear first +\* (comment them out to go deeper); anything NEW here is an +\* unregistered bug candidate - triage via DESIGN.md s8 before +\* assuming spec error. +\* Scale: sapling, > 1h - run via sapling_tlc.sbatch. State space is +\* dominated by the dependency-set branching (2^5 per request) +\* times heap fragmentation shapes. +CONSTANTS + HEAP_SIZE = 6 + INSTANCES = {1, 2, 3, 4, 5} + Size <- SizesBig + USER_POISON = TRUE + C1_ENABLED = TRUE + C2_ENABLED = TRUE + C3_ENABLED = FALSE + CLIENT_MODE = "FREE" + FIX_CAP = FALSE + FIX_SWEEP = FALSE + FIX_RPR = FALSE +INIT MCInit +NEXT MCNext +INVARIANTS + TypeOK + INV_NoOverlap + INV_InBounds + INV_Conservation + INV_CurrentMatchesGround + INV_NoReadyWhenNoPendingAllocs + INV_NoReadyAtRebuild + INV_TriggeredDeallocPresent + INV_StructuralAsserts + INV_NoOrphanTags + INV_InOrderUnblockSucceeds + INV_FutureOffsetConsistency + INV_PoisonReplayOnlyFailsAfterPoint + INV_NoDupAlloc + SAFETY_PromisesKept + PROP_NotifyOnceSafety + INV_QuiescentHeapEmpty +CONSTRAINT + SeqCtrBound diff --git a/tla/allocation/BigFixed.cfg b/tla/allocation/BigFixed.cfg new file mode 100644 index 0000000000..8c65ba973b --- /dev/null +++ b/tla/allocation/BigFixed.cfg @@ -0,0 +1,46 @@ +\* BigFixed: Big's bounds (5 instances, H=6, mixed sizes, USER_POISON) +\* with the full fix bundle on - the largest fixed-model soundness sweep. +\* TLC flags: -deadlock (invariant hunt at scale; see SafetyFixed4.cfg +\* header - deadlock/drain liveness is owned by the local +\* bundle configs, Inversion deadlock-ON included). +\* Expected: FULLY GREEN on the BUG-1/4/5/6 detector families (the +\* three-toggle bundle addresses all four). Anything else is +\* either a fix-design flaw at scale or an unregistered bug +\* candidate - triage via DESIGN.md s8 with the trace before +\* assuming either. +\* Scale: sapling, likely > 24h - submit with -t 48:00:00 and use the +\* checkpoint/-recover flow in sapling_tlc.sbatch. +CONSTANTS + HEAP_SIZE = 6 + INSTANCES = {1, 2, 3, 4, 5} + Size <- SizesBig + USER_POISON = TRUE + C1_ENABLED = TRUE + C2_ENABLED = TRUE + C3_ENABLED = FALSE + CLIENT_MODE = "FREE" + FIX_CAP = TRUE + FIX_SWEEP = TRUE + FIX_RPR = TRUE +INIT MCInit +NEXT MCNext +INVARIANTS + TypeOK + INV_NoOverlap + INV_InBounds + INV_Conservation + INV_CurrentMatchesGround + INV_NoReadyWhenNoPendingAllocs + INV_NoReadyAtRebuild + INV_TriggeredDeallocPresent + INV_StructuralAsserts + INV_NoOrphanTags + INV_InOrderUnblockSucceeds + INV_FutureOffsetConsistency + INV_PoisonReplayOnlyFailsAfterPoint + INV_NoDupAlloc + SAFETY_PromisesKept + PROP_NotifyOnceSafety + INV_QuiescentHeapEmpty +CONSTRAINT + SeqCtrBound diff --git a/tla/allocation/Composite4.cfg b/tla/allocation/Composite4.cfg new file mode 100644 index 0000000000..bf728aedd0 --- /dev/null +++ b/tla/allocation/Composite4.cfg @@ -0,0 +1,79 @@ +\* Composite4: targeted local confirmation of the PREDICTED BUG-6 -> BUG-4 +\* composite (bugs/BUG-6.md item 6): after the BUG-6 stranding, a deferred +\* admission resets rel := cur resurrecting the stranded READY tag; a later +\* request-time-TRIGGERED destroy reaches ARR full-success, swaps the +\* rel-derived state into cur, and erases the ready entry FIRING its +\* deferred notify -> permanent range leak + notify-while-tag-live, the +\* #442 class, with NO poison anywhere. +\* +\* ("4" names BUG-4; the scenario needs FIVE instances - four are provably +\* insufficient: the stranding consumes i1 (drained) and i3 (stranded), the +\* rel resurrection needs a fresh deferred create i4, and the ARR invocation +\* needs a fresh ALLOCATED instance i5, because the only other pending +\* entry (R2, oldest) drains i3's entry correctly via the oldest path.) +\* +\* Intended witness (SCRIPTED_COMPOSITE client, H=4, sizes 2,2,1,1,1; +\* TLC will find this interleaving among the explored orders): +\* 1-3. create(1)@[0,2), create(3)@[2,3), create(5)@[3,4) - heap full. +\* 4. destroy(1) requested: deps={1}+ballistic -> deferred, R1(seq1). +\* 5. create(2) sz2: cur full; fut = cur - R1 -> fits @0 -> DEFERRED +\* (lastSeq=1); rel := cur = {1,3,5}. +\* 6. destroy(2): deps={2} unfired -> deferred, R2(seq2); fut -= 2. +\* 7. destroy(3): deps={3} resolved -> request-time TRIGGERED: +\* rel -= 3, fut -= 3; ARR front gate: sz2 vs 1-cell hole -> FAIL -> +\* R3 pushed READY+defNote (seq3, cc:884-887). +\* 8. ballistic(D1) fires; EnvTriggerDestroy(1): oldest drain frees 1, +\* unblock places 2@[0,2) (lastSeq 1 < seq(R2) 2); pendingAllocs +\* empties; do-while stops at non-ready R2 -> R3 STRANDED READY +\* (BUG-6 state; both cleanup sites skipped). cur = {2,3,5}, full. +\* 9. create(4) sz1: cur full; ADA rebuild walks [R2, R3-ready] (cc:772 +\* ghost readyAtRebuild - invariant deliberately NOT checked here); +\* fut = {5}; 4 fits -> DEFERRED (lastSeq=3); rel := cur = {2,3,5} +\* - the stranded tag 3 is RESURRECTED into rel while R3 stays READY. +\* 10. destroy(5): deps={5} resolved -> request-time TRIGGERED: +\* rel -= 5 (rel = {2,3}, hole [3,4)), fut -= 5; ARR: test := rel; +\* front gate: sz1 fits @3 -> greedy places 4 -> FULL SUCCESS -> +\* cur' := test = {2@[0,2), 3@[2,3), 4@[3,4)}; R3 ERASED, its +\* deferred notify FIRES for instance 3. +\* => tag 3 in cur, notifyCount[3] = 1, no pendingReleases entry: +\* INV_NoOrphanTags VIOLATED (permanent leak of [2,3)). +\* +\* TLC flags: -deadlock (a BUG-1-shape deadlock elsewhere in the order +\* space must not preempt the invariant hunt). +\* Expected: FAIL: INV_NoOrphanTags (composite confirmed). +\* INV_NoReadyWhenNoPendingAllocs / INV_NoReadyAtRebuild are the +\* known BUG-6 markers on the path and are deliberately excluded +\* from the battery. +\* Scale: local; scripted terms keep the order-interleaving space small. +CONSTANTS + HEAP_SIZE = 4 + INSTANCES = {1, 2, 3, 4, 5} + Size <- SizesComposite + USER_POISON = FALSE + C1_ENABLED = TRUE + C2_ENABLED = TRUE + C3_ENABLED = FALSE + CLIENT_MODE = "SCRIPTED_COMPOSITE" + FIX_CAP = FALSE + FIX_SWEEP = FALSE + FIX_RPR = FALSE +INIT MCInit +NEXT MCNext +INVARIANTS + TypeOK + INV_NoOverlap + INV_InBounds + INV_Conservation + INV_CurrentMatchesGround + INV_TriggeredDeallocPresent + INV_InOrderUnblockSucceeds + INV_FutureOffsetConsistency + INV_PoisonReplayOnlyFailsAfterPoint + INV_NoOrphanTags + INV_QuiescentHeapEmpty + PROP_NotifyOnceSafety + INV_NoDupAlloc + SAFETY_PromisesKept + INV_StructuralAsserts +CONSTRAINT + SeqCtrBound diff --git a/tla/allocation/Composite4Fixed.cfg b/tla/allocation/Composite4Fixed.cfg new file mode 100644 index 0000000000..7344b81730 --- /dev/null +++ b/tla/allocation/Composite4Fixed.cfg @@ -0,0 +1,49 @@ +\* Composite4Fixed: the BUG-6 -> BUG-4 composite script (SCRIPTED_COMPOSITE, +\* see Composite4.cfg for the 10-step witness) with the full fix bundle (CAP+SWEEP+RPR) on. +\* FIX_SWEEP removes the stranding at step 8 (the pendingAllocs->empty +\* sweep drains R3), so the rel resurrection at step 9 and the leaking ARR +\* swap at step 10 can never occur; FIX_SWEEP's BUG-4-standalone rel +\* re-apply in remove_pending_release covers the poison route as well. +\* TLC flags: -deadlock (matched to Composite4.cfg). +\* Expected: FULLY GREEN - in particular INV_NoOrphanTags and +\* INV_CurrentMatchesGround (the base config's leak detectors), +\* plus the BUG-6 pair INV_NoReadyWhenNoPendingAllocs / +\* INV_NoReadyAtRebuild, which the base deliberately excluded +\* and this config ADDS. A violation of the BUG-6 pair => +\* FIX_SWEEP design bug; of INV_NoOrphanTags => the composite +\* leak survives the sweep (fix-design bug, high interest). +\* Scale: local; scripted terms keep the interleaving space small. +CONSTANTS + HEAP_SIZE = 4 + INSTANCES = {1, 2, 3, 4, 5} + Size <- SizesComposite + USER_POISON = FALSE + C1_ENABLED = TRUE + C2_ENABLED = TRUE + C3_ENABLED = FALSE + CLIENT_MODE = "SCRIPTED_COMPOSITE" + FIX_CAP = TRUE + FIX_SWEEP = TRUE + FIX_RPR = TRUE +INIT MCInit +NEXT MCNext +INVARIANTS + TypeOK + INV_NoOverlap + INV_InBounds + INV_Conservation + INV_CurrentMatchesGround + INV_NoReadyWhenNoPendingAllocs + INV_NoReadyAtRebuild + INV_TriggeredDeallocPresent + INV_InOrderUnblockSucceeds + INV_FutureOffsetConsistency + INV_PoisonReplayOnlyFailsAfterPoint + INV_NoOrphanTags + INV_QuiescentHeapEmpty + PROP_NotifyOnceSafety + INV_NoDupAlloc + SAFETY_PromisesKept + INV_StructuralAsserts +CONSTRAINT + SeqCtrBound diff --git a/tla/allocation/DESIGN.md b/tla/allocation/DESIGN.md new file mode 100644 index 0000000000..193953f731 --- /dev/null +++ b/tla/allocation/DESIGN.md @@ -0,0 +1,828 @@ +# TLA+ Model Design: Realm Deferred Instance Allocation + +Authoritative C++ → TLA+ mapping for `LocalManagedMemory`'s deferred +allocation/deletion protocol. Phase 2 spec authors implement this document +verbatim. Where this document and the code disagree, the code wins and the +document must be fixed. Where the code and the talk transcript disagree, the +code wins (differences are flagged inline). + +All citations are against the current tree: +- `cc` = `src/realm/mem_impl.cc` +- `h` = `src/realm/mem_impl.h` +- `inl` = `src/realm/mem_impl.inl` +- `ii` = `src/realm/inst_impl.cc` + +--- + +## 1. Scope & abstractions (v1) + +Modeled: one `LocalManagedMemory` instance and the client/event environment +around its five allocator entry points. + +**Atomicity.** Every protocol action corresponds to exactly one +`allocator_mutex`-holding region (`h:398`). All heap-state mutation in +`allocate_storage_deferrable` (cc:732), `attempt_deferrable_allocation` +(precondition: "must be called with allocator_mutex held", cc:747-748), +`release_storage_deferrable` (cc:838), `allocate_storage_immediate` (cc:1096), +`release_storage_immediate` (cc:1619), and `attempt_release_reordering` +(called only under the mutex) happens under one mutex acquisition, so each is +one atomic TLA+ action. Notifications (`notify_allocation` / +`notify_deallocation`) fire after the mutex is dropped; in the model they are +folded into the same action as event-state updates (see §5 for why this is +sound: the only externally visible effects are event triggers/poisons, and +interleaving between mutex-drop and notification only delays those triggers, +which the environment's nondeterministic trigger order already covers). + +**Abstractions:** + +| Aspect | v1 choice | Rationale / code excluded | +|---|---|---| +| Heap | `0..HEAP_SIZE-1`, HEAP_SIZE ≈ 4-8 units | sizes in bytes are irrelevant; fragmentation shape is what matters | +| Sizes | naturals `>= 1` | size-0 instances take the `SENTINEL` path (`inl:413-416`, `inl:183-198`) and never occupy heap; they cannot affect fragmentation or ordering decisions, only tag bookkeeping. Excluded in v1; revisit in v2 only if tag-lifetime bugs are in scope. | +| Alignment | none (alignment = 0 everywhere) | `calculate_offset` (`inl:154-165`) and the carve-a-front-block path (`inl:441-463`) never fire with alignment 0. v2 extension: alignment adds fragmentation modes and is cheap to add to the FirstFit operator. | +| Instances | fixed set `INSTANCES` ≈ 3-5, each with a model-assigned size | bounded state space; instance IDs never reused (ID-reuse interaction with #442 is v2) | +| Redistricting | **out of scope v1** | `reuse_storage_deferrable` (cc:926-1085), `reuse_storage_immediate` (cc:1326-1536), `split_range` (`inl:168-274`), `record_redistrict` (cc:1822-1837). v2. | +| External resources | out | `ext_resource` branches cc:721-729, cc:831-832, cc:1117-1134, cc:1605-1611 | +| Remote/network | out | creator-node forwarding `ii:996-1032`, active messages `h:594-627`. Single node: `assert(target == Network::my_node_id)` (cc:698-699) holds. | +| Duplicate releases | out | the "network delays → multiple deallocations of same instance" tolerance (comment cc:773-777) cannot arise single-node with one `deferred_destroy` slot per instance. v2 (see §9). | +| `Config::deferred_instance_allocation` | TRUE (default, cc:40) | the `FALSE` branch (cc:762) degenerates to a blocking allocator; not interesting | +| Poison | **intrinsic poison always-on; user poison toggled** | Intrinsic failure poison is core Realm behavior, not optional: `notify_allocation` poisons `eCreated(i)` on INSTANT_FAILURE/CANCELLED (`ii:1121-1122`) with no user poison involved, and under C2 that makes `preD[i]` fire poisoned, routing to `remove_pending_release` (cc:1754-1755). INSTANT_FAILURE is reachable in every config, and the DELAYEDDESTROY path (cc:845-849 → cc:1146-1147) queues a release for an instance that may INSTANT_FAIL at create-trigger — with poison omitted that queued destroy could never fire, producing **false deadlocks** on the full-cleanup client. The `USER_POISON` toggle gates only client-poisoned ballistic events (§5). | +| Profiling / `need_alloc_result` | out | affects only which messages are sent, not heap state | +| Dealloc-completion feedback | **out (v2)** | `InstanceStatus`/`InstanceTimeline` profiling responses on destruction (`ii:1248-1262`) let real clients derive user-event triggers from *destruction completion*, creating cycle shapes v1 cannot express. v1 clients observe only `eCreated` results. | +| `deferred_dealloc_notify` | **modeled, simplified** (see §3) | it is protocol-relevant state (drives when `notify_deallocation` fires; recent fix for slot-recycle double-tracking, `h:427-436`); we track it as one bit + check a notify-exactly-once property | + +--- + +## 2. The deterministic allocator as TLA+ operators + +An allocator state is a partial function + +``` +Alloc == [tag ∈ SUBSET INSTANCES -> [first: 0..HEAP_SIZE, size: 1..HEAP_SIZE]] +``` + +represented in TLA+ as a function with domain `DOMAIN a` = allocated tags. +Free space is **derived**: the complement of the union of allocated intervals +within `[0, HEAP_SIZE)`. + +**Faithfulness argument.** `BasicRangeAllocator` keeps two doubly linked +lists: all ranges in address order, and free ranges. The free list is a +sublist of the address-ordered range list: on `deallocate`, the insertion +neighbors `pf_idx`/`nf_idx` are found by walking the address-ordered `prev` / +`next` chains (`inl:528-537`) and the freed block is linked between them +(`inl:544-551`), and merges (cases 2-4, `inl:552-605`) splice in place. On +`allocate`, leftover blocks replace the consumed block in position +(`inl:473-495`). `add_range` establishes the base case (`inl:100-128`). +Therefore the free list is always in ascending address order, and the +first-fit walk from `ranges[SENTINEL].next_free` (`inl:424-435`, +`can_allocate` `inl:386-402`) selects the **lowest-address free gap that +fits**. Adjacent-free merging (`inl:539-605`) is implicit in the derived-gap +representation: two adjacent free intervals and one merged interval have the +same derived gap set. Hence the representation below is exactly faithful and +deterministic. + +Operators (pure, no TLA+ variables): + +``` +Gaps(a) == maximal intervals of [0,HEAP_SIZE) not covered by ranges of a + (compute: sort allocated intervals by first; walk) +FirstFitOff(a, sz) == the smallest g.first among gaps g with g.size >= sz +CanAlloc(a, sz) == ∃ gap g : g.size >= sz +DoAlloc(a, tag, sz) == a ++ (tag :> [first |-> FirstFitOff(a,sz), size |-> sz]) + (precondition CanAlloc; caller branches on it) +HasTag(a, tag) == tag ∈ DOMAIN a +DoFree(a, tag) == [restrict a to DOMAIN a \ {tag}] +``` + +`missing_ok` mapping: C++ `deallocate(tag, missing_ok)` asserts the tag is +present when `missing_ok = false` (`inl:612-614`). In the model: +`missing_ok = true` call sites use `IF HasTag(a,tag) THEN DoFree(a,tag) ELSE a`; +`missing_ok = false` call sites emit the invariant-checked form — the enclosing +action asserts `HasTag` via an invariant (§6, INV_TriggeredDeallocPresent / +INV_InOrderReleasePresent) and then frees. Do **not** guard the action on +`HasTag`: a missing tag must produce an invariant violation (that is the bug +detector), not a disabled action. + +`can_allocate` (`inl:377-406`) reads state without mutating: maps to +`CanAlloc`. Note `allocate`/`can_allocate` agree on the chosen gap +(same walk), so `CanAlloc(a,sz) => DoAlloc` well-defined is automatic. + +--- + +## 3. Protocol state variables + +``` +VARIABLES + cur, \* current_allocator (h:411) — ground truth of completed heap ops + fut, \* future_allocator (h:411) — heap after all pending ops + rel, \* release_allocator (h:411) — completed allocs + ready releases + pendingAllocs, \* Seq of [inst, size, lastSeq] (h:413-419, 444) + pendingReleases, \* Seq of [inst, isReady, seq, defNote] (h:420-443, 445) + seqCtr, \* cur_release_seqid (h:412), incremented pre-push (++x) + instState, \* [INSTANCES -> status] — see below + instOffset, \* [INSTANCES -> 0..HEAP_SIZE ∪ {OFF_NONE, OFF_FAILED}] + ... client/event vars (§5) +``` + +**`instState` values** (derived from the `INSTOFFSET_*` sentinels +`inst_impl.h:167-172` plus create/destroy bookkeeping): + +| Model status | C++ correspondence | +|---|---| +| `UNREQUESTED` | instance not yet created by client | +| `CREATE_PENDING` | `INSTOFFSET_DELAYEDALLOC` — create requested, precondition untriggered (cc:714) | +| `CREATE_PENDING_DESTROY` | `INSTOFFSET_DELAYEDDESTROY` — destroy requested while create still pending (cc:847) | +| `ALLOC_DEFERRED` | entry in `pending_allocs`; `ALLOC_DEFERRED` returned (cc:788, 802) | +| `ALLOCATED` | offset valid; `eCreated` triggered (`ii:1201-1202`) | +| `FAILED` | `INSTOFFSET_FAILED`; `eCreated` poisoned (`ii:1121-1122`) | +| `DESTROY_QUEUED` | destroy requested (deferred or bad-path-ready), entry in `pending_releases` or awaiting trigger | +| `DESTROYED` | storage freed from `cur` and `notify_deallocation` fired | + +(The Phase 2 author may split `DESTROY_QUEUED` into +untriggered/ready sub-states or derive them from `pendingReleases`; derived is +preferred — keep `instState` minimal and compute the rest.) + +**`ALLOC_DEFERRED` note:** while deferred, the instance also appears in +`pendingAllocs`; on eventual success `notify_allocation(ALLOC_EVENTUAL_SUCCESS)` +fires `eCreated` (`ii:1201-1202`); on `ALLOC_EVENTUAL_FAILURE` (poison replay +only) it is poisoned (`ii:1121-1122`). + +**Validity convention.** `fut` is meaningful only when `pending_allocs` is +nonempty — comment `h:399-405`; it is (re)built from scratch at cc:768-779 +whenever the first deferred alloc is admitted, and the code even notes +"future_allocator becomes invalid" on the failure path (cc:791). `rel` is +meaningful only when `pending_allocs` is nonempty (established cc:787, +maintained thereafter). **Model decision: keep `fut` and `rel` as ordinary +variables holding whatever the code would hold, including stale garbage.** Do +not reset them to a sentinel when they become "invalid": staleness bugs (a +path reading `fut`/`rel` when the code's implicit validity convention says it +shouldn't) are precisely the class of bug we want TLC to find. All invariants +that mention `fut`/`rel` must therefore be guarded by `pendingAllocs /= <<>>`. + +**`deferred_dealloc_notify` / notify-once.** Kept as the `defNote` bit on +each `pendingReleases` entry (`h:436`), plus a per-instance ghost counter +`notifyCount` to state PROP_NotifyOnce (§6). The bit is set on the three +bad-path sites (cc:886, cc:1469 [v2], cc:1739) and drained at cc:1244-1245, +cc:1295-1296, cc:1375-1376 [v2], cc:1644-1645. v1 models the non-redistrict +sites only. + +**seqid semantics.** `cur_release_seqid` starts 0 (cc:675) and every push +uses `++cur_release_seqid` (pre-increment: cc:859, 885, 895, 1147, 1158). +A `PendingAlloc` records `last_release_seqid := cur_release_seqid` **at +admission time** (cc:784, 801) — i.e. the seqid of the newest release pushed +so far, whether or not that release is still in the list. Model identically: +`seqCtr' = seqCtr + 1` with the new entry carrying `seqCtr'`; allocs carry +`seqCtr` unchanged. + +--- + +## 4. Actions + +One action per mutex-holding path. Pseudocode below is normative; C++ lines +cited per step. `Head`/`Tail`/`Append` are TLA+ sequence ops; "erase set S +from seq" keeps relative order. + +### 4.1 `ADA(i, sz)` — attempt_deferrable_allocation (cc:749-807) + +Helper (not a standalone action; inlined into 4.2/4.4). Returns one of +`INSTANT_SUCCESS | DEFERRED | INSTANT_FAILURE` plus state updates. + +``` +IF pendingAllocs = <<>> THEN \* cc:754 + IF CanAlloc(cur, sz) THEN + cur' := DoAlloc(cur, i, sz); result := INSTANT_SUCCESS \* cc:755-757 + ELSE IF pendingReleases = <<>> THEN + result := INSTANT_FAILURE \* cc:762-765 + ELSE + \* rebuild future from scratch \* cc:768-779 + INVARIANT CHECK: ∀ r ∈ pendingReleases: ¬r.isReady \* cc:772 assert + f := cur; FOR r ∈ pendingReleases (in order): + f := FreeMissingOk(f, r.inst) \* cc:778, missing_ok=TRUE + IF CanAlloc(f, sz) THEN \* cc:781 + fut' := DoAlloc(f, i, sz) + pendingAllocs' := << [inst|->i, size|->sz, lastSeq|->seqCtr] >> \* cc:783-784 + rel' := cur \* cc:787 + result := DEFERRED \* cc:788 + ELSE + fut' := f (stale — code computed it and left it) \* cc:790-791 + result := INSTANT_FAILURE +ELSE \* cc:795-806 + IF CanAlloc(fut, sz) THEN \* cc:798 + fut' := DoAlloc(fut, i, sz) + pendingAllocs' := Append(pendingAllocs, + [inst|->i, size|->sz, lastSeq|->seqCtr]) \* cc:800-801 + result := DEFERRED \* cc:802 + ELSE result := INSTANT_FAILURE \* cc:804 +``` + +Transcript flag: the talk describes mode-2 needing a "future fix-up" after an +instant success; the code instead **rebuilds `fut` lazily from scratch** +(cc:768) every time the first deferred alloc is admitted, so no fix-up path +exists. Model the code. + +### 4.2 `RequestCreate(i)` — allocate_storage_deferrable (cc:693-745) + +Client action; enabled when `instState[i] = UNREQUESTED` and client contract +allows (§5). The create carries precondition `preC[i]`. + +- Precondition already triggered+poisoned → `ALLOC_CANCELLED`, + `instState[i]' = FAILED`, poison `eCreated(i)` (cc:703-708; `ii:1121-1122`). +- Precondition untriggered → `instState[i]' = CREATE_PENDING` + (`INSTOFFSET_DELAYEDALLOC`, cc:714), waiter registered (cc:715), return + `DEFERRED` (cc:716). No heap state touched. +- Precondition triggered clean → run `ADA(i, sz_i)` (cc:734-736); apply + result: `INSTANT_SUCCESS` → `ALLOCATED` + fire `eCreated(i)`; + `INSTANT_FAILURE` → `FAILED` + poison `eCreated(i)`; `DEFERRED` → + `ALLOC_DEFERRED` (eCreated not fired yet). + +### 4.3 `TriggerCreate(i)` — DeferredCreate::event_triggered → allocate_storage_immediate (ii:49-56; cc:1087-1176) + +Environment action; enabled when `instState[i] ∈ {CREATE_PENDING, +CREATE_PENDING_DESTROY}` and `preC[i]` has fired (clean or poisoned). + +``` +ddExists := (instState[i] = CREATE_PENDING_DESTROY) \* cc:1106-1107 +IF preC[i] poisoned THEN + result := CANCELLED; instState[i]' = FAILED \* cc:1113-1115 + poison eCreated(i) \* ii:1038,1121-1122 +ELSE + result := ADA(i, sz_i) \* cc:1136-1138 +IF ddExists THEN \* cc:1145-1154 + seqCtr' := seqCtr + 1 + newRel := [inst|->i, isReady|->FALSE, seq|->seqCtr', defNote|->FALSE] + pendingReleases' := Append(pendingReleases, newRel) \* cc:1146-1147 (unconditional!) + IF result ∈ {INSTANT_SUCCESS, DEFERRED} ∧ pendingAllocs' /= <<>> THEN + fut' := FreeMissingOk(fut', i) \* cc:1150-1153 +``` + +Critical details, all deliberate model targets: +1. The destroy's `PendingRelease` gets its seqid **at create-trigger time**, + not at destroy-request time (cc:1147) — the total-order insertion point. +2. It is pushed **even when the allocation failed or was cancelled** + ("success or fail … we have to add it to our list so that we can find it + later", cc:1142-1143) — but applied to `fut` only on success (cc:1150). + Seeded bug #2 territory: a later in-order drain frees it from `cur` with + `missing_ok=false` (cc:1641 → cc:1847 → `inl:614`). +3. Note the ordering: `ADA` pushes the deferred alloc *before* the destroy's + seqid is assigned, so `lastSeq(alloc for i) < seq(release of i)` — an + instance never depends on its own release. +4. The pushed release's precondition is the destroy precondition already + registered via `deferred_destroy.defer` (from 4.4's DELAYEDALLOC path); + `TriggerDestroy(i)` fires later against this entry. + +### 4.4 `RequestDestroy(i)` — release_storage_deferrable (cc:810-924) + +Client action; carries precondition `preD[i]`; contract per §5. + +``` +IF preD[i] triggered ∧ poisoned THEN no-op \* cc:818-825 (silent cancel) +ELSE IF instState[i] = CREATE_PENDING THEN + IF triggered THEN structuralAssertFailed' := TRUE \* cc:846 assert(!triggered) — ghost flag, NOT an enabledness guard; + \* C2-on makes this unreachable, C2-off configs hunt it + instState[i]' := CREATE_PENDING_DESTROY \* cc:845-849 (code proceeds as if untriggered) + (waiter registered on preD[i], cc:920) +ELSE IF pendingAllocs = <<>> THEN \* cc:851-860 + IF preD[i] triggered THEN + IF instState[i] /= FAILED THEN cur' := DoFree(cur, i) \* cc:854-855, missing_ok=FALSE → missingFree ghost site + notify_deallocation(i) \* cc:915-917 + ELSE + seqCtr' := seqCtr+1 + pendingReleases' := Append(.., [i, FALSE, seqCtr', FALSE]) \* cc:858-859 + \* fut NOT touched: invalid while pendingAllocs empty; lazy rebuild covers it +ELSE \* cc:861-897 + IF preD[i] triggered THEN + IF instState[i] = FAILED THEN skip \* cc:866-868 + ELSE + rel' := DoFree(rel, i); fut' := DoFree(fut, i) \* cc:871-872, missing_ok=FALSE + INVARIANT: HasTag(rel,i) ∧ HasTag(fut,i) \* (INV_TriggeredDeallocPresent) + IF ARR() THEN (notifications inside) \* cc:875-876 (§4.6) + ELSE + seqCtr' := seqCtr+1 + pendingReleases' := Append(.., [i, TRUE, seqCtr', TRUE]) \* cc:884-887 ready, defNote + \* notify_deallocation deferred until entry drained \* cc:886-887, 916 + ELSE + IF instState[i] /= FAILED THEN fut' := FreeMissingOk(fut,i) \* cc:892-893 + seqCtr' := seqCtr+1 + pendingReleases' := Append(.., [i, FALSE, seqCtr', FALSE]) \* cc:894-895 +``` + +Untriggered paths register the `TriggerDestroy(i)` waiter (cc:920). + +### 4.5 `TriggerDestroy(i)` — DeferredDestroy::event_triggered → release_storage_immediate (ii:81-99; cc:1600-1794) + +Environment action; enabled when a destroy for `i` was deferred and `preD[i]` +has fired. + +**Poisoned:** `RemovePendingRelease(i)` (§4.7) (cc:1754-1755), no +`notify_deallocation` (cc:1789). + +**Clean (cc:1628-1753):** let `pr := pendingReleases`. If `pr = <<>>` then +`structuralAssertFailed' := TRUE` (cc:1630 `assert(!pending_releases.empty())` +— ghost flag, not a guard; the action still fires where the code would abort). + +*Oldest path* — `Head(pr).inst = i` (cc:1634): + +``` +IF pendingAllocs /= <<>> THEN rel' := DoFree(rel, i) \* cc:1635-1637 (missing_ok=FALSE) +k := 1 +REPEAT \* cc:1640-1700 do-while + e := pr[k] + cur' := DoFree(cur', e.inst) \* cc:1641 (missing_ok=FALSE → INV) + IF e.inst /= i ∧ e.defNote THEN queue notify(e.inst) \* cc:1644-1645 + \* unblock scan \* cc:1649-1695 + WHILE pendingAllocs' /= <<>> : + a := Head(pendingAllocs') + IF k+1 <= Len(pr) ∧ a.lastSeq >= pr[k+1].seq THEN break \* cc:1654-1657 + DEBUG-INV: a.lastSeq >= e.seq \* cc:1662 + INVARIANT: CanAlloc(cur', a.size) \* cc:1670 assert(ok) + off := FirstFitOff(cur', a.size); cur' := DoAlloc(cur', a.inst, a.size) + INVARIANT (FutureOffsetConsistency): \* cc:1674-1691 + IF HasTag(fut, a.inst) THEN fut[a.inst].first = off ∧ fut[a.inst].size = a.size + ELSE LET m == FIRST entry of the FULL not-yet-erased pendingReleases + (searched from index 1, cc:1679-1683) with m.inst = a.inst + IN m exists (else structuralAssertFailed, cc:1682 off-end assert) + ∧ ¬m.isReady \* cc:1687 + \* Under v1's one-release-entry-per-instance uniqueness the FIRST-match + \* form equals "the entry is ¬ready", but the exact from-begin() FIRST- + \* match semantics must be kept — it diverges once v2 duplicates exist. + fire eCreated(a.inst) EVENTUAL_SUCCESS; instState' ALLOCATED + pendingAllocs' := Tail(pendingAllocs') \* cc:1693-1697 + k := k+1 +UNTIL k > Len(pr) ∨ ¬pr[k].isReady \* cc:1700 +pendingReleases' := SubSeq(pr, k, Len(pr)) \* cc:1702 +IF someAllocsSucceeded ∧ pendingAllocs' /= <<>> THEN \* cc:1704-1717 + rel' := cur' + FOR r ∈ pendingReleases' with r.isReady: rel' := DoFree(rel', r.inst) \* cc:1713-1714, missing_ok=FALSE +notify_deallocation(i) at end \* cc:1789-1790 +``` + +Confirmed-correct staleness to model as-is (adversarial review): when the +drain frees entries but **no** allocs succeed, the cc:1704-1717 rebuild is +skipped. This is sound in the code because the drained entries were applied +to both `rel` (cc:1636 / earlier ready marks) and `cur` (cc:1641), so +`rel` stays consistent without a rebuild; the rebuild exists only because +successful allocs are applied to `cur` but not `rel`. Model exactly this +conditional — do not "fix" it. + +*Non-oldest path* (cc:1718-1742): find the FIRST entry `e` with +`e.inst = i`, searching forward; if none exists, +`structuralAssertFailed' := TRUE` (cc:1720-1723 off-end assert; ghost flag, +action still proceeds as the code would into UB). Unique in v1 single-node. +`e.isReady' := TRUE` (cc:1724). Then: +- `pendingAllocs = <<>>` → `cur' := DoFree(cur, i)` (cc:1728, missing_ok=FALSE), + erase `e` (cc:1730), `notify_deallocation(i)` (cc:1789-1790). +- else → `rel' := DoFree(rel, i)` (cc:1738, missing_ok=FALSE), + `e.defNote' := TRUE`; notify deferred (cc:1739-1740, 1789). + +*Tail step, both clean paths:* if `pendingAllocs' /= <<>>` then run `ARR()` +(cc:1751-1753); its notifications (incl. possibly `i`'s own deferred one via +defNote, cc:1744-1750) fire in-action. + +### 4.6 `ARR()` — attempt_release_reordering (cc:1207-1324) + +Helper, called only with `pendingAllocs /= <<>>` and the mutex held. Returns +TRUE/FALSE; on TRUE state is rewritten, on FALSE **no state change** (unwind, +cc:1320). + +``` +front := Head(pendingAllocs) +IF ¬CanAlloc(rel, front.size) THEN return FALSE \* cc:1211-1215 +test := rel +n := largest prefix length s.t. allocs 1..n fit greedily: \* cc:1220-1231 + FOR j = 1.. : IF CanAlloc(test, a_j.size) THEN test := DoAlloc(test, a_j.inst, a_j.size) ELSE break +ASSERT n >= 1 \* cc:1233 +IF n = Len(pendingAllocs) THEN \* full success cc:1236-1252 + cur' := test \* cc:1239 swap + pendingAllocs' := <<>> \* cc:1240 + pendingReleases' := erase all entries with isReady \* cc:1241-1250 + (queue notify for erased entries with defNote) \* cc:1244-1245 + fire eCreated EVENTUAL_SUCCESS for allocs 1..n + return TRUE + \* note: fut left stale; rel left stale (both invalid now: pendingAllocs empty) +ELSE \* partial cc:1253-1322 + tf := test \* cc:1255 + it3 walks pendingReleases from the front + FOR j = n+1 .. Len(pendingAllocs): \* cc:1258-1277 + advance it3 over entries with seq <= a_j.lastSeq: + IF ¬entry.isReady THEN tf := FreeMissingOk(tf, entry.inst) \* cc:1260-1266 + IF CanAlloc(tf, a_j.size) THEN tf := DoAlloc(tf, a_j.inst, a_j.size) \* cc:1270-1272 + ELSE return FALSE (no state change) \* cc:1274-1276, 1311-1321 + \* all future allocs replayed OK \* cc:1280 + FOR remaining entries after it3: IF ¬isReady THEN tf := FreeMissingOk(tf, entry.inst) \* cc:1284-1289 + pendingReleases' := erase all isReady entries \* cc:1292-1301 (+defNote notifies) + pendingAllocs' := SubSeq(pendingAllocs, n+1, ..) \* cc:1304 + cur' := test; fut' := tf \* cc:1306-1307 + rel' := cur' \* cc:1309 + fire eCreated EVENTUAL_SUCCESS for allocs 1..n + return TRUE +``` + +Note the asymmetry that is a prime model target: ready releases already +applied to `rel` are **not** re-applied to `tf` (they're inside `test`), while +non-ready ones are replayed positionally by seqid. The replay applies +non-ready releases *only up to each alloc's lastSeq* before that alloc, and +the rest after all allocs — faithful to cc:1258-1289. + +### 4.7 `RemovePendingRelease(i)` — remove_pending_release (cc:1538-1597) + +Called on poisoned destroy trigger (cc:1754-1755). **This action is +always-on v1 core**, not toggled: intrinsic failure poison (INSTANT_FAILURE/ +CANCELLED poisoning `eCreated(i)`, `ii:1121-1122`) propagates through C2 into +`preD[i]` in every config, so poisoned destroy triggers are reachable without +any user poison (e.g. the DELAYEDDESTROY entry queued at cc:1146-1147 for an +instance that INSTANT_FAILs at create-trigger). The `USER_POISON` toggle +(§5) gates only client-poisoned ballistic events, which widen the reachable +shapes here (needed for BUG-4-escalated / BUG-6(b)). + +``` +IF pendingAllocs = <<>> THEN + erase first entry with inst = i \* cc:1547-1553 + (if none exists: structuralAssertFailed' := TRUE — cc:1550 off-end assert) +ELSE + fut' := cur; rel' := cur \* cc:1556-1557 + found := FALSE; it2 walks pendingAllocs + FOR each entry e ∈ pendingReleases (in order): \* cc:1562-1595 + IF e.inst = i ∧ ¬found THEN found := TRUE; erase e \* cc:1567-1570 + ELSE fut' := FreeMissingOk(fut', e.inst) \* cc:1573 + FOR allocs a with a.lastSeq <= e.seq (advance it2): \* cc:1579 + IF CanAlloc(fut', a.size) THEN fut' := DoAlloc(fut', a.inst, a.size) \* cc:1581 + ELSE INVARIANT: found \* cc:1587 assert + fail a: poison eCreated(a.inst) EVENTUAL_FAILURE \* cc:1591; ii:1121-1122 + erase a from pendingAllocs \* cc:1592 +``` + +Subtle (verify in model review): allocs are replayed onto `fut'` only as the +walk passes entries with `e.seq >= a.lastSeq`; entries erased still +contribute their saved `seqid` (cc:1564). **There is no trailing replay +after the loop**: a pending alloc whose `lastSeq` exceeds every walked seqid +(possible after ARR-partial erased the ready release whose seqid the alloc +recorded) is never re-allocated into the rebuilt `fut'` at all — later +admissions test against a `fut` missing that alloc and may claim overlapping +future space. Registered as **BUG-5** (§8); the model keeps exact code +behavior so TLC adjudicates. Also note `rel'` is set to `cur` and *not* +subsequently given the ready releases — ready entries can exist here +(bad-path entries with `isReady`): each is replayed onto `fut` via cc:1573 +like any other entry but **`rel` is left without it**, violating the +`rel = current + ready releases` definition (h:399-405). Registered as +**BUG-4** (§8, escalated) — keep exact code behavior. + +--- + +## 5. Client / environment model + +**Events.** Each instance `i` has: +- `eCreated(i)`: output event; fired clean on INSTANT/EVENTUAL success + (`ii:1201-1202`), poisoned on failure/cancel (`ii:1121-1122`). +- `preC[i]`: create precondition — either `NOW` (already triggered) or an + abstract dependency term. +- `preD[i]`: destroy precondition — same. + +A dependency term is `[deps: SUBSET INSTANCES, ballistic: BOOLEAN]`. +**Firing rule (exact):** the term fires only when **all** its deps have +*resolved* — each `eCreated(j)`, `j ∈ deps`, has fired either clean or +poisoned — **and**, if `ballistic = TRUE`, its user event has fired. It +fires **poisoned iff at least one dep resolved poisoned** (or its ballistic +event was user-poisoned, `USER_POISON` configs only); otherwise clean. +There is no early-fire on first poison: the merger waits for all inputs to +resolve before firing. Ballistic user events always eventually fire (weak +fairness on their trigger action). Trigger order among enabled events is +nondeterministic. + +**Intrinsic vs user poison.** Intrinsic poison is core v1 in every config: +INSTANT_FAILURE / CANCELLED poisons `eCreated(i)` (`ii:1121-1122`) with no +client involvement, and under C2 that poisons `preD[i]`, routing +`TriggerDestroy(i)` to `RemovePendingRelease` (cc:1754-1755). The +`USER_POISON` toggle adds only one capability: the client may poison a +ballistic user event instead of triggering it cleanly. + +Environment actions `TriggerCreate(i)` / `TriggerDestroy(i)` (§4.3/§4.5) are +enabled once the respective term has fired. Trigger delivery order is +nondeterministic (models event-waiter callback scheduling). + +**Contract constraints** (named, individually toggleable; violation of a +constraint = illegal client, excluded from legal-client runs): + +- **C1 — topological sort / no back edges** (user's stated assumption): the + dependency set of `preC[i]` / `preD[i]` may contain only instances whose + *create was requested strictly earlier in the client's request order*, and + ballistic user events are always eventually triggered by the environment + regardless of Realm results. This encodes "no untriggered user event whose + trigger depends on a deferred allocation result" (transcript ~34:45-35:53). +- **C2 — destroy-after-create**: `i ∈ preD[i].deps` always. Consequences: + `preD[i]` cannot fire clean before `eCreated(i)` fires clean, and a + failed/cancelled create poisons the destroy (which the code then silently + cancels, cc:818-825, or removes, cc:1754-1755). Source: transcript + ~20:51-21:58 ("you can't destroy the instance until … it was actually + created", "I have the check in Legion to make sure"); the code asserts + the consequence at cc:846 (`assert(!triggered)` for a destroy of a + `DELAYEDALLOC` instance). +- **C3 — destroy request ordering**: the transcript mentions Realm "doesn't + want the control plane to destroy an instance until it's at least attempted + the deferrable allocation" (~20:28-20:51) — but the code fully handles + destroy-before-create-trigger via `INSTOFFSET_DELAYEDDESTROY` + (cc:845-849, cc:1106-1110). **Model decision: allow destroy request any + time after the create request.** That path is implemented and is one of the + most delicate (seqid assigned at create-trigger time), so it must be + reachable. + +**Crucially legal and required expressible (seeded bug #1 shape):** the +client may give `preD[j]` a dependency on `eCreated(i)` where `i`'s create +was *requested earlier* but *triggers later* (or never resolves before `j`'s +destroy is requested). C1 permits it (request-order forward edge); nothing in +the code forbids it. + +**Worked example the model must express (bug #1, 2 instances + 1 ballistic +event):** heap size H; sizes `sz(I0) = H`, `sz(I1) = H`. +1. `RequestCreate(I0, preC=NOW)` → INSTANT_SUCCESS (fills heap). +2. `RequestCreate(I1, preC=[deps={}, ballistic=TRUE])` → CREATE_PENDING + (cc:712-717). `eCreated(I1)` handed out. +3. `RequestDestroy(I0, preD=[deps={I0, I1}])` → untriggered (needs + `eCreated(I1)`); pushed as pending release, seq=1 (cc:858-859). +4. Ballistic event fires → `TriggerCreate(I1)` → `ADA`: `pendingAllocs` + empty; `cur` full → no; `pendingReleases` nonempty → `fut = cur - I0`; + I1 fits → **DEFERRED** with `lastSeq = 1` (cc:781-788): Realm plans I1 + into I0's space. +5. Deadlock: `TriggerDestroy(I0)` needs `preD` → needs `eCreated(I1)` → + fires only on I1's EVENTUAL_SUCCESS → needs the release of I0. + `LIVE_NoStuckAllocs` fails; in a terminating client TLC reports deadlock. + +**Result feedback wiring:** DEFERRED = `eCreated(i)` not yet fired; +EVENTUAL_SUCCESS fires it clean (`ii:1201-1202` via cc:904, 1774, etc.); +INSTANT_FAILURE / EVENTUAL_FAILURE / CANCELLED poison it (`ii:1121-1122`). +The model treats `notify_allocation`'s event trigger as part of the acting +step (§1). `notify_deallocation` sets a ghost bit (for PROP_NotifyOnce); +it fires no event visible to the client in v1. + +**Client termination:** each instance is created at most once and destroyed +at most once; when all requested work is drained, only environment stutter +remains. For deadlock-check configs the client must *request destroy for +every instance it creates* (a full-cleanup client), so quiescence = +everything freed. + +--- + +## 6. Invariants and properties + +Allocator-state soundness (representation makes overlap impossible by +construction **only if** DoAlloc is used correctly; state them anyway to +catch spec typos — they're nearly free): + +- **INV_NoOverlap / INV_InBounds**: allocated intervals of `cur` are + pairwise disjoint and within `[0, HEAP_SIZE)`. (Conservation of total size + is implied by disjoint+derived gaps; no separate check needed.) +- **INV_CurrentMatchesGround**: `DOMAIN cur =` + `{i : instState[i] ∈ {ALLOCATED, ALLOC-completed-awaiting-destroy}} ∪` + `{i : ∃ e ∈ pendingReleases : e.inst = i ∧ (¬e.isReady ∨ e.defNote)}` + — precise form: an instance's tag is in `cur` iff it completed a real + allocation (offset valid) and no release of it has yet been *applied to + `cur`*. Ready releases queued on the bad path (cc:884-887) and non-oldest + ready marks (cc:1738-1740) have been applied to `rel`/`fut` but NOT `cur` — + the defining subtlety. Phase 2: define via a ghost variable + `curFreed ⊆ INSTANCES` set exactly where the code calls + `cur.deallocate` — then `INV_CurrentMatchesGround` is + `DOMAIN cur = allocatedEver \ curFreed`, and staleness is checked by + construction. +- **INV_NoReadyWhenNoPendingAllocs** (strengthening of cc:772 `assert`): + `pendingAllocs = <<>> => ∀ e ∈ pendingReleases : ¬e.isReady`. + **Expected to FAIL — this is BUG-6 (§8).** Both adversarial reviewers + constructed legal-client violations (one poison-free with 4 instances, one + via the poison path with 3); the stranded-ready state then trips the real + cc:772 assert on the next oversized alloc request that reaches the rebuild. + Model both this strengthened state invariant AND the exact in-`ADA` check + (a `structuralAssertFailed`-style flag at the rebuild site) so TLC shows + the full request-to-abort trace, not just the stranded state. +- **INV_TriggeredDeallocPresent** (cc:871-872, missing_ok=FALSE): + in `RequestDestroy` triggered/nonempty-pendingAllocs path with + `instState[i] /= FAILED`: `HasTag(rel,i) ∧ HasTag(fut,i)` at the point of + the free. Encode as action-local check (invariant on the pre-state guarded + by action enabledness) or as a ghost "assertFailed" flag set when a + missing_ok=FALSE free finds no tag — **recommended: one global ghost flag + `missingFree` set by any missing_ok=FALSE free on a missing tag; invariant + `¬missingFree`.** This covers cc:855, cc:871-872, cc:1636/1641/1728/1738, + cc:1713-1714 uniformly (each is the code's `assert(missing_ok)` at + `inl:614` firing). +- **INV_StructuralAsserts**: `¬structuralAssertFailed`. The ghost flag is set + by the structural asserts the code makes about its own bookkeeping: + cc:846 (`assert(!triggered)` on a DELAYEDALLOC destroy), cc:1630 + (`assert(!pending_releases.empty())`), and the find-loop off-end asserts + cc:1720-1723 and cc:1548-1551 (plus cc:1682 inside the DEBUG cross-check). + These must be ghost flags, NOT enabledness guards: guarding would silently + *disable* exactly the transitions that crash the real code, hiding bugs in + C2-off (BUG-2 hunt) configs. Each action still performs the step the code + would take as it proceeds into UB/abort; once the flag is set the remainder + of the trace is not meaningful — TLC halts and reports at the violating + state anyway. +- **INV_NoOrphanTags**: every tag in `DOMAIN cur` either belongs to a live + instance (allocated, `notify_deallocation` not yet fired) or has an entry + in `pendingReleases`. Backstop: `Quiescent => DOMAIN cur = {}` (full- + cleanup client). This is the detector for the escalated BUG-4 mechanism + (§8): a tag stranded in `cur` after its release entry was erased and its + dealloc notify fired is invisible to the `curFreed` ghost form of + INV_CurrentMatchesGround, because `deallocate` genuinely is never called. +- **INV_InOrderUnblockSucceeds** (cc:1670 `assert(ok)`): ghost flag + `unblockFailed` set in §4.5's oldest path if `¬CanAlloc(cur', a.size)`; + invariant `¬unblockFailed`. (Same pattern for the cc:1401 twin in v2.) +- **INV_FutureOffsetConsistency** (cc:1674-1691 DEBUG check): at the same + point: `HasTag(fut, a.inst) => (fut[a.inst].first = off ∧ + fut[a.inst].size = a.size)`, and `¬HasTag(fut, a.inst) => ∃` a later + `pendingReleases` entry for `a.inst` with `¬isReady`. Ghost-flag encoding. +- **INV_PoisonReplayOnlyFailsAfterPoint** (cc:1587 `assert(found)`): + in §4.7's replay, an alloc failure before the poisoned entry was found is a + violation. Ghost-flag. Checked in every config (intrinsic poison makes + `RemovePendingRelease` reachable everywhere); the deep shapes need + USER_POISON (Poison4/Big). +- **SAFETY_PromisesKept** (unconditional, all configs): an instance that + ever entered `ALLOC_DEFERRED` reaches `FAILED` **only via + `RemovePendingRelease`** (the EVENTUAL_FAILURE at cc:1591, `ii:1121-1122`) + — goal 1, transcript ~6:28-6:54: "we never want to say that we can + allocate something that can't actually work". Since intrinsic poison is + core, EVENTUAL_FAILURE is reachable in every config; the property is that + no *other* path ever fails a promised allocation. Ghost variables + `wasDeferred` + a `failedVia` tag recorded at the failing action. +- **PROP_NotifyOnce**: `notifyCount[i] <= 1` always, and (liveness, below) + `= 1` eventually for every destroyed instance. Safety half is an invariant. +- **LIVE_NoStuckAllocs**: `∀ i : (instState[i] = ALLOC_DEFERRED) ~> + (instState[i] ∈ {ALLOCATED, FAILED})` under weak fairness on all + environment trigger actions and ballistic events. **Expected to FAIL — + bug #1.** Mark the config with the expected-fail annotation like the + barrier campaign's `LiveG2NoFlush`. +- **LIVE_AllDrains / deadlock detection**: with a terminating full-cleanup + client, run TLC **with** deadlock checking (do not pass `-deadlock`). + Caveat: a client that drains *successfully* also reaches a no-successor + state, which TLC would misreport as deadlock. Fix: define + `Quiescent == all requests issued ∧ pendingAllocs = <<>> ∧ + pendingReleases = <<>> ∧ ∀ i : instState[i] ∈ {DESTROYED, FAILED-with- + destroy-resolved}` and add an explicit self-loop disjunct + `Done == Quiescent ∧ UNCHANGED vars` to `Next`. TLC deadlock reports then + fire **exactly** on stuck non-quiescent states — the cheap primary + detector, catching BUG-1 without temporal checking. Backstop invariant: + `Quiescent => DOMAIN cur = {}` (also the BUG-4 detector, see + INV_NoOrphanTags). Temporal `LIVE_*` configs run with `-deadlock` + (deadlock checking OFF — the `Done` stutter-loop plus fairness handles + termination there). + +--- + +## 7. Module & config plan + +``` +tla/allocation/ + DESIGN.md (this file) + DeferredAlloc.tla protocol: allocator operators (§2), state (§3), actions (§4) + MCDeferredAlloc.tla client/env (§5), constants, ghost vars, invariants (§6) + Smoke.cfg Safety.cfg Liveness.cfg EventLoop.cfg Poison4.cfg Big.cfg + run.sh patterned on tla/barrier/run.sh (JAVA=/opt/homebrew/opt/openjdk/bin/java, + JAR=../barrier/tools/tla2tools.jar) + sapling_tlc.sbatch for runs projected > 1h +``` + +All configs: intrinsic poison on (core); `USER_POISON` off unless noted. +"dlk" = TLC deadlock check ON (with the `Done` self-loop, §6); temporal +configs pass `-deadlock` (check off). + +| Config | Constants | Checks | Expectation | Where/Est. | +|---|---|---|---|---| +| Smoke | H=3, 2 insts (sz 2,2), free deps within C1/C2 | all INV_*, dlk | INV pass; deadlock traces possible (BUG-1 shape reachable even here) | local, seconds-minutes, <10^6 states | +| Safety | H=4, 4 insts (sz 2,1,1,2 — the BUG-6(a) shape; makes ARR-partial and the cc:772 rebuild reachable: needs ≥2 pending allocs + ≥2 pending releases with one ready) | all INV_*, SAFETY_*, PROP_NotifyOnce(safety), dlk | **FAIL expected: INV_NoReadyWhenNoPendingAllocs (BUG-6(a))**; other INVs hunt (BUG-3) | local, minutes-hours, ~10^7-10^9; sapling fallback | +| EventLoop | the §5 worked-example client, hardcoded shape (H=3, 2 insts sz 3,3, 1 ballistic) | dlk | **FAIL (BUG-1)** | local, seconds | +| Liveness | H=4, 3 insts, WF on env, `-deadlock` | LIVE_NoStuckAllocs | **FAIL (BUG-1)**; re-run with a client constraint excluding the BUG-1 shape → expect pass | local, tens of minutes | +| Poison4 | H=4, 4 insts (Safety sizes) + USER_POISON on | + INV_PoisonReplayOnlyFailsAfterPoint, INV_NoOrphanTags, dlk | hunt: BUG-4-escalated (needs 4 insts + user poison), BUG-5, BUG-6(b) | **sapling** likely; try constrained-client local first | +| Big | H=5-6, 4-5 insts, mixed sizes, USER_POISON on | Safety set + Poison4 set | open hunt | **sapling**, >1h → sbatch | + +State-space control: sizes fixed per instance (not chosen nondeterministically) +in v1 configs; dependency sets chosen nondeterministically at request time +within C1/C2 (this is where the client-behavior branching lives). Add a +`StateConstraint` on `seqCtr` (≤ ~2×INSTANCES) as a backstop; it should be +naturally bounded since each instance releases at most once in v1. + +**Symmetry: none in v1.** Instances are not symmetric: they carry distinct +sizes, and even equal-sized instances are distinguished by request order, +seqids, and first-fit offsets. TLC symmetry sets over `INSTANCES` would be +unsound (order-sensitive state) — skip. + +--- + +## 8. Seeded bug hypotheses + +- **BUG-1 — deferred-create ordered at trigger time (event-loop deadlock).** + A pending alloc's `last_release_seqid` and its future-allocator basis are + fixed when the *precondition triggers* (cc:781-788 via cc:1136, admission + at cc:783-784/800-801), not when the create was *requested* — although + `eCreated(i)` is handed out at request time. Releases requested after the + create but before its trigger are in `pendingReleases` and get consumed by + the plan (cc:768-779), yet their preconditions may legally depend on + `eCreated(i)` (C1 forward edge). Cycle: alloc waits on release, release's + precondition waits on alloc's completion. The transcript speaker suspects + exactly this (~24:47-25:35, "doesn't necessarily get put in the right spot + in the overall ordering… can cause an event loop"). Detector: EventLoop + config deadlock check; LIVE_NoStuckAllocs. Fix direction (for the report, + not to implement): snapshot `cur_release_seqid` at request time in + `DeferredCreate` and restrict the future-rebuild replay (cc:769) and the + admission seqid (cc:784) to releases with `seqid <=` snapshot; or refuse to + defer against newer releases and fail instantly. +- **BUG-2 — failed alloc + deferred destroy → missing-tag free.** + cc:1144-1154 pushes the destroy's `PendingRelease` even when `result` is + INSTANT_FAILURE/CANCELLED; the tag was never allocated. The in-order drain + frees with missing_ok=FALSE (cc:1641 → cc:1839-1848 → `inl:609-614` + `assert(missing_ok)`), as does the non-oldest path (cc:1728/1738). + Contract C2 should force `preD` poisoned whenever the alloc failed + (poisoned `eCreated` propagates), routing to `remove_pending_release` + instead — the model must confirm C2 closes *every* such path (incl. the + CANCELLED create with clean-then-poisoned orderings). Detector: + INV_TriggeredDeallocPresent / the `missingFree` ghost under the Poison + config and under a C2-off config (to document what happens to + contract-violating clients). +- **BUG-3 — replay soundness after partial reordering.** The `assert(ok)` + at cc:1670 and the determinism cross-check cc:1674-1691 rest on "current + replays the future's history in the same order"; after a partial + `attempt_release_reordering` (cc:1253-1310) history has been rewritten + (`cur`/`fut`/`rel` swapped, ready entries erased, allocs erased) and the + surviving `lastSeq` values refer to erased entries' seqids. Whether every + interleaving preserves the assert is exactly what TLC will decide. + Detector: INV_InOrderUnblockSucceeds + INV_FutureOffsetConsistency under + Safety/Big configs. +- **BUG-4 (flagged in §4.7, ESCALATED to candidate SAFETY bug) — `rel` + rebuilt without ready releases in `remove_pending_release`.** cc:1556-1557 + rebuilds `rel := cur` but the replay at cc:1573 applies surviving entries + (ready ones included) only to `fut`; `rel` never gets the ready frees, + violating the `rel = current + ready releases` definition (h:399-405). + Escalated mechanism (reviewer B): a surviving READY entry's tag is still + in `cur` — and now also still in `rel`. A later triggered destroy runs + `ARR()` with `test := rel` *still containing that tag*; on the + full-success path `cur' := test` (cc:1239) and the ready entry is erased + with its `defNote` notify fired (cc:1241-1250). The tag was never + deallocated from the allocator state that became `cur`: **permanent range + leak with `notify_deallocation` already fired while the tag is live** — + the #442 slot-recycle double-tracking class. The `curFreed` ghost form of + INV_CurrentMatchesGround does NOT catch this (deallocate genuinely never + called); detector is **INV_NoOrphanTags** + the `Quiescent => DOMAIN cur + = {}` backstop. Reaching it needs ≥4 instances + user poison → Poison4 + config. Secondary (conservative) consequence: missed unblocks via a `rel` + missing ready frees — liveness only. +- **BUG-5 (reviewer A) — poison rebuild drops trailing pending allocs from + `fut`.** The `remove_pending_release` rebuild loop (cc:1562-1595) replays + a pending alloc onto the rebuilt `fut` only when the walk passes an entry + with `seq >= lastSeq` (cc:1579); there is **no trailing alloc replay after + the loop**. Reachable shape: ARR-partial (cc:1253-1310) erases the ready + release whose seqid a surviving alloc recorded as `lastSeq`; a strictly + older release is then poisoned → the rebuild walk's seqids all fall below + that alloc's `lastSeq` → the alloc is silently omitted from `fut'`. Later + admissions (cc:798) test against a `fut` missing a promised allocation and + may claim overlapping future space; the overlap materializes when both + unblock into `cur`. Candidate safety bug. Detectors: + INV_InOrderUnblockSucceeds / INV_FutureOffsetConsistency / + INV_NoOverlap-on-`cur` under Poison4. +- **BUG-6 (both reviewers) — cc:772 `assert(!it->is_ready)` reachable on + legal input; INV_NoReadyWhenNoPendingAllocs is violable.** + Variant (a), poison-free, 4 instances, H=4: `I1`(sz2)@0, `I2`(sz1)@2, + `I3`(sz1)@3 fill the heap; create `I4`(sz2) → DEFERRED with + `lastSeq = seq(R1)` where `R1` = pending destroy(`I1`); destroy(`I2`) + untriggered → `R2`; destroy(`I3`) triggered → cc:871 frees `rel`/`fut`, + ARR front-gate fails → `R3` pushed READY (cc:884-887); TriggerDestroy(`I1`): + oldest drain frees `I1`, unblock scan admits `I4` + (`I4.lastSeq < R2.seq`), `pendingAllocs` EMPTIES; the do-while stops at + `R2` (non-ready), leaving `[R2 ¬ready, R3 ready]` with no pending allocs. + The next oversized alloc request reaches the cc:768 rebuild and trips + cc:772 (abort in a DEBUG build; in release the ready release is replayed + with missing_ok — benign-looking, but `rel := cur` at cc:787 then omits + `R3`'s readiness → BUG-4-shape conservatism). Variant (b), poison path, + 3 instances: `[R1 ¬ready, R2 ready(defNote)]` + pending alloc `A`; + poison `R1` → `remove_pending_release` erases `R1`, replay fails `A` + (cc:1587-1592) → `pendingAllocs` empties with `R2` ready stranded. + Detectors: INV_NoReadyWhenNoPendingAllocs (expected FAIL, §6) + the + in-`ADA` rebuild-site flag for the full request-to-abort trace. TLC + adjudicates downstream severity. + +--- + +## 9. v2 roadmap + +1. **Redistricting**: model `split_range` (`inl:168-274`) — carve N new tags + out of one old range **in place** at ascending offsets, partial success + returns count `i` of tags placed and *deallocates the old tag in every + exit path* (`inl:189, 207, 254, 264`); zero-sized handling; then + `reuse_storage_deferrable` (cc:926-1085) and `reuse_storage_immediate` + (cc:1326-1536) as new actions, `PendingRelease.redistrict_*` payloads + (h:422-424), the `deferred_redistrict` handoff (cc:1078-1080; ii:84-98). + The `offsets` out-param feeds `notify_allocation` per new instance + (cc:1527-1533). +2. **Alignment**: add `align` to instance parameters, extend `FirstFitOff` + with `calculate_offset` semantics (`inl:154-165`) incl. the freed-padding + behavior in allocate (`inl:441-463`) — pure operator change. +3. **Duplicate releases** (multi-node artifact, comment cc:773-777): + introduce an action that enqueues a second release for one instance and + check the missing_ok=FALSE drain paths (cc:1641) against it. +4. **notify-once / instance-slot reuse**: full `deferred_dealloc_notify` + semantics vs. `new_instance` slot recycling (#442, h:427-436) — needs a + model of the instance-slot free list. +5. **Size-0 instances** via the SENTINEL path if tag-lifetime bugs become + interesting. +5b. **Dealloc-completion feedback to the client**: model the + destruction-side profiling responses (`InstanceStatus` / + `InstanceTimeline`, `ii:1248-1262`) as a client-visible "destroyed(i)" + event that user-event triggers may depend on — adds cycle shapes through + destruction completion that v1 cannot express (§1 exclusion). +6. **`SizedRangeAllocator`** (`inl:645-1169`): different fit policy + (size-binned, not address-ordered first-fit) — swap the FirstFit operator + and re-run; determinism assumptions of the protocol must hold for ANY + deterministic allocator, so this is a cheap second data point. diff --git a/tla/allocation/DeferredAlloc.tla b/tla/allocation/DeferredAlloc.tla new file mode 100644 index 0000000000..18589e3249 --- /dev/null +++ b/tla/allocation/DeferredAlloc.tla @@ -0,0 +1,1208 @@ +--------------------------- MODULE DeferredAlloc --------------------------- +(***************************************************************************) +(* Protocol model of Realm's LocalManagedMemory deferred instance *) +(* allocation/deletion logic. Implements DESIGN.md (tla/allocation) *) +(* sections 2-4 and the ghost/invariant machinery of section 6. *) +(* *) +(* Code citations: *) +(* cc = src/realm/mem_impl.cc *) +(* h = src/realm/mem_impl.h *) +(* inl = src/realm/mem_impl.inl *) +(* ii = src/realm/inst_impl.cc *) +(* *) +(* Each action corresponds to exactly one allocator_mutex-holding region *) +(* (h:398, DESIGN.md section 1). Client/environment wiring (preconditions,*) +(* contract C1-C3, fairness, Quiescent/Done) lives in MCDeferredAlloc. *) +(* *) +(* Actions take (trig, pois) parameters describing the state of the *) +(* operation's precondition at the moment of the call; the MC module *) +(* supplies them from its event layer. *) +(***************************************************************************) +EXTENDS Integers, Sequences, FiniteSets, TLC + +CONSTANTS + HEAP_SIZE, \* heap is 0..HEAP_SIZE-1 (DESIGN 1) + INSTANCES, \* finite set of instance ids (never reused, DESIGN 1) + Size, \* [INSTANCES -> 1..HEAP_SIZE] model-assigned sizes + FIX_CAP, \* BUG-1 fix variant: request-time seqid cap on admissions + \* (bugs/BUG-1.md as amended; FALSE = current Realm behavior) + FIX_SWEEP, \* BUG-6 fix A + BUG-4-standalone re-apply (bugs/BUG-6.md; + \* FALSE = current Realm behavior) + FIX_RPR \* BUG-5 fix: trailing alloc replay in remove_pending_release + \* (bugs/BUG-5.md; FALSE = current Realm behavior). Required + \* in the FIX_CAP bundle: the cap raises poisoned-release + \* frequency, and a trailing alloc stranded by the cc:1595 + \* walk end deadlocks in every trigger order + \* (traces/Inversion-bug5-deadlock.txt) + +ASSUME /\ HEAP_SIZE \in Nat \ {0} + /\ Size \in [INSTANCES -> 1..HEAP_SIZE] + /\ FIX_CAP \in BOOLEAN + /\ FIX_SWEEP \in BOOLEAN + /\ FIX_RPR \in BOOLEAN + +\* instOffset sentinels (INSTOFFSET_* stand-ins, inst_impl.h:167-172) +OFF_NONE == -1 +OFF_FAILED == -2 + +\* AllocationResult values used internally (h:88-96) +\* "IS" = ALLOC_INSTANT_SUCCESS, "IF" = ALLOC_INSTANT_FAILURE, +\* "DEF" = ALLOC_DEFERRED, "CANC" = ALLOC_CANCELLED +Statuses == {"UNREQUESTED", "CREATE_PENDING", "CREATE_PENDING_DESTROY", + "ALLOC_DEFERRED", "ALLOCATED", "FAILED", "DESTROYED"} + +VARIABLES + \* --- protocol state (DESIGN 3) --- + cur, \* current_allocator (h:411) + fut, \* future_allocator (h:411) - kept stale exactly as code does + rel, \* release_allocator (h:411) - kept stale exactly as code does + pendingAllocs, \* Seq of [inst, size, lastSeq] (h:413-419, 444) + pendingReleases, \* Seq of [inst, isReady, seq, defNote] (h:420-443, 445) + seqCtr, \* cur_release_seqid (h:412); pushes use pre-increment + reqCap, \* [INSTANCES -> Nat] FIX_CAP only: cur_release_seqid + \* snapshot at create-REQUEST time (C++: new + \* DeferredCreate::seqid_cap field, set in the + \* allocate_storage_deferrable deferral path cc:712-717); + \* all-0 when FIX_CAP = FALSE + instState, \* [INSTANCES -> Statuses] + instOffset, \* [INSTANCES -> -2..HEAP_SIZE] + eCreated, \* [INSTANCES -> {"UNFIRED","CLEAN","POISONED"}] (ii:1121-1122, 1201-1202) + \* --- ghost variables (DESIGN 6) --- + allocatedEver, \* insts whose tag ever entered cur + curFreed, \* insts whose tag left cur (explicit deallocate or realized swap-erasure) + missingFree, \* any missing_ok=FALSE free found no tag (inl:614 assert) + structuralAssertFailed, \* cc:846, 1630, 1662, 1682, 1720-1723, 1548-1551 + unblockFailed, \* cc:1670 assert(ok) + futMismatch, \* cc:1674-1691 DEBUG cross-check + poisonReplayBad, \* cc:1587 assert(found) + readyAtRebuild, \* cc:772 assert(!it->is_ready) at the ADA rebuild site + dupAlloc, \* a materialized DoAlloc hit a live tag (inl:500 leak class, #442) + wasDeferred, \* insts that ever got ALLOC_DEFERRED + failedVia, \* [INSTANCES -> {"NONE","INSTANT","CANCELLED","RPR"}] + notifyCount \* notify_deallocation count per inst (PROP_NotifyOnce) + +protoVars == << cur, fut, rel, pendingAllocs, pendingReleases, seqCtr, + reqCap, instState, instOffset, eCreated, allocatedEver, curFreed, + missingFree, structuralAssertFailed, unblockFailed, + futMismatch, poisonReplayBad, readyAtRebuild, dupAlloc, + wasDeferred, failedVia, notifyCount >> + +ghostVars == << allocatedEver, curFreed, missingFree, structuralAssertFailed, + unblockFailed, futMismatch, poisonReplayBad, readyAtRebuild, + dupAlloc, wasDeferred, failedVia, notifyCount >> + +----------------------------------------------------------------------------- +(***************************************************************************) +(* Section 2: the deterministic first-fit allocator. *) +(* An allocator state is a function tag -> [first, size] over a subset *) +(* of INSTANCES; free space is derived. Faithful to BasicRangeAllocator's *) +(* address-ordered free list (inl:528-551, 424-435): first fit = smallest *) +(* offset o such that [o, o+sz) is entirely free (DESIGN 2). *) +(***************************************************************************) + +EmptyAlloc == [t \in {} |-> [first |-> 0, size |-> 0]] + +HasTag(a, t) == t \in DOMAIN a + +IsFree(a, x) == \A t \in DOMAIN a : + ~(a[t].first <= x /\ x < a[t].first + a[t].size) + +IsFreeRange(a, o, sz) == \A x \in o..(o+sz-1) : IsFree(a, x) + +CanAlloc(a, sz) == \E o \in 0..(HEAP_SIZE - sz) : IsFreeRange(a, o, sz) + +\* lowest-address placement (inl:424-435); minimal o is always a gap start +FirstFitOff(a, sz) == + CHOOSE o \in 0..(HEAP_SIZE - sz) : + /\ IsFreeRange(a, o, sz) + /\ \A o2 \in 0..(HEAP_SIZE - sz) : IsFreeRange(a, o2, sz) => o <= o2 + +\* REPRESENTATION LIMIT: the C++ allocate() on an already-allocated tag does +\* allocated[tag] = idx (inl:500) - the NEW range wins and the old range is +\* LEAKED while still linked (the #442 double-tracking class). The partial- +\* function representation cannot express that leak (@@ is left-biased, so +\* the OLD placement would win here). Call sites therefore flag any +\* materialized DoAlloc onto a live tag via the dupAlloc ghost +\* (INV_NoDupAlloc) instead of modeling the leak. +DoAlloc(a, t, sz) == a @@ (t :> [first |-> FirstFitOff(a, sz), size |-> sz]) + +DoFree(a, t) == [u \in (DOMAIN a) \ {t} |-> a[u]] + +\* missing_ok=TRUE call sites (inl:610-614 tolerated-miss form) +FreeMissingOk(a, t) == IF HasTag(a, t) THEN DoFree(a, t) ELSE a + +----------------------------------------------------------------------------- +(* small helpers *) + +Min(S) == CHOOSE x \in S : \A y \in S : x <= y + +ToSet(s) == {s[k] : k \in DOMAIN s} + +RemoveAt(s, m) == SubSeq(s, 1, m-1) \o SubSeq(s, m+1, Len(s)) + +\* first pending_releases index for inst j, 0 if none (begin()-first search) +FirstRelIdx(pr, j) == + LET ks == {k \in 1..Len(pr) : pr[k].inst = j} + IN IF ks = {} THEN 0 ELSE Min(ks) + +\* indices of ready entries, and the notified/erased sets ARR produces +ReadySet(pr) == {pr[k].inst : k \in {kk \in 1..Len(pr) : pr[kk].isReady}} +ReadyDefNoteSet(pr) == {pr[k].inst : k \in {kk \in 1..Len(pr) : + pr[kk].isReady /\ pr[kk].defNote}} +NonReadyOnly(pr) == SelectSeq(pr, LAMBDA e : ~e.isReady) + +\* apply all pending releases with missing_ok=TRUE, in order (cc:769-779 replay) +RECURSIVE ApplyFreesMissingOk(_, _) +ApplyFreesMissingOk(a, s) == + IF s = <<>> THEN a + ELSE ApplyFreesMissingOk(FreeMissingOk(a, Head(s).inst), Tail(s)) + +\* apply ready releases with missing_ok=FALSE (cc:1706-1717 rebuild); +\* returns [a, miss] where miss records any inl:614 assert(missing_ok) firing +RECURSIVE ApplyReadyFreesStrict(_, _) +ApplyReadyFreesStrict(a, s) == + IF s = <<>> THEN [a |-> a, miss |-> FALSE] + ELSE LET e == Head(s) + IN IF e.isReady + THEN LET m == ~HasTag(a, e.inst) + r == ApplyReadyFreesStrict(FreeMissingOk(a, e.inst), Tail(s)) + IN [a |-> r.a, miss |-> m \/ r.miss] + ELSE ApplyReadyFreesStrict(a, Tail(s)) + +\* FIX_SWEEP (BUG-6 fix A, bugs/BUG-6.md): when pending_allocs is empty, +\* ready entries must not strand in pending_releases - walk in list order, +\* apply each is_ready entry to cur, fire its deferred dealloc-notify, and +\* erase it. C++: new LocalManagedMemory::sweep_ready_releases() helper +\* called (mutex held) from release_storage_immediate after the cc:1702 +\* prefix erase, and from remove_pending_release after the cc:1562-1595 +\* walk, whenever pending_allocs is empty. attempt_release_reordering's +\* full-success path (cc:1236-1252) already erases every ready entry, so it +\* needs no sweep. Running the sweep whenever pending_allocs is empty (not +\* only on the transition) is idempotent and establishes the invariant +\* inductively. Returns [cur, pr, notified, freed, miss]. +RECURSIVE Sweep(_, _) +Sweep(curA, s) == + IF s = <<>> + THEN [cur |-> curA, pr |-> <<>>, notified |-> {}, freed |-> {}, + miss |-> FALSE] + ELSE LET e == Head(s) + IN IF e.isReady + THEN LET m == ~HasTag(curA, e.inst) \* C++ free is missing_ok=FALSE + r == Sweep(FreeMissingOk(curA, e.inst), Tail(s)) + IN [cur |-> r.cur, pr |-> r.pr, + notified |-> (IF e.defNote THEN {e.inst} ELSE {}) + \cup r.notified, + freed |-> {e.inst} \cup r.freed, + miss |-> m \/ r.miss] + ELSE LET r == Sweep(curA, Tail(s)) + IN [cur |-> r.cur, pr |-> <> \o r.pr, + notified |-> r.notified, freed |-> r.freed, + miss |-> r.miss] + +----------------------------------------------------------------------------- +(***************************************************************************) +(* Section 4.1: ADA - attempt_deferrable_allocation (cc:749-807). *) +(* Pure helper; inlined into RequestCreate / TriggerCreate. *) +(* Returns [res, cur, fut, rel, pa, ready772]. *) +(***************************************************************************) +ADARes(i, sz, curA, futA, relA, paA, prA, seqNow) == + IF paA = <<>> THEN \* cc:754 + IF CanAlloc(curA, sz) THEN \* cc:755-757 + [res |-> "IS", cur |-> DoAlloc(curA, i, sz), fut |-> futA, + rel |-> relA, pa |-> paA, ready772 |-> FALSE, + dup |-> HasTag(curA, i)] + ELSE IF prA = <<>> THEN \* cc:762-765 + \* (Config::deferred_instance_allocation is TRUE in v1, DESIGN 1) + [res |-> "IF", cur |-> curA, fut |-> futA, rel |-> relA, + pa |-> paA, ready772 |-> FALSE, dup |-> FALSE] + ELSE + \* rebuild future from scratch (cc:768-779); cc:772 assert(!is_ready) + LET r772 == \E k \in 1..Len(prA) : prA[k].isReady + f == ApplyFreesMissingOk(curA, prA) \* cc:778 missing_ok=TRUE + IN IF CanAlloc(f, sz) THEN \* cc:781 + [res |-> "DEF", + cur |-> curA, + fut |-> DoAlloc(f, i, sz), + rel |-> curA, \* cc:787 + pa |-> << [inst |-> i, size |-> sz, lastSeq |-> seqNow] >>, \* cc:783-784 + ready772 |-> r772, dup |-> HasTag(f, i)] + ELSE + [res |-> "IF", cur |-> curA, + fut |-> f, \* cc:790-791 stale, kept as code leaves it + rel |-> relA, pa |-> paA, ready772 |-> r772, dup |-> FALSE] + ELSE \* cc:795-806 + IF CanAlloc(futA, sz) THEN \* cc:798 + [res |-> "DEF", cur |-> curA, fut |-> DoAlloc(futA, i, sz), + rel |-> relA, + pa |-> Append(paA, [inst |-> i, size |-> sz, lastSeq |-> seqNow]), \* cc:800-801 + ready772 |-> FALSE, dup |-> HasTag(futA, i)] + ELSE + [res |-> "IF", cur |-> curA, fut |-> futA, rel |-> relA, + pa |-> paA, ready772 |-> FALSE, dup |-> FALSE] + +----------------------------------------------------------------------------- +(***************************************************************************) +(* FIX_CAP variant of the admission test (BUG-1 fix blueprint, *) +(* bugs/BUG-1.md as amended). *) +(* C++ sites: the allocate_storage_deferrable deferral path (cc:712-717) *) +(* gains a DeferredCreate::seqid_cap field (atomic snapshot of *) +(* cur_release_seqid at REQUEST time); attempt_deferrable_allocation *) +(* (cc:749-807) replaces the cc:768-779 arrival-order rebuild and the *) +(* cc:798 append-test with the monotone-cap guard + capped canonical *) +(* replay below; admission records last_release_seqid := cap instead of *) +(* cur_release_seqid (cc:784/801). Releases newer than the cap can no *) +(* longer fund the allocation, so a release whose precondition depends on *) +(* this instance's eCreated is never load-bearing for it - the BUG-1 *) +(* cycle is cut. Drain, ARR and RPR are unchanged. *) +(***************************************************************************) + +\* releases in prA with seq <= bound, applied in list order. List order IS +\* seq order: every push uses ++cur_release_seqid, so seqs are strictly +\* increasing along pending_releases and erasures preserve that. +RECURSIVE ApplyCappedFrees(_, _, _) +ApplyCappedFrees(aA, prA, bound) == + IF prA = <<>> \/ Head(prA).seq > bound THEN aA + ELSE ApplyCappedFrees(FreeMissingOk(aA, Head(prA).inst), Tail(prA), bound) + +\* canonical replay: queued allocs placed at their lastSeq watermarks, +\* interleaved with surviving releases (a release funds only if its seq is +\* <= both the next alloc's watermark and the bound); trailing releases with +\* seq <= bound applied after the last alloc. Frees are missing-ok +\* (survivors for FAILED instances have no tag - mirrors cc:778). +\* ok = FALSE if a queued alloc fails to place: prior admissions all tested +\* this same canonical state, so by determinism the C++ fix would assert +\* this cannot happen - a FALSE here is a fix bug (surfaced via +\* structuralAssertFailed at the call sites). +RECURSIVE CanonReplay(_, _, _, _) +CanonReplay(aA, prA, paA, bound) == + IF paA = <<>> + THEN [a |-> ApplyCappedFrees(aA, prA, bound), ok |-> TRUE, dup |-> FALSE] + ELSE LET al == Head(paA) + IN IF prA /= <<>> /\ Head(prA).seq <= al.lastSeq + /\ Head(prA).seq <= bound + THEN CanonReplay(FreeMissingOk(aA, Head(prA).inst), + Tail(prA), paA, bound) + ELSE IF CanAlloc(aA, al.size) + THEN LET r == CanonReplay(DoAlloc(aA, al.inst, al.size), + prA, Tail(paA), bound) + IN [a |-> r.a, ok |-> r.ok, + dup |-> HasTag(aA, al.inst) \/ r.dup] + ELSE [a |-> aA, ok |-> FALSE, dup |-> FALSE] + +ADAResCap(i, sz, cap, curA, futA, relA, paA, prA) == + IF paA = <<>> /\ CanAlloc(curA, sz) THEN + \* cc:754-757 current-allocator fast path, kept verbatim under FIX_CAP + [res |-> "IS", cur |-> DoAlloc(curA, i, sz), fut |-> futA, + rel |-> relA, pa |-> paA, ready772 |-> FALSE, + dup |-> HasTag(curA, i), capAssert |-> FALSE] + ELSE IF paA /= <<>> /\ cap < paA[Len(paA)].lastSeq THEN + \* monotone-cap guard: keeps lastSeq non-decreasing along the queue + \* (C++: instant failure; caller poisons eCreated) + [res |-> "IF", cur |-> curA, fut |-> futA, rel |-> relA, + pa |-> paA, ready772 |-> FALSE, dup |-> FALSE, capAssert |-> FALSE] + ELSE + \* capped canonical test on a copy of cur. If cap < seq of the front + \* survivor the funding set is empty and this degenerates to a cur-only + \* test (the intended C++ fast path). NO ready-fold in v1: is_ready + \* entries with seq > cap do NOT fund (agreed: spurious instant-fail + \* acceptable; ARR still helps opportunistically, unchanged). + LET t == CanonReplay(curA, prA, paA, cap) + IN IF t.ok /\ CanAlloc(t.a, sz) + THEN LET newPa == Append(paA, + [inst |-> i, size |-> sz, lastSeq |-> cap]) + maxSeq == IF prA = <<>> THEN cap ELSE prA[Len(prA)].seq + f == CanonReplay(curA, prA, newPa, + IF maxSeq > cap THEN maxSeq ELSE cap) + IN [res |-> "DEF", cur |-> curA, + \* fut maintained CANONICALLY (cur + ALL surviving releases + \* + ALL queued allocs at their watermarks, incl. the new + \* one at cap): INV_FutureOffsetConsistency is expected to + \* HOLD under FIX_CAP - a canonical-vs-drain placement + \* disagreement is a fix bug TLC must catch. + fut |-> f.a, + rel |-> IF paA = <<>> THEN curA ELSE relA, \* cc:787 analog + pa |-> newPa, ready772 |-> FALSE, + dup |-> f.dup, capAssert |-> ~f.ok] + ELSE + [res |-> "IF", cur |-> curA, fut |-> futA, rel |-> relA, + pa |-> paA, ready772 |-> FALSE, dup |-> FALSE, + capAssert |-> ~t.ok] + +\* dispatcher: cap = reqCap[i] for trigger-deferred creates, seqCtr-now for +\* request-triggered ones (callers pass it); legacy path ignores cap. +ADAResv(i, sz, cap, curA, futA, relA, paA, prA, seqNow) == + IF FIX_CAP THEN ADAResCap(i, sz, cap, curA, futA, relA, paA, prA) + ELSE ADARes(i, sz, curA, futA, relA, paA, prA, seqNow) + +----------------------------------------------------------------------------- +(***************************************************************************) +(* Section 4.6: ARR - attempt_release_reordering (cc:1207-1324). *) +(* Pure helper; called only with paA # <<>>. *) +(* Returns [changed, cur, fut, rel, pa, pr, placed, notified, erasedReady].*) +(* On changed = FALSE the caller keeps its own values (unwind, cc:1320). *) +(***************************************************************************) + +\* greedy prefix of pending allocs onto test allocator (cc:1220-1231) +RECURSIVE Greedy(_, _) +Greedy(test, pa) == + IF pa = <<>> \/ ~CanAlloc(test, Head(pa).size) + THEN [test |-> test, n |-> 0, placed |-> <<>>, dup |-> FALSE] + ELSE LET a == Head(pa) + off == FirstFitOff(test, a.size) \* placement-time offset (FIX 1) + r == Greedy(DoAlloc(test, a.inst, a.size), Tail(pa)) + IN [test |-> r.test, n |-> 1 + r.n, + placed |-> <<[inst |-> a.inst, off |-> off]>> \o r.placed, + dup |-> HasTag(test, a.inst) \/ r.dup] + +\* trailing non-ready releases after all allocs replayed (cc:1284-1289) +RECURSIVE TrailingFrees(_, _, _) +TrailingFrees(tf, pr, idx) == + IF idx > Len(pr) THEN tf + ELSE TrailingFrees(IF pr[idx].isReady THEN tf + ELSE FreeMissingOk(tf, pr[idx].inst), \* cc:1286 missing_ok=TRUE + pr, idx + 1) + +\* partial-path replay (cc:1258-1289): for each remaining alloc, first apply +\* non-ready releases with seq <= its lastSeq, then allocate; trailing +\* non-ready releases after the last alloc. Ready entries advance the walk +\* but are NOT re-applied (already inside test, DESIGN 4.6 note). +RECURSIVE Replay(_, _, _, _) +Replay(tf, pr, idx, paRem) == + IF paRem = <<>> THEN + [ok |-> TRUE, tf |-> TrailingFrees(tf, pr, idx), dup |-> FALSE] + ELSE LET a == Head(paRem) + IN IF idx <= Len(pr) /\ pr[idx].seq <= a.lastSeq \* cc:1260-1266 + THEN Replay(IF pr[idx].isReady THEN tf + ELSE FreeMissingOk(tf, pr[idx].inst), \* cc:1263 missing_ok=TRUE + pr, idx + 1, paRem) + ELSE IF CanAlloc(tf, a.size) \* cc:1270-1272 + THEN LET rr == Replay(DoAlloc(tf, a.inst, a.size), + pr, idx, Tail(paRem)) + IN [ok |-> rr.ok, tf |-> rr.tf, + dup |-> HasTag(tf, a.inst) \/ rr.dup] + ELSE [ok |-> FALSE, tf |-> tf, dup |-> FALSE] \* cc:1274-1276 -> unwind + +ARRNone == [changed |-> FALSE] + +ARRFun(curA, futA, relA, paA, prA) == + \* front-only gate (cc:1211-1215); guarantees the cc:1233 assert(n >= 1) + IF ~CanAlloc(relA, Head(paA).size) + THEN ARRNone + ELSE LET g == Greedy(relA, paA) + IN IF g.n = Len(paA) THEN + \* full success (cc:1236-1252): cur := test, clear allocs, + \* erase ready releases; fut and rel left stale (cc:1252 note) + [changed |-> TRUE, + cur |-> g.test, fut |-> futA, rel |-> relA, + pa |-> <<>>, + pr |-> NonReadyOnly(prA), \* cc:1241-1250 + placed |-> g.placed, + notified |-> ReadyDefNoteSet(prA), \* cc:1244-1245 + erasedReady |-> ReadySet(prA), + dup |-> g.dup] + ELSE + LET rp == Replay(g.test, prA, 1, + SubSeq(paA, g.n + 1, Len(paA))) \* cc:1255-1289 + IN IF rp.ok THEN + [changed |-> TRUE, + cur |-> g.test, \* cc:1306 + fut |-> rp.tf, \* cc:1307 + rel |-> g.test, \* cc:1309 rel := cur' + pa |-> SubSeq(paA, g.n + 1, Len(paA)), \* cc:1304 + pr |-> NonReadyOnly(prA), \* cc:1292-1301 + placed |-> g.placed, + notified |-> ReadyDefNoteSet(prA), \* cc:1295-1296 + erasedReady |-> ReadySet(prA), + dup |-> g.dup \/ rp.dup] + ELSE ARRNone \* cc:1311-1321 unwind + +----------------------------------------------------------------------------- +(***************************************************************************) +(* Section 4.5 oldest-path machinery: the cc:1640-1700 do-while drain and *) +(* its nested unblock scan (cc:1649-1695). *) +(***************************************************************************) + +\* unblock scan at drain position k: cur already reflects the k-th free; +\* fut is the untouched future_allocator (read-only cross-checks). +RECURSIVE UnblockScan(_, _, _, _, _) +UnblockScan(paA, pr, k, curA, futA) == + IF paA = <<>> + THEN [cur |-> curA, pa |-> paA, succ |-> <<>>, + ubFail |-> FALSE, futBad |-> FALSE, structF |-> FALSE, dup |-> FALSE] + ELSE LET a == Head(paA) + IN IF k + 1 <= Len(pr) /\ a.lastSeq >= pr[k+1].seq \* cc:1654-1657 break + THEN [cur |-> curA, pa |-> paA, succ |-> <<>>, + ubFail |-> FALSE, futBad |-> FALSE, structF |-> FALSE, + dup |-> FALSE] + ELSE + LET orderBad == a.lastSeq < pr[k].seq \* cc:1662 DEBUG assert + IN IF ~CanAlloc(curA, a.size) + THEN \* cc:1670 assert(ok) fails: flag and stop the scan + [cur |-> curA, pa |-> paA, succ |-> <<>>, + ubFail |-> TRUE, futBad |-> FALSE, structF |-> orderBad, + dup |-> FALSE] + ELSE + LET off == FirstFitOff(curA, a.size) \* placement-time (cc:1668) + cur2 == DoAlloc(curA, a.inst, a.size) + fm == FirstRelIdx(pr, a.inst) \* cc:1679-1683 begin()-first + fb == IF HasTag(futA, a.inst) \* cc:1674-1677 + THEN ~(futA[a.inst].first = off /\ + futA[a.inst].size = a.size) + ELSE fm /= 0 /\ pr[fm].isReady \* cc:1687 must be !ready + sf == orderBad \/ + (~HasTag(futA, a.inst) /\ fm = 0) \* cc:1682 off-end + r == UnblockScan(Tail(paA), pr, k, cur2, futA) + IN [cur |-> r.cur, pa |-> r.pa, + succ |-> <<[inst |-> a.inst, off |-> off]>> \o r.succ, + ubFail |-> r.ubFail, + futBad |-> fb \/ r.futBad, + structF |-> sf \/ r.structF, + dup |-> HasTag(curA, a.inst) \/ r.dup] + +\* the do-while (cc:1640-1700): frees pr[k] from cur, runs the unblock scan, +\* continues while the next entry is ready. Returns k' = first surviving +\* index (cc:1702 erases the prefix). +RECURSIVE DrainLoop(_, _, _, _, _, _) +DrainLoop(pr, k, i, curA, futA, paA) == + LET e == pr[k] + miss == ~HasTag(curA, e.inst) \* cc:1641 missing_ok=FALSE + cur1 == FreeMissingOk(curA, e.inst) + nf == IF e.inst /= i /\ e.defNote THEN {e.inst} ELSE {} \* cc:1644-1645 + s == UnblockScan(paA, pr, k, cur1, futA) + cont == k + 1 <= Len(pr) /\ pr[k+1].isReady \* cc:1700 + IN IF cont + THEN LET r == DrainLoop(pr, k + 1, i, s.cur, futA, s.pa) + IN [cur |-> r.cur, pa |-> r.pa, k |-> r.k, + succ |-> s.succ \o r.succ, + notified |-> nf \cup r.notified, + curFreedSet |-> {e.inst} \cup r.curFreedSet, + missFree |-> miss \/ r.missFree, + ubFail |-> s.ubFail \/ r.ubFail, + futBad |-> s.futBad \/ r.futBad, + structF |-> s.structF \/ r.structF, + dup |-> s.dup \/ r.dup] + ELSE [cur |-> s.cur, pa |-> s.pa, k |-> k + 1, + succ |-> s.succ, notified |-> nf, + curFreedSet |-> {e.inst}, missFree |-> miss, + ubFail |-> s.ubFail, futBad |-> s.futBad, structF |-> s.structF, + dup |-> s.dup] + +----------------------------------------------------------------------------- +(***************************************************************************) +(* Section 4.7 machinery: remove_pending_release (cc:1538-1597). *) +(***************************************************************************) + +\* inner alloc-replay loop (cc:1579-1593): consume allocs with +\* lastSeq <= seqid; successes stay in the list (it2 advances), failures are +\* erased and reported. Returns [fut, kept, failed, rest]. +RECURSIVE InnerAllocs(_, _, _) +InnerAllocs(futA, paA, seqid) == + IF paA = <<>> \/ Head(paA).lastSeq > seqid + THEN [fut |-> futA, kept |-> <<>>, failed |-> {}, rest |-> paA, + dup |-> FALSE] + ELSE LET a == Head(paA) + IN IF CanAlloc(futA, a.size) \* cc:1581 + THEN LET r == InnerAllocs(DoAlloc(futA, a.inst, a.size), + Tail(paA), seqid) + IN [fut |-> r.fut, kept |-> <> \o r.kept, + failed |-> r.failed, rest |-> r.rest, + dup |-> HasTag(futA, a.inst) \/ r.dup] + ELSE LET r == InnerAllocs(futA, Tail(paA), seqid) \* cc:1591-1592 erase + IN [fut |-> r.fut, kept |-> r.kept, + failed |-> {a.inst} \cup r.failed, rest |-> r.rest, + dup |-> r.dup] + +\* outer walk (cc:1562-1595). The erased target still contributes its saved +\* seqid (cc:1564). NOTE (BUG-5, DESIGN 4.7/8): there is no trailing alloc +\* replay after the loop - allocs with lastSeq above every walked seqid are +\* never re-placed into the rebuilt fut. Faithful to the code. +RECURSIVE RPRLoop(_, _, _, _, _, _) +RPRLoop(pr, idx, i, found, futA, paA) == + IF idx > Len(pr) + THEN [pr |-> <<>>, fut |-> futA, paOut |-> paA, + trail |-> paA, \* the never-examined remainder (it2's final + \* position onward) - the ONLY part FIX_RPR's + \* trailing pass may see (bugs/DUPALLOC-TRIAGE.md) + failed |-> {}, bad |-> FALSE, foundOut |-> found, dup |-> FALSE] + ELSE LET e == pr[idx] + isT == e.inst = i /\ ~found \* cc:1567 first match only + fut1 == IF isT THEN futA + ELSE FreeMissingOk(futA, e.inst) \* cc:1573 missing_ok=TRUE + inner == InnerAllocs(fut1, paA, e.seq) \* cc:1579 saved seqid + badH == inner.failed /= {} /\ ~(found \/ isT) \* cc:1587 assert(found) + r == RPRLoop(pr, idx + 1, i, found \/ isT, inner.fut, inner.rest) + IN [pr |-> (IF isT THEN r.pr ELSE <> \o r.pr), + fut |-> r.fut, + paOut |-> inner.kept \o r.paOut, \* = full survivors: kept \o trail + trail |-> r.trail, + failed |-> inner.failed \cup r.failed, + bad |-> badH \/ r.bad, + foundOut |-> r.foundOut, + dup |-> inner.dup \/ r.dup] + +\* FIX_RPR (BUG-5 fix, bugs/BUG-5.md): trailing alloc replay. C++: after +\* the outer walk in remove_pending_release ends at cc:1595, run the +\* cc:1579-1594 inner loop ONCE MORE with no seqid bound - allocs whose +\* lastSeq exceeds every walked seqid (possible after ARR-partial erased the +\* ready release they recorded) are otherwise neither refunded into fut nor +\* failed, and strand forever. Placement mirrors InnerAllocs: success lands +\* in fut and the alloc stays queued (lastSeq unchanged); failure is +\* EVENTUAL_FAILUREd exactly like the in-walk path (failedVia "RPR", +\* eCreated poisoned, erased - downstream poison cascades are correct). +\* The cc:1587 assert(found) analog is trivially satisfied here: the target +\* erase happened before any trailing processing (no 'bad' field needed). +RECURSIVE TrailingRPR(_, _) +TrailingRPR(futA, paA) == + IF paA = <<>> + THEN [fut |-> futA, kept |-> <<>>, failed |-> {}, dup |-> FALSE] + ELSE LET a == Head(paA) + IN IF CanAlloc(futA, a.size) + THEN LET r == TrailingRPR(DoAlloc(futA, a.inst, a.size), Tail(paA)) + IN [fut |-> r.fut, kept |-> <> \o r.kept, + failed |-> r.failed, + dup |-> HasTag(futA, a.inst) \/ r.dup] + ELSE LET r == TrailingRPR(futA, Tail(paA)) + IN [fut |-> r.fut, kept |-> r.kept, + failed |-> {a.inst} \cup r.failed, dup |-> r.dup] + +----------------------------------------------------------------------------- +(* batch-update helpers over the per-instance maps. These read the *) +(* current (unprimed) variables; actions apply them to compute primes. *) + +\* eCreated after firing succS clean (EVENTUAL/INSTANT success, +\* ii:1201-1202) and poisoning failS (ii:1121-1122) +ECreatedAfter(succS, failS) == + [j \in INSTANCES |-> IF j \in succS THEN "CLEAN" + ELSE IF j \in failS THEN "POISONED" + ELSE eCreated[j]] + +\* instState after: failures -> FAILED; successful allocs -> ALLOCATED +\* (or DESTROYED if their dealloc notify fires in the same action); +\* notified ALLOCATED instances -> DESTROYED; others keep their state. +StatusAfter(succS, failS, notifS) == + [j \in INSTANCES |-> + IF j \in failS THEN "FAILED" + ELSE IF j \in notifS + THEN IF j \in succS \/ instState[j] = "ALLOCATED" + THEN "DESTROYED" ELSE instState[j] + ELSE IF j \in succS THEN "ALLOCATED" + ELSE instState[j]] + +\* instOffset after: successes read their placement from allocator a2 +\* (all placed tags end up in the final cur - DESIGN 4.5/4.6) +OffsetsAfter(a2, succS, failS) == + [j \in INSTANCES |-> IF j \in succS THEN a2[j].first + ELSE IF j \in failS THEN OFF_FAILED + ELSE instOffset[j]] + +NotifyAfter(S) == + [j \in INSTANCES |-> notifyCount[j] + (IF j \in S THEN 1 ELSE 0)] + +InstsOf(pairs) == {p.inst : p \in pairs} + +\* placement-time offsets (FIX 1): the C++ captures each unblocked alloc's +\* offset at placement time (cc:1668, cc:1693) and never re-reads the +\* allocator; re-reading the FINAL allocator is wrong when a later ready +\* entry drained in the same action frees the just-placed tag (reachable +\* with contract C2 off), and aborts TLC on the missing key. pairs is a +\* set of [inst, off] records; an instance is placed at most once per action. +OffsetsAfterPairs(pairs, failS) == + [j \in INSTANCES |-> + IF \E p \in pairs : p.inst = j + THEN (CHOOSE p \in pairs : p.inst = j).off + ELSE IF j \in failS THEN OFF_FAILED ELSE instOffset[j]] + +FailedViaAfter(S, tag) == + [j \in INSTANCES |-> IF j \in S THEN tag ELSE failedVia[j]] + +----------------------------------------------------------------------------- +(***************************************************************************) +(* Section 4.2: RequestCreate(i, trig, pois) *) +(* allocate_storage_deferrable (cc:693-745). trig/pois describe preC[i] *) +(* at request time (supplied by MC). *) +(***************************************************************************) +RequestCreate(i, trig, pois) == + /\ instState[i] = "UNREQUESTED" + /\ IF trig /\ pois THEN + \* ALLOC_CANCELLED (cc:703-708); eCreated poisoned (ii:1121-1122) + /\ instState' = [instState EXCEPT ![i] = "FAILED"] + /\ instOffset' = [instOffset EXCEPT ![i] = OFF_FAILED] + /\ eCreated' = [eCreated EXCEPT ![i] = "POISONED"] + /\ failedVia' = [failedVia EXCEPT ![i] = "CANCELLED"] + /\ UNCHANGED << cur, fut, rel, pendingAllocs, pendingReleases, seqCtr, + reqCap, allocatedEver, curFreed, missingFree, + structuralAssertFailed, unblockFailed, futMismatch, + poisonReplayBad, readyAtRebuild, dupAlloc, wasDeferred, + notifyCount >> + ELSE IF ~trig THEN + \* defer (cc:712-717): INSTOFFSET_DELAYEDALLOC, waiter registered + \* FIX_CAP: snapshot cur_release_seqid at REQUEST time (C++: new + \* DeferredCreate::seqid_cap field, set in the cc:712-717 deferral + \* path under the atomicity the C++ needs for that read) + /\ instState' = [instState EXCEPT ![i] = "CREATE_PENDING"] + /\ reqCap' = IF FIX_CAP THEN [reqCap EXCEPT ![i] = seqCtr] ELSE reqCap + /\ UNCHANGED << cur, fut, rel, pendingAllocs, pendingReleases, seqCtr, + instOffset, eCreated, allocatedEver, curFreed, + missingFree, structuralAssertFailed, unblockFailed, + futMismatch, poisonReplayBad, readyAtRebuild, dupAlloc, + wasDeferred, failedVia, notifyCount >> + ELSE + \* triggered clean: ADA (cc:734-736); request == trigger, so under + \* FIX_CAP the cap is seqCtr-now (uniform with the deferred case) + LET r == ADAResv(i, Size[i], seqCtr, cur, fut, rel, + pendingAllocs, pendingReleases, seqCtr) + IN /\ cur' = r.cur /\ fut' = r.fut /\ rel' = r.rel + /\ pendingAllocs' = r.pa + /\ readyAtRebuild' = (readyAtRebuild \/ r.ready772) + /\ dupAlloc' = (dupAlloc \/ r.dup) + \* FIX_CAP: capAssert = canonical-replay-failed (fix-internal + \* assert the intended C++ would carry) + /\ structuralAssertFailed' = + (structuralAssertFailed \/ (IF FIX_CAP THEN r.capAssert + ELSE FALSE)) + /\ UNCHANGED reqCap + /\ CASE r.res = "IS" -> + /\ instState' = [instState EXCEPT ![i] = "ALLOCATED"] + /\ instOffset' = [instOffset EXCEPT ![i] = r.cur[i].first] + /\ eCreated' = [eCreated EXCEPT ![i] = "CLEAN"] + /\ allocatedEver' = allocatedEver \cup {i} + /\ UNCHANGED << wasDeferred, failedVia >> + [] r.res = "DEF" -> + /\ instState' = [instState EXCEPT ![i] = "ALLOC_DEFERRED"] + /\ wasDeferred' = wasDeferred \cup {i} + /\ UNCHANGED << instOffset, eCreated, allocatedEver, + failedVia >> + [] r.res = "IF" -> + /\ instState' = [instState EXCEPT ![i] = "FAILED"] + /\ instOffset' = [instOffset EXCEPT ![i] = OFF_FAILED] + /\ eCreated' = [eCreated EXCEPT ![i] = "POISONED"] + /\ failedVia' = [failedVia EXCEPT ![i] = "INSTANT"] + /\ UNCHANGED << wasDeferred, allocatedEver >> + /\ UNCHANGED << pendingReleases, seqCtr, curFreed, missingFree, + unblockFailed, futMismatch, + poisonReplayBad, notifyCount >> + +----------------------------------------------------------------------------- +(***************************************************************************) +(* Section 4.3: TriggerCreate(i, pois) *) +(* DeferredCreate::event_triggered -> allocate_storage_immediate *) +(* (ii:49-56; cc:1087-1176). pois = preC[i] fired poisoned. *) +(***************************************************************************) +TriggerCreate(i, pois) == + /\ instState[i] \in {"CREATE_PENDING", "CREATE_PENDING_DESTROY"} + /\ LET dd == instState[i] = "CREATE_PENDING_DESTROY" \* cc:1106-1107 + r == IF pois + THEN [res |-> "CANC", cur |-> cur, fut |-> fut, rel |-> rel, + pa |-> pendingAllocs, ready772 |-> FALSE, + dup |-> FALSE, capAssert |-> FALSE] \* cc:1113-1115 + ELSE ADAResv(i, Size[i], reqCap[i], cur, fut, rel, + pendingAllocs, pendingReleases, seqCtr) \* cc:1136-1138 + \* FIX_CAP: cap = seqid_cap snapshot from request time + seq2 == IF dd THEN seqCtr + 1 ELSE seqCtr \* cc:1147 ++seqid + pr2 == IF dd \* cc:1146-1147 unconditional push + THEN Append(pendingReleases, + [inst |-> i, isReady |-> FALSE, + seq |-> seq2, defNote |-> FALSE]) + ELSE pendingReleases + fut2 == IF dd /\ r.res \in {"IS", "DEF"} /\ r.pa /= <<>> + THEN FreeMissingOk(r.fut, i) \* cc:1150-1153 + ELSE r.fut + IN /\ cur' = r.cur /\ fut' = fut2 /\ rel' = r.rel + /\ pendingAllocs' = r.pa + /\ pendingReleases' = pr2 + /\ seqCtr' = seq2 + /\ readyAtRebuild' = (readyAtRebuild \/ r.ready772) + /\ dupAlloc' = (dupAlloc \/ r.dup) + /\ structuralAssertFailed' = + (structuralAssertFailed \/ (IF FIX_CAP THEN r.capAssert + ELSE FALSE)) + \* FIX_CAP: the seqid_cap dies with the DeferredCreate object; + \* reset to 0 purely to canonicalize the model state + /\ reqCap' = IF FIX_CAP THEN [reqCap EXCEPT ![i] = 0] ELSE reqCap + /\ CASE r.res = "IS" -> + /\ instState' = [instState EXCEPT ![i] = "ALLOCATED"] + /\ instOffset' = [instOffset EXCEPT ![i] = r.cur[i].first] + /\ eCreated' = [eCreated EXCEPT ![i] = "CLEAN"] + /\ allocatedEver' = allocatedEver \cup {i} + /\ UNCHANGED << wasDeferred, failedVia >> + [] r.res = "DEF" -> + /\ instState' = [instState EXCEPT ![i] = "ALLOC_DEFERRED"] + /\ wasDeferred' = wasDeferred \cup {i} + /\ UNCHANGED << instOffset, eCreated, allocatedEver, + failedVia >> + [] r.res = "IF" -> + /\ instState' = [instState EXCEPT ![i] = "FAILED"] + /\ instOffset' = [instOffset EXCEPT ![i] = OFF_FAILED] + /\ eCreated' = [eCreated EXCEPT ![i] = "POISONED"] + /\ failedVia' = [failedVia EXCEPT ![i] = "INSTANT"] + /\ UNCHANGED << wasDeferred, allocatedEver >> + [] r.res = "CANC" -> + /\ instState' = [instState EXCEPT ![i] = "FAILED"] + /\ instOffset' = [instOffset EXCEPT ![i] = OFF_FAILED] + /\ eCreated' = [eCreated EXCEPT ![i] = "POISONED"] + /\ failedVia' = [failedVia EXCEPT ![i] = "CANCELLED"] + /\ UNCHANGED << wasDeferred, allocatedEver >> + /\ UNCHANGED << curFreed, missingFree, + unblockFailed, futMismatch, poisonReplayBad, + notifyCount >> + +----------------------------------------------------------------------------- +(***************************************************************************) +(* Section 4.4: RequestDestroy(i, trig, pois) *) +(* release_storage_deferrable (cc:810-924). trig/pois describe preD[i] *) +(* at request time. MC guarantees at most one destroy request per inst. *) +(***************************************************************************) +RequestDestroy(i, trig, pois) == + /\ instState[i] \notin {"UNREQUESTED", "DESTROYED"} + /\ UNCHANGED reqCap \* release path never touches the FIX_CAP snapshot + /\ IF trig /\ pois THEN + \* silent cancel (cc:818-825) + UNCHANGED protoVars + ELSE IF instState[i] = "CREATE_PENDING" THEN + \* DELAYEDALLOC -> DELAYEDDESTROY (cc:845-849); cc:846 assert(!triggered) + \* (FIX 4: with trig, a release build proceeds past the assert and + \* still acks the triggered destroy at cc:915-917 - model as-code; + \* the state is already condemned by INV_StructuralAsserts) + /\ instState' = [StatusAfter({}, {}, IF trig THEN {i} ELSE {}) + EXCEPT ![i] = "CREATE_PENDING_DESTROY"] + /\ structuralAssertFailed' = (structuralAssertFailed \/ trig) + /\ notifyCount' = NotifyAfter(IF trig THEN {i} ELSE {}) + /\ UNCHANGED << cur, fut, rel, pendingAllocs, pendingReleases, seqCtr, + instOffset, eCreated, allocatedEver, curFreed, + missingFree, unblockFailed, futMismatch, + poisonReplayBad, readyAtRebuild, dupAlloc, wasDeferred, + failedVia >> + ELSE IF pendingAllocs = <<>> THEN \* cc:851-860 + IF trig THEN + \* apply directly to current state (cc:852-855), ack (cc:915-917) + LET failedI == instState[i] = "FAILED" + miss == ~failedI /\ ~HasTag(cur, i) \* cc:855 missing_ok=FALSE + IN /\ cur' = IF failedI THEN cur ELSE FreeMissingOk(cur, i) + /\ curFreed' = IF failedI THEN curFreed ELSE curFreed \cup {i} + /\ missingFree' = (missingFree \/ miss) + /\ notifyCount' = NotifyAfter({i}) + /\ instState' = StatusAfter({}, {}, {i}) + /\ UNCHANGED << fut, rel, pendingAllocs, pendingReleases, seqCtr, + instOffset, eCreated, allocatedEver, + structuralAssertFailed, unblockFailed, + futMismatch, poisonReplayBad, readyAtRebuild, + dupAlloc, wasDeferred, failedVia >> + ELSE + \* push, no future state yet (cc:857-859); waiter registered (cc:920) + /\ seqCtr' = seqCtr + 1 + /\ pendingReleases' = Append(pendingReleases, + [inst |-> i, isReady |-> FALSE, + seq |-> seqCtr + 1, defNote |-> FALSE]) + /\ UNCHANGED << cur, fut, rel, pendingAllocs, instState, instOffset, + eCreated, allocatedEver, curFreed, missingFree, + structuralAssertFailed, unblockFailed, futMismatch, + poisonReplayBad, readyAtRebuild, dupAlloc, + wasDeferred, failedVia, notifyCount >> + ELSE \* cc:861-897 + IF trig THEN + IF instState[i] = "FAILED" THEN + \* ready destruction of a failed alloc: skip heap (cc:866-868), + \* still acked (cc:915-917) + /\ notifyCount' = NotifyAfter({i}) + /\ instState' = StatusAfter({}, {}, {i}) + /\ UNCHANGED << cur, fut, rel, pendingAllocs, pendingReleases, + seqCtr, instOffset, eCreated, allocatedEver, + curFreed, missingFree, structuralAssertFailed, + unblockFailed, futMismatch, poisonReplayBad, + readyAtRebuild, dupAlloc, wasDeferred, failedVia >> + ELSE + \* cc:871-872 missing_ok=FALSE frees, then ARR (cc:875-876) + LET miss == ~HasTag(rel, i) \/ ~HasTag(fut, i) + rel1 == FreeMissingOk(rel, i) + fut1 == FreeMissingOk(fut, i) + arr == ARRFun(cur, fut1, rel1, pendingAllocs, pendingReleases) + IN IF arr.changed THEN + LET succP == ToSet(arr.placed) + succS == InstsOf(succP) + notifS == arr.notified \cup {i} \* i: cc:915-917; defNote drains: cc:922-923 + IN /\ cur' = arr.cur /\ fut' = arr.fut /\ rel' = arr.rel + /\ pendingAllocs' = arr.pa + /\ pendingReleases' = arr.pr + /\ eCreated' = ECreatedAfter(succS, {}) + /\ instState' = StatusAfter(succS, {}, notifS) + /\ instOffset' = OffsetsAfterPairs(succP, {}) + /\ notifyCount' = NotifyAfter(notifS) + /\ allocatedEver' = allocatedEver \cup succS + /\ curFreed' = curFreed \cup arr.erasedReady \cup {i} + /\ missingFree' = (missingFree \/ miss) + /\ dupAlloc' = (dupAlloc \/ arr.dup) + /\ UNCHANGED << seqCtr, structuralAssertFailed, + unblockFailed, futMismatch, + poisonReplayBad, readyAtRebuild, + wasDeferred, failedVia >> + ELSE + \* unwind failed: push ready entry, defer the ack (cc:884-887) + /\ rel' = rel1 /\ fut' = fut1 + /\ seqCtr' = seqCtr + 1 + /\ pendingReleases' = + Append(pendingReleases, + [inst |-> i, isReady |-> TRUE, + seq |-> seqCtr + 1, defNote |-> TRUE]) + /\ missingFree' = (missingFree \/ miss) + /\ UNCHANGED << cur, pendingAllocs, instState, instOffset, + eCreated, allocatedEver, curFreed, + structuralAssertFailed, unblockFailed, + futMismatch, poisonReplayBad, readyAtRebuild, + dupAlloc, wasDeferred, failedVia, + notifyCount >> + ELSE + \* untriggered with pending allocs (cc:890-896) + /\ fut' = IF instState[i] = "FAILED" THEN fut + ELSE FreeMissingOk(fut, i) \* cc:892-893 missing_ok=TRUE + /\ seqCtr' = seqCtr + 1 + /\ pendingReleases' = Append(pendingReleases, + [inst |-> i, isReady |-> FALSE, + seq |-> seqCtr + 1, defNote |-> FALSE]) + /\ UNCHANGED << cur, rel, pendingAllocs, instState, instOffset, + eCreated, allocatedEver, curFreed, missingFree, + structuralAssertFailed, unblockFailed, futMismatch, + poisonReplayBad, readyAtRebuild, dupAlloc, + wasDeferred, failedVia, notifyCount >> + +----------------------------------------------------------------------------- +(***************************************************************************) +(* Section 4.5: TriggerDestroy(i, pois) *) +(* DeferredDestroy::event_triggered -> release_storage_immediate *) +(* (ii:81-99; cc:1600-1794). Enabled by MC once preD[i] has fired for a *) +(* destroy that was deferred. pois = fired poisoned. *) +(***************************************************************************) + +\* clean, oldest path (cc:1634-1717 + tail ARR cc:1751-1753) +TriggerDestroyOldest(i) == + LET pr0 == pendingReleases + pa0 == pendingAllocs + relMiss == pa0 /= <<>> /\ ~HasTag(rel, i) \* cc:1636 missing_ok=FALSE + rel1 == IF pa0 /= <<>> THEN FreeMissingOk(rel, i) ELSE rel \* cc:1635-1637 + d == DrainLoop(pr0, 1, i, cur, fut, pa0) + pr1 == SubSeq(pr0, d.k, Len(pr0)) \* cc:1702 erase prefix + needRb == d.succ /= <<>> /\ d.pa /= <<>> \* cc:1704-1706 + rb == IF needRb THEN ApplyReadyFreesStrict(d.cur, pr1) \* cc:1707-1717 + ELSE [a |-> rel1, miss |-> FALSE] + arrGo == d.pa /= <<>> \* cc:1751-1753 + arr == IF arrGo THEN ARRFun(d.cur, fut, rb.a, d.pa, pr1) ELSE ARRNone + chg == arrGo /\ arr.changed + cur2 == IF chg THEN arr.cur ELSE d.cur + fut2 == IF chg THEN arr.fut ELSE fut + rel2 == IF chg THEN arr.rel ELSE rb.a + pa2 == IF chg THEN arr.pa ELSE d.pa + pr2 == IF chg THEN arr.pr ELSE pr1 + \* FIX_SWEEP: sweep_ready_releases() when pending_allocs is empty + \* (C++: release_storage_immediate, after the cc:1702 prefix erase + \* and the cc:1751 tail ARR). If pa2 emptied via ARR full-success, + \* arr.pr has no ready entries left and the sweep is a no-op. + sw == IF FIX_SWEEP /\ pa2 = <<>> + THEN Sweep(cur2, pr2) + ELSE [cur |-> cur2, pr |-> pr2, notified |-> {}, + freed |-> {}, miss |-> FALSE] + succP == ToSet(d.succ) \cup (IF chg THEN ToSet(arr.placed) ELSE {}) + succS == InstsOf(succP) + notifS == d.notified \cup (IF chg THEN arr.notified ELSE {}) + \cup sw.notified + \cup {i} \* i: cc:1789-1790; defNote drains: cc:1792-1793 + freedS == d.curFreedSet \cup (IF chg THEN arr.erasedReady ELSE {}) + \cup sw.freed + IN /\ cur' = sw.cur /\ fut' = fut2 /\ rel' = rel2 + /\ pendingAllocs' = pa2 + /\ pendingReleases' = sw.pr + /\ eCreated' = ECreatedAfter(succS, {}) + /\ instState' = StatusAfter(succS, {}, notifS) + /\ instOffset' = OffsetsAfterPairs(succP, {}) + /\ notifyCount' = NotifyAfter(notifS) + /\ allocatedEver' = allocatedEver \cup succS + /\ curFreed' = curFreed \cup freedS + /\ missingFree' = (missingFree \/ relMiss \/ d.missFree \/ rb.miss + \/ sw.miss) + /\ unblockFailed' = (unblockFailed \/ d.ubFail) + /\ futMismatch' = (futMismatch \/ d.futBad) + /\ structuralAssertFailed' = (structuralAssertFailed \/ d.structF) + /\ dupAlloc' = (dupAlloc \/ d.dup \/ (IF chg THEN arr.dup ELSE FALSE)) + /\ UNCHANGED << seqCtr, poisonReplayBad, readyAtRebuild, wasDeferred, + failedVia >> + +\* clean, non-oldest path (cc:1718-1742 + tail ARR cc:1751-1753); the C++ +\* find loop starts at the SECOND entry (cc:1719-1721 pre-increments) +TriggerDestroyNonOldest(i) == + LET pr0 == pendingReleases + ks == {k \in 2..Len(pr0) : pr0[k].inst = i} + IN IF ks = {} THEN + \* cc:1720-1723 off-end assert: flag, no other step is defined + /\ structuralAssertFailed' = TRUE + /\ UNCHANGED << cur, fut, rel, pendingAllocs, pendingReleases, seqCtr, + instState, instOffset, eCreated, allocatedEver, + curFreed, missingFree, unblockFailed, futMismatch, + poisonReplayBad, readyAtRebuild, dupAlloc, wasDeferred, + failedVia, notifyCount >> + ELSE + LET m == Min(ks) + pr1 == [pr0 EXCEPT ![m].isReady = TRUE] \* cc:1724 + IN IF pendingAllocs = <<>> THEN + \* apply to current state directly (cc:1726-1730) + LET miss == ~HasTag(cur, i) \* cc:1728 missing_ok=FALSE + IN /\ cur' = FreeMissingOk(cur, i) + /\ pendingReleases' = RemoveAt(pr1, m) \* cc:1730 + /\ curFreed' = curFreed \cup {i} + /\ missingFree' = (missingFree \/ miss) + /\ notifyCount' = NotifyAfter({i}) \* cc:1789-1790 + /\ instState' = StatusAfter({}, {}, {i}) + /\ UNCHANGED << fut, rel, pendingAllocs, seqCtr, instOffset, + eCreated, allocatedEver, + structuralAssertFailed, unblockFailed, + futMismatch, poisonReplayBad, readyAtRebuild, + dupAlloc, wasDeferred, failedVia >> + ELSE + \* apply to release allocator, defer the ack (cc:1731-1741), + \* then tail ARR (cc:1751-1753) + LET miss == ~HasTag(rel, i) \* cc:1738 missing_ok=FALSE + rel1 == FreeMissingOk(rel, i) + pr2 == [pr1 EXCEPT ![m].defNote = TRUE] \* cc:1739 + arr == ARRFun(cur, fut, rel1, pendingAllocs, pr2) + IN IF arr.changed THEN + LET succP == ToSet(arr.placed) + succS == InstsOf(succP) + notifS == arr.notified \* i included iff its entry drained (cc:1792-1793) + IN /\ cur' = arr.cur /\ fut' = arr.fut /\ rel' = arr.rel + /\ pendingAllocs' = arr.pa + /\ pendingReleases' = arr.pr + /\ eCreated' = ECreatedAfter(succS, {}) + /\ instState' = StatusAfter(succS, {}, notifS) + /\ instOffset' = OffsetsAfterPairs(succP, {}) + /\ notifyCount' = NotifyAfter(notifS) + /\ allocatedEver' = allocatedEver \cup succS + /\ curFreed' = curFreed \cup arr.erasedReady + /\ missingFree' = (missingFree \/ miss) + /\ dupAlloc' = (dupAlloc \/ arr.dup) + /\ UNCHANGED << seqCtr, structuralAssertFailed, + unblockFailed, futMismatch, + poisonReplayBad, readyAtRebuild, + wasDeferred, failedVia >> + ELSE + /\ rel' = rel1 + /\ pendingReleases' = pr2 + /\ missingFree' = (missingFree \/ miss) + /\ UNCHANGED << cur, fut, pendingAllocs, seqCtr, instState, + instOffset, eCreated, allocatedEver, + curFreed, structuralAssertFailed, + unblockFailed, futMismatch, poisonReplayBad, + readyAtRebuild, dupAlloc, wasDeferred, + failedVia, notifyCount >> + +\* poisoned path: remove_pending_release (cc:1538-1597 via cc:1754-1755); +\* no notify_deallocation (cc:1789 guard) +TriggerDestroyPoisoned(i) == + IF pendingAllocs = <<>> THEN + \* cc:1544-1553: erase first entry for i; off-end assert cc:1548-1551 + LET ks == {k \in 1..Len(pendingReleases) : pendingReleases[k].inst = i} + IN IF ks = {} THEN + /\ structuralAssertFailed' = TRUE + /\ UNCHANGED << cur, fut, rel, pendingAllocs, pendingReleases, + seqCtr, instState, instOffset, eCreated, + allocatedEver, curFreed, missingFree, unblockFailed, + futMismatch, poisonReplayBad, readyAtRebuild, + dupAlloc, wasDeferred, failedVia, notifyCount >> + ELSE + /\ pendingReleases' = RemoveAt(pendingReleases, Min(ks)) + /\ UNCHANGED << cur, fut, rel, pendingAllocs, seqCtr, instState, + instOffset, eCreated, allocatedEver, curFreed, + missingFree, structuralAssertFailed, unblockFailed, + futMismatch, poisonReplayBad, readyAtRebuild, + dupAlloc, wasDeferred, failedVia, notifyCount >> + ELSE + \* cc:1554-1596: rewrite future history; legacy: rel := cur WITHOUT + \* ready releases (BUG-4, kept faithful when FIX_SWEEP = FALSE); no + \* trailing alloc replay (BUG-5, unchanged by these fixes) + LET L == RPRLoop(pendingReleases, 1, i, FALSE, cur, pendingAllocs) + \* FIX_RPR (BUG-5): trailing replay AFTER the outer walk ends + \* (cc:1595) - the cc:1579-1594 inner loop once more, no seqid + \* bound, against the rebuilt fut. It CONTINUES from the walk's + \* final alloc cursor: only L.trail (the never-examined remainder) + \* is processed - feeding it the walk-KEPT prefix re-DoAllocs + \* already-placed allocs (the sapling SafetyFixed4 dupAlloc + \* artifact, bugs/DUPALLOC-TRIAGE.md). FALSE = identity. + tr == IF FIX_RPR THEN TrailingRPR(L.fut, L.trail) + ELSE [fut |-> L.fut, kept |-> L.trail, failed |-> {}, + dup |-> FALSE] + \* L.paOut = KeptPrefix \o L.trail by construction, so this exactly + \* reattaches the walk-kept prefix ahead of the trailing survivors + \* (queue order preserved); with FIX_RPR = FALSE, paF = L.paOut. + KeptPrefix == SubSeq(L.paOut, 1, Len(L.paOut) - Len(L.trail)) + paF == IF FIX_RPR THEN KeptPrefix \o tr.kept ELSE L.paOut + failS == L.failed \cup tr.failed + \* FIX_SWEEP: sweep_ready_releases() when the queue is empty + \* (C++: remove_pending_release, after the cc:1562-1595 walk AND + \* after the FIX_RPR trailing pass). Swept defNote acks fire like + \* the cc:1792-1793 loop. COMPOSITION POINT (verified): the sweep + \* condition and input use paF, the post-trailing queue - if the + \* trailing pass fails every remaining alloc and empties the queue, + \* the sweep still runs; C++ must sequence sweep after the trailing + \* replay for the same reason. + sw == IF FIX_SWEEP /\ paF = <<>> + THEN Sweep(cur, L.pr) + ELSE [cur |-> cur, pr |-> L.pr, notified |-> {}, + freed |-> {}, miss |-> FALSE] + \* FIX_SWEEP (BUG-4-standalone re-apply): when pending_allocs stays + \* NONEMPTY (post-trailing), re-apply surviving is_ready entries to + \* the rebuilt rel (mirrors cc:1713-1714) so rel = current + ready + \* releases (h:399-405) holds again. rel is invalid when the queue + \* emptied, so the sweep branch leaves it at the legacy cc:1557 + \* value. + relRe == IF FIX_SWEEP /\ paF /= <<>> + THEN ApplyReadyFreesStrict(cur, L.pr) + ELSE [a |-> cur, miss |-> FALSE] + IN /\ fut' = tr.fut + /\ rel' = relRe.a \* cc:1557 (legacy: plain cur) + /\ cur' = sw.cur + /\ pendingReleases' = sw.pr + /\ pendingAllocs' = paF + /\ eCreated' = ECreatedAfter({}, failS) \* cc:1591; ii:1121-1122 + /\ instState' = StatusAfter({}, failS, sw.notified) + /\ instOffset' = OffsetsAfter(cur, {}, failS) + /\ failedVia' = FailedViaAfter(failS, "RPR") + /\ notifyCount' = NotifyAfter(sw.notified) + /\ curFreed' = curFreed \cup sw.freed + /\ missingFree' = (missingFree \/ sw.miss \/ relRe.miss) + /\ poisonReplayBad' = (poisonReplayBad \/ L.bad) + \* FIX 2: cc:1542 assert(!pending_releases.empty()) on entry + /\ structuralAssertFailed' = + (structuralAssertFailed \/ pendingReleases = <<>>) + /\ dupAlloc' = (dupAlloc \/ L.dup \/ tr.dup) + /\ UNCHANGED << seqCtr, allocatedEver, + unblockFailed, futMismatch, readyAtRebuild, + wasDeferred >> + +TriggerDestroy(i, pois) == + /\ UNCHANGED reqCap \* release path never touches the FIX_CAP snapshot + /\ (IF pois THEN TriggerDestroyPoisoned(i) + ELSE IF pendingReleases = <<>> THEN + \* cc:1630 assert(!pending_releases.empty()): flag, no defined step + /\ structuralAssertFailed' = TRUE + /\ UNCHANGED << cur, fut, rel, pendingAllocs, pendingReleases, seqCtr, + instState, instOffset, eCreated, allocatedEver, + curFreed, missingFree, unblockFailed, futMismatch, + poisonReplayBad, readyAtRebuild, dupAlloc, + wasDeferred, failedVia, notifyCount >> + ELSE IF Head(pendingReleases).inst = i \* cc:1634 + THEN TriggerDestroyOldest(i) + ELSE TriggerDestroyNonOldest(i)) + +----------------------------------------------------------------------------- +(***************************************************************************) +(* Section 6: invariants (DESIGN 6). MC selects which to check per config.*) +(***************************************************************************) + +Cells(a, t) == a[t].first .. (a[t].first + a[t].size - 1) + +UsedCells(a) == UNION {Cells(a, t) : t \in DOMAIN a} + +RECURSIVE SumSizes(_) +SumSizes(a) == IF DOMAIN a = {} THEN 0 + ELSE LET t == CHOOSE t \in DOMAIN a : TRUE + IN a[t].size + SumSizes(DoFree(a, t)) + +INV_NoOverlap == + \A t1, t2 \in DOMAIN cur : + t1 /= t2 => Cells(cur, t1) \cap Cells(cur, t2) = {} + +INV_InBounds == + \A t \in DOMAIN cur : + cur[t].first >= 0 /\ cur[t].first + cur[t].size <= HEAP_SIZE + +\* spec-typo catcher: with no overlap and in-bounds this is an equality +INV_Conservation == Cardinality(UsedCells(cur)) = SumSizes(cur) + +INV_CurrentMatchesGround == DOMAIN cur = allocatedEver \ curFreed + +\* strengthened cc:772 assert; EXPECTED TO FAIL with FIX_SWEEP = FALSE +\* (BUG-6, DESIGN 6/8) and EXPECTED TO HOLD with FIX_SWEEP = TRUE (the +\* sweep drains every stranded ready entry - that is the fix's claim) +INV_NoReadyWhenNoPendingAllocs == + pendingAllocs = <<>> => + \A k \in 1..Len(pendingReleases) : ~pendingReleases[k].isReady + +\* companion: the exact in-ADA rebuild-site check (cc:772); expected FAIL +\* with FIX_SWEEP = FALSE, expected HOLD with FIX_SWEEP = TRUE. Note the +\* FIX_CAP admission path has no cc:768 rebuild, so under FIX_CAP this flag +\* can only be set by the legacy path (i.e. never when FIX_CAP = TRUE). +INV_NoReadyAtRebuild == ~readyAtRebuild + +\* all missing_ok=FALSE frees found their tag (inl:614): +\* cc:855, cc:871-872, cc:1636, cc:1641, cc:1713-1714, cc:1728, cc:1738 +INV_TriggeredDeallocPresent == ~missingFree + +INV_StructuralAsserts == ~structuralAssertFailed + +INV_NoOrphanTags == + \A t \in DOMAIN cur : + \/ notifyCount[t] = 0 + \/ \E k \in 1..Len(pendingReleases) : pendingReleases[k].inst = t + +INV_InOrderUnblockSucceeds == ~unblockFailed \* cc:1670 + +\* cc:1674-1691; under FIX_CAP the canonically maintained fut is CLAIMED to +\* agree with drain placement - a violation with FIX_CAP = TRUE is a fix bug +INV_FutureOffsetConsistency == ~futMismatch + +INV_PoisonReplayOnlyFailsAfterPoint == ~poisonReplayBad \* cc:1587 + +\* FIX 3 detector: the C++ allocate() on a live tag leaks the old range +\* (allocated[tag] = idx, inl:500 - new range wins, old range stays linked: +\* the #442 double-tracking class). The partial-function representation +\* cannot express the leak, so any materialized DoAlloc onto a live tag is +\* flagged instead of modeled. +INV_NoDupAlloc == ~dupAlloc + +\* a DEFERRED promise fails only via RemovePendingRelease (DESIGN 6) +SAFETY_PromisesKept == + \A j \in wasDeferred : instState[j] = "FAILED" => failedVia[j] = "RPR" + +PROP_NotifyOnceSafety == \A j \in INSTANCES : notifyCount[j] <= 1 + +----------------------------------------------------------------------------- +(* type and init *) + +AllocatorType(a) == + /\ DOMAIN a \subseteq INSTANCES + /\ \A t \in DOMAIN a : a[t] \in [first : 0..HEAP_SIZE, size : 1..HEAP_SIZE] + +TypeOK == + /\ AllocatorType(cur) /\ AllocatorType(fut) /\ AllocatorType(rel) + /\ \A k \in 1..Len(pendingAllocs) : + pendingAllocs[k] \in [inst : INSTANCES, size : 1..HEAP_SIZE, + lastSeq : Nat] + /\ \A k \in 1..Len(pendingReleases) : + pendingReleases[k] \in [inst : INSTANCES, isReady : BOOLEAN, + seq : Nat, defNote : BOOLEAN] + /\ seqCtr \in Nat + /\ reqCap \in [INSTANCES -> Nat] + /\ instState \in [INSTANCES -> Statuses] + /\ instOffset \in [INSTANCES -> -2..HEAP_SIZE] + /\ eCreated \in [INSTANCES -> {"UNFIRED", "CLEAN", "POISONED"}] + /\ allocatedEver \subseteq INSTANCES /\ curFreed \subseteq INSTANCES + /\ wasDeferred \subseteq INSTANCES + /\ missingFree \in BOOLEAN /\ structuralAssertFailed \in BOOLEAN + /\ unblockFailed \in BOOLEAN /\ futMismatch \in BOOLEAN + /\ poisonReplayBad \in BOOLEAN /\ readyAtRebuild \in BOOLEAN + /\ dupAlloc \in BOOLEAN + /\ failedVia \in [INSTANCES -> {"NONE", "INSTANT", "CANCELLED", "RPR"}] + /\ notifyCount \in [INSTANCES -> Nat] + +InitProto == + /\ cur = EmptyAlloc /\ fut = EmptyAlloc /\ rel = EmptyAlloc \* cc:680; fut/rel default-constructed, written before any read (DESIGN 3) + /\ pendingAllocs = <<>> /\ pendingReleases = <<>> + /\ seqCtr = 0 \* cc:675 + /\ reqCap = [j \in INSTANCES |-> 0] + /\ instState = [j \in INSTANCES |-> "UNREQUESTED"] + /\ instOffset = [j \in INSTANCES |-> OFF_NONE] + /\ eCreated = [j \in INSTANCES |-> "UNFIRED"] + /\ allocatedEver = {} /\ curFreed = {} /\ wasDeferred = {} + /\ missingFree = FALSE /\ structuralAssertFailed = FALSE + /\ unblockFailed = FALSE /\ futMismatch = FALSE + /\ poisonReplayBad = FALSE /\ readyAtRebuild = FALSE /\ dupAlloc = FALSE + /\ failedVia = [j \in INSTANCES |-> "NONE"] + /\ notifyCount = [j \in INSTANCES |-> 0] + +============================================================================= diff --git a/tla/allocation/EXPECTED.md b/tla/allocation/EXPECTED.md new file mode 100644 index 0000000000..6a2f4cbcb3 --- /dev/null +++ b/tla/allocation/EXPECTED.md @@ -0,0 +1,189 @@ +# Expected TLC outcomes per configuration + +Drive file for the Phase 4 verification loop. Every run is judged against +this table; anything off-table is either a spec bug or an unregistered Realm +bug candidate — triage against DESIGN.md §8 before assuming either. + +All configs check `MCDeferredAlloc.tla` (INIT/NEXT `MCInit`/`MCNext` unless +the row says SPECIFICATION). "dlk ON" = run TLC **without** `-deadlock` +(deadlock checking enabled); "dlk OFF" = pass `-deadlock`. Constant +substitution uses the protocol module's `Size` constant (`Size <- SizesX`). + +Run mechanics (validated 2026-08-25): SANY and TLC work with +`/opt/homebrew/opt/openjdk/bin/java` + `../barrier/tools/tla2tools.jar`. +TLC must run **outside the bash sandbox** (its `states/` metadir and trace +writes are blocked by the sandbox write allowlist; no RMI issue was seen with +`-Djava.io.tmpdir=`). **Exception found in Phase 4:** liveness +(SPECIFICATION/temporal) runs additionally bind a local RMI socket at startup +and die under the sandbox with `java.rmi.server.ExportException: Listen +failed on port: 0 / Operation not permitted` even with writable tmp/metadir — +temporal configs (Liveness, LivenessNoCross) strictly require an +unsandboxed JVM; safety-only configs only need writable tmp/metadir paths. + +Post-Phase-3 fix batch (FIX 1-5, 2026-08-25, applied to DeferredAlloc.tla): +instOffset is now built from placement-time offsets carried through the +drain/ARR helpers (cc:1668/cc:1693 semantics; the old final-allocator re-read +aborted TLC in C2-off configs); the cc:1542 entry assert is ghost-flagged; +the DELAYEDDESTROY-with-triggered-preD branch now acks (cc:915-917 +release-build behavior); and a new **`INV_NoDupAlloc`** detector (`dupAlloc` +ghost; inl:500 duplicate-tag leak, the #442 class — the partial-function +representation cannot express the leak itself) was added to the +Safety/SafetyMini/Poison4/Big batteries. Expected green in contract-ON +configs; a violation is a new bug candidate, not noise. Baseline outcome +classes below re-validated after the batch (state counts may shift slightly). + +| Config | Extra TLC flags | Checks | Expected outcome | Status / if it deviates | +|---|---|---|---|---| +| Smoke | *(none — dlk ON)* | green battery (expected-FAIL invariants BUG-6 excluded) | **PASS or a BUG-1-shape deadlock** (both acceptable; BUG-1 is reachable even here). | **RAN: deadlock at depth 7** (4029 states) — BUG-1 shape via the DELAYEDDESTROY variant; trace `traces/Smoke-run1.txt`. Any **invariant** violation ⇒ spec bug. | +| EventLoop | *(none — dlk ON)* | dlk + small green set | **FAIL: deadlock**, 7 states, matching the hand-simulated trace in MCDeferredAlloc.tla's trailer. | **RAN: CONFIRMED** — deadlock in exactly 7 states (57 generated), final state `instState = `, `pendingAllocs` waiting on a release whose precondition needs `eCreated(2)`. Trace `traces/EventLoop.trace.txt`. **BUG-1 confirmed by TLC.** | +| Safety | `-deadlock` | full battery incl. INV_NoReadyWhenNoPendingAllocs + INV_NoReadyAtRebuild | **FAIL: INV_NoReadyWhenNoPendingAllocs** (BUG-6). | **RAN ON SAPLING (round 1, job 77808): CONFIRMED** — INV_NoReadyWhenNoPendingAllocs violated at depth 10 after 4.4B generated / 1.44B distinct, 5h04m, 40 workers. **BUG-6(a) confirmed at 4-instance scale.** Trace in `slurm-77808-Safety.out`. TLC halted there, so the deeper BUG-3 hunt was PREEMPTED → superseded by **SafetyHunt.cfg** (round 2). | +| SafetyMini | `-deadlock` | same battery, 3 inst / H=3 / sizes (2,2,1) | **FAIL: INV_NoReadyWhenNoPendingAllocs** (BUG-6, canonical witness). | **RAN: CONFIRMED in 10s** (5.5M states, violation at depth 9). Final state: `pendingAllocs = <<>>`, `pendingReleases = [destroy(2) ¬ready, destroy(3) READY+defNote]` — the stranded-ready shape, action-for-action the canonical witness (destroy(3) triggered at request in this variant). Trace `traces/SafetyMini.trace.txt`. **BUG-6 confirmed by TLC.** | +| SafetyHunt (was "Safety iter. 2") | `-deadlock` | battery minus the two BUG-6 invariants (checked-in cfg, round 2) | **GREEN = BUG-3 absent at these bounds**; a hit on INV_InOrderUnblockSucceeds / INV_FutureOffsetConsistency is the first BUG-3 witness | Not yet run (sapling round 2). Anything found is a fresh Realm bug candidate → Phase 5. Likely runs longer than Safety's 5h04m — it will not stop early; use `-recover`. | +| PoisonHunt | `-deadlock` | Poison4's battery minus the two BUG-6 invariants (checked-in cfg, round 2) | a hit = first witness of **BUG-4-standalone** (INV_NoOrphanTags / INV_CurrentMatchesGround / INV_QuiescentHeapEmpty), **unfixed BUG-5** (INV_FutureOffsetConsistency / INV_InOrderUnblockSucceeds / INV_NoOverlap), or **cc:1587** (INV_PoisonReplayOnlyFailsAfterPoint); GREEN = absent at these bounds | Not yet run (sapling round 2). | +| Liveness | `-deadlock` | SPECIFICATION LiveSpec; PROPERTY LIVE_NoStuckAllocs | **FAIL: LIVE_NoStuckAllocs** via a BUG-1 lasso. | **RAN: CONFIRMED** at reduced bounds (2 inst / H=3, SizesEventLoop — the original 3-inst/H=4 run's throughput collapsed ~5x from behavior-graph maintenance at ~500k distinct/10 min and was stopped; reduction documented in the cfg header). 8-action counterexample ending in **Stuttering**: `instState[1] = ALLOC_DEFERRED`, its enabling release destroy(2) has `preD[2].deps = {1,2}` with `eCreated(1)` UNFIRED — the BUG-1 wait cycle. 7,273 gen / 3,531 distinct, seconds. Trace `traces/Liveness-bug1.txt`. | +| LivenessNoCross (confirmatory) | `-deadlock`; own cfg (`LivenessNoCross.cfg`, `CLIENT_MODE = "NO_CROSS_DEPS"`, same bounds as Liveness) | same | **PASS** (BUG-1 shape excluded). | **RAN: PASS** — complete state space, no error (3,929 gen / 1,987 distinct, seconds). Same-bounds control: the Liveness failure is specifically the cross-instance destroy dependency cycle. Log `traces/LivenessNoCross-pass.txt`. A future FAIL would be a *second* liveness bug → Phase 5, high interest. | +| Composite4 | `-deadlock` | 5 inst / H=4 / sizes (2,2,1,1,1), `CLIENT_MODE = "SCRIPTED_COMPOSITE"`; battery minus the two BUG-6 markers | **FAIL: INV_CurrentMatchesGround / INV_NoOrphanTags** — the BUG-6→BUG-4 composite leak (bugs/BUG-6.md item 6). | **RAN: CONFIRMED in 5s** (93,795 gen / 51,904 distinct, violation at depth 12). Final state: tag 3 in `cur`, `instState[3]=DESTROYED`, `notifyCount[3]=1`, no pendingReleases entry, `readyAtRebuild=TRUE` (path went through the BUG-6 stranding) — permanent range leak + notify-while-tag-live, **no poison anywhere**. TLC reported INV_CurrentMatchesGround (first in battery order); INV_NoOrphanTags is violated in the same state. Trace `traces/Composite4.txt`. The predicted composite is now machine-confirmed. | +| Poison4 | `-deadlock` | full battery, USER_POISON | **FAIL: INV_NoReadyWhenNoPendingAllocs / INV_NoReadyAtRebuild** first (BUG-6). | **RAN ON SAPLING (round 1, job 77809): CONFIRMED** — INV_NoReadyWhenNoPendingAllocs violated after 887M generated / 272M distinct, 47min. **BUG-6 confirmed on the poison paths.** Trace in `slurm-77809-Poison4.out`. TLC halted there, so the BUG-4-standalone / unfixed-BUG-5 / cc:1587 hunts were PREEMPTED → superseded by **PoisonHunt.cfg** (round 2). | +| Big | `-deadlock` | full battery, USER_POISON, 5 inst / H=6 | Known violations first (comment out to go deeper); then open hunt. | **SAPLING ROUND 1 (job 77812): IN PROGRESS / RESUMABLE** — no violation through 6.26B generated / 2.87B distinct at depth 9 (~12h in). Resume with `-recover` (see SAPLING_JOBS.md) or check `squeue` — it may still be running. Anything new ⇒ Phase 5 with full trace. | + +## Canonical BUG-6 witness (3 instances, H=3 — found by TLC in Phase 2) + +1. destroy(i1) requested, deferred → R1 (¬ready). +2. create(i2) → DEFERRED, `lastSeq = seq(R1)`. +3. destroy(i2) requested, deferred → R2 (legal: request ≠ trigger; C2 only + constrains when the precondition can *fire*). +4. destroy(i3) **triggered** → cc:871 frees rel/fut; ARR front-gate fails → + R3 pushed **READY** (cc:884-887). +5. trigger destroy(i1) → oldest drain frees i1, unblock scan places i2 + (`i2.lastSeq < seq(R2)`), `pendingAllocs` empties; the do-while stops at + non-ready R2 → **ready R3 stranded** behind it with no pending allocs. +6. Any later alloc request that reaches the cc:768 rebuild trips the cc:772 + `assert(!it->is_ready)` (INV_NoReadyAtRebuild); the stranded state itself + violates INV_NoReadyWhenNoPendingAllocs. + +## Iteration protocol (Phase 4) + +1. Order: Smoke → EventLoop → Safety → Liveness → Poison4 → Big. +2. On an **expected** violation: save the trace (TLC stdout) under + `traces/-.txt` (or `.trace.txt`), comment out that + invariant/property in the cfg, re-run, repeat until the config passes or + produces an unexpected result. +3. On an **unexpected** violation: stop iterating that config; hand the trace + to Phase 5. Do not "fix" the spec to make a violation disappear without a + line-level fidelity argument against mem_impl.cc. +4. A TLC *error* (parse, undefined name, type) is a Phase 2 seam issue — the + Phase 2 reconciliation is complete, so treat any new one as a regression. + +## Known model caveats (do not misread as Realm bugs) + +- **Poisoned-destroy leaks are legal terminals.** A destroy whose + precondition fires poisoned is silently cancelled (cc:818-825) or removed + (cc:1754); the instance stays ALLOCATED with its tag in `cur` forever + ("POSSIBLE LEAK", ii:87). `Quiescent` admits this via + `DestroyResolvedLeak`, and `INV_QuiescentHeapEmpty` permits exactly those + tags — a BUG-4-stranded tag (DESTROYED/notified instance) is still flagged. + With intrinsic poison + C2 this is reachable in every config, including + Smoke. +- `fut`/`rel` are initialized to the empty-domain allocator, which under the + derived-gap representation reads as **all-free**, whereas the code's + default-constructed allocators have **no managed range** (CanAlloc ≡ false; + only `current_allocator` gets `add_range`, cc:680). Neither is read before + first assignment under the code's validity convention; a trace that reads + them earlier is a genuine staleness finding, but its concrete allocator + values in that window will not match the C++. +- Smoke/EventLoop keep deadlock checking ON; Safety/Poison4/Big deliberately + run `-deadlock` so short BUG-1 deadlock traces don't preempt deeper + invariant hunts (TLC halts at the first violation). This deviates from the + "dlk" column of DESIGN.md §7's table for Safety/Poison4/Big — intentional. +- With deadlock ON, Smoke halts at its first BUG-1-shape deadlock, so its + invariant coverage is truncated; Smoke is a parse/typecheck/fast-sanity + gate, not a coverage run. +- `SeqCtrBound` (`seqCtr <= 2·|INSTANCES|`) is a backstop CONSTRAINT in every + config; in v1 (one release per instance) it should never bind. If a run + reports states being constrained away, that itself is a finding. + +## Fix validation (v-next) — three-toggle bundle + +Three spec-side toggles model the candidate fixes before any C++ changes +(all declared in DeferredAlloc.tla): + +- **FIX_CAP** — BUG-1 capped admission: request-time seqid snapshot + (`reqCap`), capped canonical-order admission test, monotone-cap guard, + capped-fail → ALLOC_INSTANT_FAILURE. Pure cap — **no ready-fold in v1**. +- **FIX_SWEEP** — BUG-6 stranded-ready sweep at the pendingAllocs→empty + transitions, plus the BUG-4-standalone `rel` re-apply in + remove_pending_release. +- **FIX_RPR** — BUG-5 close: remove_pending_release's replay processes + **trailing** pending allocs after the walk (place-or-EVENTUAL_FAIL), so a + poisoned release can no longer strand an admitted alloc it funded. + +**Composition discovery — explicit C++ gate:** the first validation round +proved **FIX_CAP without FIX_RPR is NOT shippable**. The cap increases +poisoned-release frequency (honest capped failures poison eCreated → +dependent destroy preconditions fire poisoned → remove_pending_release runs +more often), which makes BUG-5's trailing-alloc hole load-bearing for drain +liveness: Inversion with deadlock checking ON deadlocked via a trailing +alloc that the replay never revisited — neither failed nor refunded +(witness kept: `traces/Inversion-bug5-deadlock.txt`; it doubles as the +canonical 3-instance BUG-5 liveness witness). The verified bundle is +**CAP + SWEEP + RPR**: any C++ landing must take all three together (at +minimum, the cap is strictly gated on the RPR trailing replay). At +2-instance bounds the stranding shape is unreachable (C1 request order + +the cap leave no admissible trailing alloc), which is why the 2-instance +bundle configs pass their deadlock/liveness checks regardless. + +**Battery hardening (post round-1 triage):** `INV_NoDupAlloc` is now checked +in **every** bundle config (SmokeFixed, EventLoopFixed, EventLoopCapOnly, +LivenessFixed, SafetyMiniFixed, SafetyMiniSweepOnly, Composite4Fixed, +GCRipple, Inversion — plus the sapling three, where it was already present). +Round 1's wiring bug was caught first at 4-instance sapling scale precisely +because no local bundle config checked the detector; that coverage gap is +closed. Local bundle configs should re-run green with the corrected +TrailingRPR wiring before any sapling resubmission. + +Toggle pinning: every non-bundle config pins `FIX_RPR = FALSE` (regression +semantics unchanged — the constant reproduces pre-fix behavior when FALSE). +The attribution pair deliberately stays partial: EventLoopCapOnly +(CAP only — sound because its 2-instance bounds cannot reach the BUG-5 +shape) and SafetyMiniSweepOnly (SWEEP only — sound because without the cap +its bounds produce no poisoned funding releases; flags are `-deadlock` +regardless). + +| Config | CAP | SWEEP | RPR | Expectation | RESULT — two-toggle round (2026-08-26); re-validation with FIX_RPR pending | +|---|---|---|---|---|---| +| SmokeFixed | on | on | on | **fully green incl. deadlock check**; BUG-6 invariants checked and passing | PASS, complete — 13,303 gen / 6,337 distinct, depth 13, ~1s | +| EventLoopFixed | on | on | on | green: BUG-1 client's create INSTANT-FAILS, cascade drains, deadlock check passing | PASS, complete — 71 gen / 47 distinct, depth 9 | +| EventLoopCapOnly | on | off | off | green — **BUG-1 fixed by the cap alone** (BUG-5 shape unreachable at 2 inst) | PASS, complete — identical counts to EventLoopFixed; attribution confirmed | +| LivenessFixed | on | on | on | **LIVE_NoStuckAllocs PASSES** (FAILED is in the resolved target set) | PASS — no error, 13,303 gen / 6,337 distinct, 41s; same bounds where base FAILS | +| SafetyMiniFixed | on | on | on | fully green; BUG-6 pair in battery and passing at the base's 10s-fail bounds | PASS, complete exhaustion — 64.7M gen / 23.5M distinct, depth 19, 5m40s | +| SafetyMiniSweepOnly | off | on | off | green — **BUG-6 fixed by the sweep alone** | PASS, complete exhaustion — 53.3M gen / 20.5M distinct, depth 19, 5m33s | +| Composite4Fixed | on | on | on | fully green; INV_NoOrphanTags + INV_CurrentMatchesGround passing, BUG-6 pair added and passing | PASS, complete — 245,493 gen / 119,728 distinct, depth 16, ~10s | +| GCRipple | on | on | on | green; both intent invariants hold; both end classes reachable (honest INSTANT-FAIL = the **accepted behavior change** of the pure cap) | PASS after one harness fix (create-order guard added to `CreateOrderOK`; misfire trace `traces/GCRipple-orderguard-misfire.txt`) — 72 gen / 43 distinct, depth 10 | +| Inversion | on | on | on | **GREEN over the full space with deadlock checking ON**: the cap/monotone guard instant-fails A in every interleaving, B ends **EVENTUAL_FAILURE cleanly via the RPR trailing replay**, every interleaving drains; INV_InversionCapped + SAFETY_PromisesKept hold. A deadlock here ⇒ **FIX_RPR design bug** (trailing replay missed a case). | two-toggle round: `-deadlock` PASS full space (479/257); deadlock-ON **deadlocked = the BUG-5 composition witness** (see gate above). Superseded by the bundle expectation. | +| Inversion (CAP+SWEEP, RPR=FALSE) — historical, not a checked-in config | on | on | off | *documents the gate:* reproduces the BUG-5 stranding as a deadlock — **FIX_CAP without FIX_RPR is not shippable** | witness: `traces/Inversion-bug5-deadlock.txt` | + +Regression sweep (all toggles FALSE) re-validated on the two-toggle round: +Smoke deadlock @ 7 ✓, EventLoop deadlock @ 7 ✓, SafetyMini +INV_NoReadyWhenNoPendingAllocs @ 9 ✓, Composite4 INV_CurrentMatchesGround +@ 12 ✓. FIX_RPR = FALSE preserves pre-fix RPR behavior, so these +expectations carry over unchanged; the verification pass should spot-check +one of them after the FIX_RPR spec edit lands. + +### Sapling fix-validation configs (created by the verification fork) + +| Config | CAP | SWEEP | RPR | Expectation | Round-1 result (2026-08-26) | +|---|---|---|---|---|---| +| SafetyFixed4 | on | on | on | **FULLY GREEN** — BUG-5 detectors included (the bundle now addresses BUG-5); any violation = fix-design bug or new candidate | **VIOLATED: INV_NoDupAlloc** at depth 12 (job 77810; 12.8B generated / 4.0B distinct, 14h10m; trace in `slurm-77810-SafetyFixed4.out` from line 901). **TRIAGED (bugs/DUPALLOC-TRIAGE.md): verdict (a) — spec artifact.** The FIX_RPR *call-site wiring* fed `TrailingRPR` the full survivor list instead of the trailing remainder (a two-line DeferredAlloc.tla correction, applied); the fix DESIGN is unaffected. The same wiring bug has a second in-model flavor — a kept (already-replayed) alloc spuriously EVENTUAL_FAILED with its placement stranded in `fut` — covered by the same correction. **Corrected + locally re-validated (2026-08-26): full local matrix green on the corrected spec — fast set incl. Inversion deadlock-ON (478/255), SafetyMiniFixed exhaustion 64.68M/23.54M, SweepOnly 53.30M/20.55M, LivenessFixed pass, toggles-off regressions exact. Fresh resubmission pending** (spec changed → old checkpoints invalid). | +| Poison4Fixed | on | on | on | **FULLY GREEN** — BUG-5 detectors included (flipped from "expected-possible" now that FIX_RPR closes BUG-5); a BUG-5-detector hit here ⇒ FIX_RPR design bug on the poison paths | **DID NOT RUN** — round-1 submission typo (`PoisonFixed4` instead of `Poison4Fixed`, job 77811 exited immediately: "no such config"). Resubmit with the correct name **after the TrailingRPR wiring correction re-validates locally** (fresh start; it runs the corrected bundle). | +| BigFixed | on | on | on | FULLY GREEN on the BUG-1/4/5/6 families; anything else = new bug candidate → Phase 5 | **HELD** (was: in progress, job 77813, clean through 1.06B generated / 470M distinct at depth 9). Round-1 progress checked the dup detector against the STALE TrailingRPR wiring, and the spec correction invalidates the checkpoint — restart fresh after local re-validation; do not `-recover`. | + +## Sapling round 1 summary (2026-08-26) + +| Job | Config | Toggles | Outcome | +|---|---|---|---| +| 77808 | Safety | off | **BUG-6(a) confirmed at 4-inst scale** (INV_NoReadyWhenNoPendingAllocs, depth 10; 4.4B gen / 1.44B distinct, 5h04m). Expected. BUG-3 hunt preempted → SafetyHunt (round 2). | +| 77809 | Poison4 | off | **BUG-6 confirmed on poison paths** (887M gen / 272M distinct, 47min). Expected. BUG-4-standalone / unfixed-BUG-5 / cc:1587 hunts preempted → PoisonHunt (round 2). | +| 77810 | SafetyFixed4 | bundle | **UNEXPECTED: INV_NoDupAlloc violated** at depth 12 (12.8B gen / 4.0B distinct, 14h10m). **Triaged: spec artifact** — TrailingRPR call-site wiring, corrected; fresh resubmission after local re-validation. | +| 77811 | (Poison4Fixed) | bundle | **Did not run** — submission typo `PoisonFixed4`. Resubmit round 2 (after local re-validation of the wiring correction). | +| 77812 | Big | off | In progress / resumable — clean through 6.26B gen / 2.87B distinct, depth 9. Toggles-off: unaffected by the wiring correction; resume freely. | +| 77813 | BigFixed | bundle | Was in progress (clean through 1.06B gen / 470M distinct, depth 9) — **HELD**: checkpoint invalidated by the spec correction; restart fresh after local re-validation. | diff --git a/tla/allocation/EventLoop.cfg b/tla/allocation/EventLoop.cfg new file mode 100644 index 0000000000..5cb1ca5898 --- /dev/null +++ b/tla/allocation/EventLoop.cfg @@ -0,0 +1,34 @@ +\* EventLoop: minimal scripted reproduction of BUG-1 (deferred create is +\* ordered at trigger time, not request time -> event-loop deadlock). +\* Client is hardcoded to the DESIGN.md s5 worked example: +\* preC(1)=NOW, preC(2)=ballistic, preD(1)=deps{1,2}, preD(2)=deps{2}; +\* sizes 3,3 on H=3 so instance 2 can only be planned into 1's space. +\* TLC flags: NONE (deadlock checking ON). +\* Expected: FAIL - TLC deadlock counterexample, ~7 steps, matching the +\* hand-simulated trace in MCDeferredAlloc.tla's trailer comment. +\* A clean pass here means the model LOST the bug (spec error). +\* Scale: local, seconds. +CONSTANTS + HEAP_SIZE = 3 + INSTANCES = {1, 2} + Size <- SizesEventLoop + USER_POISON = FALSE + C1_ENABLED = TRUE + C2_ENABLED = TRUE + C3_ENABLED = FALSE + CLIENT_MODE = "SCRIPTED_EVENTLOOP" + FIX_CAP = FALSE + FIX_SWEEP = FALSE + FIX_RPR = FALSE +INIT MCInit +NEXT MCNext +INVARIANTS + TypeOK + INV_NoOverlap + INV_InBounds + INV_Conservation + INV_CurrentMatchesGround + INV_StructuralAsserts + SAFETY_PromisesKept +CONSTRAINT + SeqCtrBound diff --git a/tla/allocation/EventLoopCapOnly.cfg b/tla/allocation/EventLoopCapOnly.cfg new file mode 100644 index 0000000000..579c6c0fd7 --- /dev/null +++ b/tla/allocation/EventLoopCapOnly.cfg @@ -0,0 +1,35 @@ +\* EventLoopCapOnly: attribution isolation - the BUG-1 client with ONLY +\* FIX_CAP enabled (FIX_SWEEP off). BUG-1 must be fixed by the cap alone; +\* the sweep addresses the independent BUG-6/BUG-4 mechanism. +\* TLC flags: NONE (deadlock checking ON). +\* Expected: GREEN - no deadlock, full drain, same outcome class as +\* EventLoopFixed. A deadlock here but not in EventLoopFixed +\* would mean the sweep is (wrongly) load-bearing for BUG-1 - +\* an attribution error in the fix design. +\* Scale: local, seconds. +CONSTANTS + HEAP_SIZE = 3 + INSTANCES = {1, 2} + Size <- SizesEventLoop + USER_POISON = FALSE + C1_ENABLED = TRUE + C2_ENABLED = TRUE + C3_ENABLED = FALSE + CLIENT_MODE = "SCRIPTED_EVENTLOOP" + FIX_CAP = TRUE + FIX_SWEEP = FALSE + FIX_RPR = FALSE +INIT MCInit +NEXT MCNext +INVARIANTS + TypeOK + INV_NoOverlap + INV_InBounds + INV_Conservation + INV_CurrentMatchesGround + INV_StructuralAsserts + INV_NoDupAlloc + SAFETY_PromisesKept + INV_QuiescentHeapEmpty +CONSTRAINT + SeqCtrBound diff --git a/tla/allocation/EventLoopFixed.cfg b/tla/allocation/EventLoopFixed.cfg new file mode 100644 index 0000000000..dc8c8ca14f --- /dev/null +++ b/tla/allocation/EventLoopFixed.cfg @@ -0,0 +1,36 @@ +\* EventLoopFixed: the exact BUG-1 client (SCRIPTED_EVENTLOOP) with both +\* fixes on. Under FIX_CAP, I2's deferred create is capped (its funding +\* set excludes the later-requested destroy of I1) and INSTANT-FAILS +\* instead of deferring into the cycle; the poison cascade resolves both +\* destroys and the run drains (I1 ends as a documented poisoned-destroy +\* leak terminal, instState[2] = FAILED). +\* TLC flags: NONE (deadlock checking ON). +\* Expected: GREEN - no deadlock, full drain. A deadlock here means +\* FIX_CAP does not fix BUG-1 (fix-design bug). +\* Scale: local, seconds. +CONSTANTS + HEAP_SIZE = 3 + INSTANCES = {1, 2} + Size <- SizesEventLoop + USER_POISON = FALSE + C1_ENABLED = TRUE + C2_ENABLED = TRUE + C3_ENABLED = FALSE + CLIENT_MODE = "SCRIPTED_EVENTLOOP" + FIX_CAP = TRUE + FIX_SWEEP = TRUE + FIX_RPR = TRUE +INIT MCInit +NEXT MCNext +INVARIANTS + TypeOK + INV_NoOverlap + INV_InBounds + INV_Conservation + INV_CurrentMatchesGround + INV_StructuralAsserts + INV_NoDupAlloc + SAFETY_PromisesKept + INV_QuiescentHeapEmpty +CONSTRAINT + SeqCtrBound diff --git a/tla/allocation/FINDINGS.md b/tla/allocation/FINDINGS.md new file mode 100644 index 0000000000..940e79f312 --- /dev/null +++ b/tla/allocation/FINDINGS.md @@ -0,0 +1,160 @@ +# Realm Deferred Allocation — TLA+ Verification Campaign: Findings + +Status: **Phase 4 (local verification) complete; fix bundle modeled and +locally verified.** Sapling runs pending (see `SAPLING_JOBS.md`). No Realm +source code has been modified — all proposed fixes are analysis-only, +recorded in `bugs/`. + +**Fix-bundle addendum (2026-08-26):** the three candidate fixes are modeled +as spec toggles — `FIX_CAP` (BUG-1 capped admission), `FIX_SWEEP` +(BUG-6/BUG-4 stranded-ready sweep), `FIX_RPR` (BUG-5 trailing-alloc replay) +— and the full bundle is **green across the entire local matrix**, including +full-exhaustion passes at the exact bounds where the unfixed model fails and +the Inversion client green with deadlock checking ON. The validation round +itself produced a fourth adjudicated report: **BUG-5 is confirmed +load-bearing for the cap fix** (a capped rejection's poison cascade strands +any trailing dependent alloc; witness +`traces/Inversion-bug5-deadlock.txt`), so the bundle is indivisible — +FIX_CAP must not land in C++ without FIX_RPR. Four bug reports are final: +`bugs/BUG-1.md`, `bugs/BUG-5.md`, `bugs/BUG-6.md` (BUG-4 escalated inside +BUG-6), plus `bugs/FIX-REVIEW.md`. Details in EXPECTED.md's "Fix +validation" section. *Round-1 sapling addendum (2026-08-26):* the scale +runs confirmed BUG-6 at 4 instances and on the poison paths, and surfaced +one spec wiring artifact in the FIX_RPR toggle (TrailingRPR call-site fed +the full survivor list; corrected and re-validated green across the whole +local matrix — the C++ blueprint gained the continue-from-cursor rule, +`bugs/DUPALLOC-TRIAGE.md`); Big/BigFixed remain in progress on sapling. + +**C++ port started (2026-08-30):** the port of the indivisible bundle began +on the accumulated evidence — local full exhaustions plus sapling job 77858 +(7.52B distinct states violation-free, depth 11 complete, ~3.9B states into +depth 12; the pre-registered depth-13 gate proved physically out of reach on +single-node disk). `FUTURE-VERIFICATION.md` records the options if deeper +verification is ever wanted. + +The model covers deferred instance allocation and deletion in +`LocalManagedMemory` (`src/realm/mem_impl.{h,cc,inl}`): the +current/future/release allocator triple, `pending_allocs`/`pending_releases`, +seqid ordering, release reordering, and poison handling. Instance +redistricting is deferred to v2 (roadmap in `DESIGN.md` §9). Client behavior +is constrained by the documented contract: topologically sorted requests (no +back edges), destroy preconditions incorporate the created event. + +## Machine-confirmed bugs (3) + +All three reproduce with **legal clients** under the stated contract, with +TLC traces on disk and line-cited C++ executions in the bug reports. + +### BUG-1 — Event-loop deadlock from trigger-time ordering (`bugs/BUG-1.md`) + +A deferred create is inserted into the release/alloc total order when its +precondition **triggers** (`mem_impl.cc:784/801`), but its `e_created` was +published at **request** time (cc:712-717 records nothing). Any release +requested in the request→trigger window may legally depend on that +`e_created`; the future rebuild (cc:768-781) counts its space anyway, so +Realm plans the allocation out of a release that can only happen after the +allocation completes. Result: permanent silent hang — worse, the mapper is +first told `InstanceAllocResult{success=true}` (inst_impl.cc:1140-1142). +This is the bug Sean suspected in the design talk (~24:47); his conservatism +rule is enforced against the wrong clock. Minimal witness: 2 instances + +1 user event (`traces/EventLoop.trace.txt`, 7 states; also `traces/Smoke-run1.txt`, +`traces/Liveness-bug1.txt` with passing `LivenessNoCross` control). + +**Fix (hardened by adversarial review, `bugs/FIX-REVIEW.md`):** fund a +deferred admission only from {releases with seqid ≤ request-time cap} ∪ +{releases whose precondition has already triggered clean at admission}, +with a monotone-cap queue rule; capped-fit failure → honest +`ALLOC_INSTANT_FAILURE` instead of a hang. The pure request-time cap is +**not** shippable: it false-fails the canonical GC-ripple pattern, and no +arrival-order-only policy can distinguish that client from the cycle client +(identical arrival sequences; the difference is in the event graph Realm +cannot see). **Open question for Legion:** does the GC-ripple `e_pre` fire +at-or-after the victims' destroy preconditions? If not, the fallback is a +ballistic-style declaration flag on destroy. + +### BUG-6 — Stranded ready release; `assert(!it->is_ready)` reachable (`bugs/BUG-6.md`) + +An ARR-failure pushback (cc:884-887) leaves a READY entry behind a non-ready +one; the oldest-drain then empties `pending_allocs`, and both cleanup sites +(cc:1706 rebuild, cc:1751 tail ARR) are skipped precisely because the queue +just emptied. The next allocation needing deferral fires the cc:772 assert in +debug builds. In release builds the future rebuild survives, but the +documented `release = current + ready releases` invariant (mem_impl.h:399-405) +is broken from cc:787 on, costing reorderings and delaying dealloc acks. +Witness: 3 instances, H=3, 9 steps, no poison (`traces/SafetyMini.trace.txt`). + +**Fix:** a shared `sweep_stranded_ready_releases()` helper at all three +`pending_allocs`→empty transitions (both oldest-drain tails **and** +`remove_pending_release`), sweeping ready entries into `current_allocator` +in list order, redistrict-aware, firing their deferred notifies. Fix B +(tolerate-ready + re-apply after both resets) is a safety-equivalent smaller +fallback. + +### BUG-6→BUG-4 composite — Permanent range leak, notify-while-tag-live (`bugs/BUG-6.md` §6) + +After the BUG-6 stranding, the next deferred admission resets +`release_allocator := current` **including the stale-but-ready tag** +(cc:787; same shape as the cc:1556-1557 reset flagged as BUG-4); a later +triggered destroy reaching ARR full-success swaps that state into +`current_allocator` and erases the ready entry, firing its deferred dealloc +notify — the tag is never deallocated. Permanent leak + instance-slot +recycling with the tag still tracked: the #442 double-tracking class, with +**zero poison involved**. Witness: `Composite4.cfg`, 5 instances (4 provably +insufficient), 12-step trace (`traces/Composite4.txt`), violates +`INV_NoOrphanTags`/`INV_CurrentMatchesGround`. The BUG-6 sweep fix closes it. + +## Registered candidates pending sapling runs + +- **BUG-3** — soundness of the in-order unblock `assert(ok)` (cc:1670) and + the future-offset cross-check (cc:1674-1691) after partial reorderings + have rewritten history. Hunted by `Safety.cfg` (4 inst; ~110M+ distinct + states locally at 12 min, sapling-bound). +- **BUG-4 standalone (poison variant)** — the cc:1556-1557 stale-release + reset reached via a poisoned release. Hunted by `Poison4.cfg`. +- **BUG-5** — `remove_pending_release` never replays trailing pending allocs + (lastSeq beyond every walked seqid) onto the rebuilt future (cc:1562-1595) + → potential overlapping future planning. Hunted by `Poison4.cfg`. +- **BUG-2** — whether the client contract truly closes every + `missing_ok=false` path for failed allocations (C2-off configs document + what breaks without the contract). +- **BUG-7** (`bugs/BUG-7.md`) — `reuse_storage_immediate`'s oldest-drain + applies the offsets-flavor release to every drained prefix entry + (main:1372): debug abort / child mis-notification leak / OOB write on + mixed redistrict-plain prefixes. Pre-existing on main; found during the + C++ fidelity review of the fix branch; TLC-unverified — first + pre-registered expected-FAIL for the v2 (redistrict) model. + +Submit with `SAPLING_JOBS.md`: `sbatch sapling_tlc.sbatch Safety` (est. +1-6 h), `Poison4` (several hours), `Big` (use `-t 48:00:00`). Checkpoint +resume via TLC `-recover` is wired into the sbatch script. + +## Model confidence + +- Design doc (`DESIGN.md`, 828 lines) adversarially reviewed twice before + authoring; every behavioral claim carries a `file:line` citation. +- Spec (`DeferredAlloc.tla`, 949 lines) fidelity-reviewed by six independent + function-level passes: **zero blocking divergences**; five minor fixes + applied and revalidated (baseline outcomes reproduced exactly). +- C++ asserts are modeled as ghost-flag invariants, never action guards, so + assert-reachable states are reported, not pruned. Known bugs (BUG-4/5) are + deliberately present in the spec. +- First-fit abstraction (tag→interval map, minimal-feasible-offset) proven + equivalent to `BasicRangeAllocator`'s address-ordered first fit, twice + independently. +- Every expected-green config is green; every expected-red config is red for + the predicted reason (`EXPECTED.md` is the authoritative matrix). + +## Running + +Local: `./run.sh` (Smoke → EventLoop → SafetyMini → Composite4 → Liveness → +LivenessNoCross; seconds each at current bounds). Safety-only runs need a +writable tmp/metadir; **temporal (liveness) runs need an unsandboxed JVM** +(TLC's liveness checker binds an RMI socket). Sapling: `SAPLING_JOBS.md`. + +## v2 roadmap (DESIGN.md §9) + +Redistricting (`split_range`/reuse paths — required before trusting the +BUG-6 sweep fix's redistrict arm), alignment, duplicate releases from +network delays, dealloc-completion feedback shapes, instance-ID reuse +interaction (#442), and the v-next fix-validation specs pre-registered in +both bug reports (capped-ADA variant; three-site sweep variant). diff --git a/tla/allocation/FUTURE-VERIFICATION.md b/tla/allocation/FUTURE-VERIFICATION.md new file mode 100644 index 0000000000..9563679cbb --- /dev/null +++ b/tla/allocation/FUTURE-VERIFICATION.md @@ -0,0 +1,157 @@ +# Deeper Verification Options — Deferred Allocation Model + +Written 2026-08-30, when the C++ port of the fix bundle (FIX_CAP + FIX_SWEEP ++ FIX_RPR) began on the evidence below. This records what we would do if we +ever wanted more detailed verification than the campaign has already +delivered — cheap wins first, structural levers second, then the model +extensions that carry real verification debt, and the C++-side options that +complement TLC entirely. + +## 1. Where verification stands (the baseline) + +Local, **fully exhaustive** (state space completely drained, all green): + +| Config | Gen / distinct | Notes | +|---|---|---| +| SafetyMiniFixed | 64.7M / 23.5M | full battery incl. `INV_NoDupAlloc`, depth 19 | +| SafetyMiniSweepOnly | 53.3M / 20.6M | sweep-alone attribution | +| Composite4Fixed | 245k / 120k | the BUG-6→BUG-4 leak composite, closed | +| Inversion | 478 / 255 | **deadlock checking ON** — monotone-cap + trailing replay | +| GCRipple / EventLoopFixed / SmokeFixed | small | scripted intent invariants | +| LivenessFixed | 13.3k / 6.3k | temporal `LIVE_NoStuckAllocs` holds | + +Toggles-off regressions reproduce the four known counterexamples at exact +trace depths (EventLoop deadlock@7, SafetyMini violation@9, etc.). + +Sapling, **bounded but very large** (all violation-free at death): + +- **Job 77858** (SafetyFixed4, bundle, `-gzip`): 23.9B generated / **7.52B + distinct**, depth 11 COMPLETE at 3.58B, ~3.9B states into depth 12; died + at 25.5h on the fingerprint-set merge ("No space left", 261G free at + start — the node carried 435G of dead-job leftovers). +- **Poison4Fixed** (77843): 1.90B distinct clean through depth 10 — the + poison paths, where FIX_SWEEP's third site and FIX_RPR live. +- **Hunts** (77854/77855): SafetyHunt 1.67B, PoisonHunt 1.42B distinct, + clean through depth 10-11 — still **no witness** for BUG-3 + (mem_impl.cc:1668-1691 replay soundness), BUG-4-standalone, or unfixed + BUG-5 in the current-code model. + +The pre-registered gate (SafetyFixed4 clean through depth 13, `Progress(14)`) +was **not met**: completing depth 13 plausibly needs 15-25B distinct states +and 600-800G+ of node-local disk — beyond single-node sapling hardware. +Accepted rationale for proceeding anyway: 7.52B clean distinct states is +~2x the total coverage at which the only scale-level defect ever observed +(the round-1 TrailingRPR wiring artifact, a spec bug — corrected and +re-validated) appeared (depth 12, 4.0B distinct), on top of the local full +exhaustions of every behavior class the model expresses. + +## 2. Cheap wins if we resume scale runs (in effort order) + +1. **Sweep node /tmp first.** 77858 ran with only 261G of a 733G disk + (435G of leftovers from scancel'ed jobs whose cleanup traps never ran). + A swept node is ~2.7x the capacity for zero engineering. +2. **Keep the fingerprint set in RAM**: submit with `--mem=256G` (if nodes + allow) and pass TLC `-fpmem 0.6` (fraction of heap for the fpset). + Arithmetic: 7.5-15B fingerprints × 8B = 60-120G, which fits a ~200G heap + at fpmem 0.6-0.7. This eliminates the on-disk fp files AND the merge + transient that actually killed 77858, leaving the whole disk budget to + the gzipped queue (measured ~34-44 B/state → a swept 733G node holds + roughly 16-20B queued states). +3. **Wider nodes**: raise `--cpus-per-task` past 40 if sapling nodes have + more cores — TLC worker scaling is roughly linear until memory + bandwidth saturates (77858 sustained 17-21M states/min at 40 workers). + +With all three: depth-12 completion is plausible in ~40-70h; depth 13 +likely remains out of reach on a single node. + +## 3. Structural options (bigger levers) + +- **Distributed TLC** (TLCServer + TLCWorker + distributed fpset servers + across several nodes): removes the single-node disk/memory ceiling + entirely. Real setup cost, fragile interactions with `-gzip` and + checkpointing, and cluster etiquette concerns — worth it only if a + specific claim (e.g., "depth 13 complete") becomes load-bearing. +- **Targeted adversarial configs instead of blanket depth**: scripted + clients aimed at specific interleavings (the pattern that confirmed the + BUG-6→BUG-4 composite locally in 5 seconds via `Composite4.cfg` after + blanket Safety needed sapling). Note the claim changes from "all + behaviors ≤ depth D" to "all behaviors of shape S" — pre-register the + shape in EXPECTED.md as we did for Composite4. +- **Symmetry reduction**: the no-symmetry decision (DESIGN.md §7) holds for + mixed-size configs, but configs with EQUAL instance sizes admit + permutation symmetry (TLC `SYMMETRY` over a model-value instance set), + typically 10-100x state-space reduction for equal-size hunts. Caveat: + TLC's symmetry is unsound for liveness checking — safety configs only. +- **Simulation mode** (`-simulate` with `-depth N`): probabilistic + deep-trace sampling far past any BFS frontier. No completeness claim, + but the best bug-hunting per node-hour for "is there anything lurking at + depth 20+" questions, and it barely touches disk. + +## 4. Model-extension verification debt (v2 roadmap) + +Each of these is UNVERIFIED territory today; extending the model is the +prerequisite for trusting the corresponding code path or fix arm: + +- **Redistricting** (split_range: mem_impl.inl:168-274; + reuse_storage_deferrable/immediate: cc:926-1085, cc:1326-1536). + **Required before trusting the sweep fix's redistrict arm in + production** — bugs/BUG-6.md fix A is redistrict-aware by design, but v1 + models plain frees only. Needs: redistrict actions + PendingRelease + redistrict fields + a split_range operator in DeferredAlloc.tla; + redistrict variants of SafetyMini/Composite4; child-offset-consistency + invariants and the INV_NoOrphanTags extension to child tags. +- **The BUG-1 union rule** ("cap ∪ clean-triggered releases", + bugs/BUG-1.md): only needed if the GC-ripple pattern under memory + pressure produces unacceptable spurious instant-failures. It is + currently UNVERIFIED — extending the funding gate in `ADAResCap` and + re-running the FULL validation matrix is mandatory before any C++ use. +- **Alignment** (calculate_offset, mem_impl.inl:154-165): richer + fragmentation; extends the §2 allocator operators and the first-fit + equivalence argument. +- **Duplicate releases via network delays** (tolerated by cc:773-778): + relax v1's one-release-per-instance uniqueness; the exact first-match + form of INV_FutureOffsetConsistency was written to survive this. Note: + the sweep fix's strict void free would debug-assert on the second entry + of a duplicate ready pair — identical strictness to the pre-existing + in-order drain (so not a regression), but the duplicates model must + account for it when this item is taken up. +- **Multi-node create ordering** (the remote-create snapshot window): a + creation issued on a non-owner node publishes `e_created` at the creator + before the owner takes the `release_seqid_cap` snapshot on + MemStorageAllocRequest receipt, so a release requested inside that + window can slip under the cap and readmit the BUG-1 funding cycle + across nodes. Fix directions to evaluate in a multi-node model: take the + snapshot at the creator and carry it in the active message, or adjust + the cap on the owner at AM receipt (e.g. exclude releases whose request + provably raced the create's AM). +- **Dealloc-completion feedback shapes** (clients deriving triggers from + destruction profiling responses — excluded in DESIGN.md §1). +- **Multi-memory / remote request paths** (MemStorageAlloc/Release + messages, remote notify forwarding). + +## 5. C++-side verification (complementary to TLC) + +- **Randomized stress harness**: drive the public API with + create/destroy/user-event-trigger sequences shaped like the model's + client contract (topologically sorted, destroy-after-create, ballistic + triggers) at sizes chosen to force deferral, reordering, and poison. + `tests/random_config_test.cc` is an existing repo pattern to follow. +- **Debug-build soak**: the fix bundle restores mem_impl.cc:772's + `assert(!it->is_ready)` as a true invariant — long debug-build runs with + the cc:772-family asserts active are now meaningful regression signal + rather than known-false alarms. +- **Runtime shadow-checker**: maintain the model's key ghosts as + DEBUG_REALM counters inside LocalManagedMemory — a tag-vs-live-instance + audit (INV_NoOrphanTags) at pending-queue-empty transitions and a + notify-once counter per instance — turning the two #442-class detectors + into cheap in-situ checks. + +## 6. The standing regression oracle + +The local matrix is the durable payoff: `./run.sh` (Smoke → EventLoop → +SafetyMini → Composite4 → Liveness → LivenessNoCross, plus the Fixed +family) runs in seconds-to-minutes and must be green before and after ANY +change to mem_impl.cc allocator logic — with the toggles mirroring whether +the change is pre- or post-fix-bundle semantics. The sapling configs only +re-enter the picture when the MODEL itself changes (v2 extensions above); +routine code work never needs them. diff --git a/tla/allocation/GCRipple.cfg b/tla/allocation/GCRipple.cfg new file mode 100644 index 0000000000..82aa6ab5d8 --- /dev/null +++ b/tla/allocation/GCRipple.cfg @@ -0,0 +1,49 @@ +\* GCRipple: the FIX-REVIEW.md trade-off client 1 (SCRIPTED_GCRIPPLE) with +\* the full fix bundle on. Script: I1 (sz 3) fills the heap immediately; I2 (sz 3) +\* is gated on ballistic B; destroy(I1) is requested AFTER I2's create +\* (DestroyOrderOK) and gated on ballistic D. The environment fires D and +\* B in nondeterministic order, and trigger delivery interleaves further. +\* Intent under the pure cap (FIX_CAP v1, no ready-fold): +\* - destroy APPLIED before I2's create-trigger -> I2 INSTANT-SUCCEEDS +\* (completed releases fund via current regardless of cap); +\* - otherwise -> I2 INSTANT-FAILS honestly (ACCEPTED BEHAVIOR CHANGE: +\* the cap excludes the later-requested pending destroy - this is the +\* documented false-OOM cost of the pure cap; the ready-fold / +\* ballistic-lite variants that rescue it are v-next-next). +\* Both end classes are reachable; I2 must never be ALLOC_DEFERRED +\* (INV_GCRippleNoDefer) and never stuck (deadlock check). +\* TLC flags: NONE (deadlock checking ON - full drain required). +\* Expected: GREEN. INV_GCRippleNoDefer violation => the cap admitted a +\* deferral it must reject (fix-design bug). +\* INV_GCRippleSuccessFunded violation => a success not funded +\* by the completed release (spec bug). Deadlock => cascade +\* failed to drain (fix-design bug). +\* Scale: local, seconds. +CONSTANTS + HEAP_SIZE = 3 + INSTANCES = {1, 2} + Size <- SizesGCRipple + USER_POISON = FALSE + C1_ENABLED = TRUE + C2_ENABLED = TRUE + C3_ENABLED = FALSE + CLIENT_MODE = "SCRIPTED_GCRIPPLE" + FIX_CAP = TRUE + FIX_SWEEP = TRUE + FIX_RPR = TRUE +INIT MCInit +NEXT MCNext +INVARIANTS + TypeOK + INV_NoOverlap + INV_InBounds + INV_Conservation + INV_CurrentMatchesGround + INV_StructuralAsserts + INV_NoDupAlloc + SAFETY_PromisesKept + INV_GCRippleNoDefer + INV_GCRippleSuccessFunded + INV_QuiescentHeapEmpty +CONSTRAINT + SeqCtrBound diff --git a/tla/allocation/Inversion.cfg b/tla/allocation/Inversion.cfg new file mode 100644 index 0000000000..6375044795 --- /dev/null +++ b/tla/allocation/Inversion.cfg @@ -0,0 +1,62 @@ +\* Inversion: the bugs/BUG-1.md A/R*/B trigger-inversion construction +\* (SCRIPTED_INVERSION) with the FULL fix bundle (CAP + SWEEP + RPR). +\* Script (H=3): +\* I1 (sz 3) fills the heap immediately; +\* A = I2 (sz 1, ballistic BA) requested next -> cap(A) = 0; +\* destroy(I1) = R1 (deps {1,2} - depends on eCreated(A): the cycle edge); +\* B = I3 (sz 2, ballistic BB) requested AFTER destroy(1) (CreateOrderOK) +\* -> cap(B) = 1 >= seq(R1). +\* Inverted triggers (BB before BA): B legally defers against R1; A then +\* arrives with cap(A) < cap(B). Sized so that WITHOUT the monotone-cap +\* guard A would fit in fut behind B (free cell [2,3)) and be admitted - +\* rebuilding the BUG-1 cycle through the no-queue-jumping fut test. The +\* guard INSTANT-FAILs A; the poison cascade removes R1 (its precondition +\* fires poisoned) and FIX_RPR's trailing-alloc replay in +\* remove_pending_release EVENTUAL_FAILs B cleanly; the run drains +\* (I1 ends as a documented leak terminal). +\* TLC flags: NONE (deadlock checking ON - full drain required). +\* Expected: GREEN over the full space, deadlock check included: +\* INV_InversionCapped and SAFETY_PromisesKept hold everywhere +\* and every interleaving drains; B ends EVENTUAL_FAILURE +\* cleanly via the RPR trailing replay. +\* History: with FIX_RPR = FALSE this config reproduces the BUG-5 +\* stranding as a deadlock (witness kept: +\* traces/Inversion-bug5-deadlock.txt): the cap correctly +\* failed A, R1 fired poisoned and was removed, but B - a +\* TRAILING alloc whose lastSeq exceeds every walked seqid - +\* was never revisited by the RPR replay: neither failed nor +\* refunded. That run established the composition finding +\* that FIX_CAP without FIX_RPR is NOT shippable (the cap +\* increases poisoned-release frequency, making BUG-5's hole +\* load-bearing for drain liveness); hence the three-toggle +\* bundle. A deadlock in THIS config => FIX_RPR design bug +\* (trailing replay missed a case). +\* Scale: local, seconds. +CONSTANTS + HEAP_SIZE = 3 + INSTANCES = {1, 2, 3} + Size <- SizesInversion + USER_POISON = FALSE + C1_ENABLED = TRUE + C2_ENABLED = TRUE + C3_ENABLED = FALSE + CLIENT_MODE = "SCRIPTED_INVERSION" + FIX_CAP = TRUE + FIX_SWEEP = TRUE + FIX_RPR = TRUE +INIT MCInit +NEXT MCNext +INVARIANTS + TypeOK + INV_NoOverlap + INV_InBounds + INV_Conservation + INV_CurrentMatchesGround + INV_StructuralAsserts + SAFETY_PromisesKept + INV_InversionCapped + INV_PoisonReplayOnlyFailsAfterPoint + INV_NoDupAlloc + INV_QuiescentHeapEmpty +CONSTRAINT + SeqCtrBound diff --git a/tla/allocation/Liveness.cfg b/tla/allocation/Liveness.cfg new file mode 100644 index 0000000000..726c29d2a3 --- /dev/null +++ b/tla/allocation/Liveness.cfg @@ -0,0 +1,39 @@ +\* Liveness: temporal check that every DEFERRED allocation eventually +\* resolves, under weak fairness on client requests, ballistic events and +\* trigger delivery (full-cleanup client). +\* TLC flags: -deadlock (deadlock checking OFF - the Done self-loop plus +\* fairness handles termination; DESIGN.md s6). +\* Expected: FAIL - LIVE_NoStuckAllocs via a BUG-1 lasso (an ALLOC_DEFERRED +\* instance whose enabling release waits on its own eCreated). +\* LivenessNoCross.cfg is the matched control at IDENTICAL bounds +\* with CLIENT_MODE = "NO_CROSS_DEPS" -> expect PASS. +\* Bounds: REDUCED (Phase 4, 2026-08-25) from 3 inst/H=4 to 2 inst/H=3 +\* (SizesEventLoop): the 3-instance liveness run's throughput +\* collapsed ~5x from behavior-graph maintenance (~500k distinct +\* at 10 min, queue still growing) and blew the local budget. +\* 2 instances retain the BUG-1 lasso (EventLoop scale) and make +\* the FAIL/PASS pair a same-bounds controlled comparison. +\* Scale: local, seconds-to-minutes at the reduced bounds. +CONSTANTS + HEAP_SIZE = 3 + INSTANCES = {1, 2} + Size <- SizesEventLoop + USER_POISON = FALSE + C1_ENABLED = TRUE + C2_ENABLED = TRUE + C3_ENABLED = FALSE + CLIENT_MODE = "FREE" + FIX_CAP = FALSE + FIX_SWEEP = FALSE + FIX_RPR = FALSE +SPECIFICATION LiveSpec +PROPERTIES + LIVE_NoStuckAllocs +INVARIANTS + TypeOK + INV_NoOverlap + INV_InBounds + INV_StructuralAsserts + SAFETY_PromisesKept +CONSTRAINT + SeqCtrBound diff --git a/tla/allocation/LivenessFixed.cfg b/tla/allocation/LivenessFixed.cfg new file mode 100644 index 0000000000..80423d0179 --- /dev/null +++ b/tla/allocation/LivenessFixed.cfg @@ -0,0 +1,35 @@ +\* LivenessFixed: the Liveness temporal check with the full fix bundle (CAP+SWEEP+RPR) on. +\* LIVE_NoStuckAllocs's target set includes FAILED, so a capped +\* INSTANT_FAILURE (or a poison-cascade EVENTUAL_FAILURE) counts as +\* resolved - the property demands no allocation stays DEFERRED forever. +\* TLC flags: -deadlock (SPECIFICATION mode; Done self-loop + fairness). +\* Expected: PASS - LIVE_NoStuckAllocs holds under FIX_CAP (the wait +\* graph is a DAG; every deferral resolves). A FAIL is a +\* FIX_CAP design bug (residual cycle or starvation). +\* Bounds: matched to Liveness.cfg's reduced bounds (2 inst / H=3) so +\* FAIL(base) vs PASS(fixed) is a same-bounds comparison. +\* Scale: local, seconds-to-minutes. +CONSTANTS + HEAP_SIZE = 3 + INSTANCES = {1, 2} + Size <- SizesEventLoop + USER_POISON = FALSE + C1_ENABLED = TRUE + C2_ENABLED = TRUE + C3_ENABLED = FALSE + CLIENT_MODE = "FREE" + FIX_CAP = TRUE + FIX_SWEEP = TRUE + FIX_RPR = TRUE +SPECIFICATION LiveSpec +PROPERTIES + LIVE_NoStuckAllocs +INVARIANTS + TypeOK + INV_NoOverlap + INV_InBounds + INV_StructuralAsserts + INV_NoDupAlloc + SAFETY_PromisesKept +CONSTRAINT + SeqCtrBound diff --git a/tla/allocation/LivenessNoCross.cfg b/tla/allocation/LivenessNoCross.cfg new file mode 100644 index 0000000000..2c96e02f4c --- /dev/null +++ b/tla/allocation/LivenessNoCross.cfg @@ -0,0 +1,37 @@ +\* LivenessNoCross: confirmatory control for the Liveness config. +\* Identical to Liveness.cfg except CLIENT_MODE = "NO_CROSS_DEPS": destroy +\* preconditions may not wait on another instance's eCreated, which excludes +\* exactly the BUG-1 cycle shape. +\* TLC flags: -deadlock. +\* Expected: PASS - LIVE_NoStuckAllocs holds; this is the control showing +\* the Liveness failure is specifically the cross-dependency +\* cycle (BUG-1), not fairness or model artifacts. A FAIL here +\* is a SECOND, independent liveness bug -> Phase 5, high +\* interest. +\* Bounds: matched to Liveness.cfg's reduced bounds (2 inst / H=3, +\* SizesEventLoop) so the FAIL/PASS pair is a same-bounds +\* controlled comparison; see Liveness.cfg header for why. +\* Scale: local, seconds-to-minutes at the reduced bounds. +CONSTANTS + HEAP_SIZE = 3 + INSTANCES = {1, 2} + Size <- SizesEventLoop + USER_POISON = FALSE + C1_ENABLED = TRUE + C2_ENABLED = TRUE + C3_ENABLED = FALSE + CLIENT_MODE = "NO_CROSS_DEPS" + FIX_CAP = FALSE + FIX_SWEEP = FALSE + FIX_RPR = FALSE +SPECIFICATION LiveSpec +PROPERTIES + LIVE_NoStuckAllocs +INVARIANTS + TypeOK + INV_NoOverlap + INV_InBounds + INV_StructuralAsserts + SAFETY_PromisesKept +CONSTRAINT + SeqCtrBound diff --git a/tla/allocation/MCDeferredAlloc.tla b/tla/allocation/MCDeferredAlloc.tla new file mode 100644 index 0000000000..15ec692453 --- /dev/null +++ b/tla/allocation/MCDeferredAlloc.tla @@ -0,0 +1,405 @@ +--------------------------- MODULE MCDeferredAlloc --------------------------- +(***************************************************************************) +(* Model-checking harness for the Realm deferred instance allocation *) +(* protocol (DeferredAlloc.tla). Implements DESIGN.md sections 5-7: *) +(* - client actions: create/destroy request issuance with *) +(* nondeterministic dependency-term choice under contract toggles *) +(* C1 (topological sort), C2 (destroy-after-create), C3 (destroy *) +(* only after create attempted) *) +(* - environment actions: precondition firing (the exact DESIGN.md s5 *) +(* rule: a term fires only when ALL deps have resolved AND its *) +(* ballistic user event has fired; poisoned iff >=1 dep poisoned or *) +(* the ballistic event was user-poisoned), ballistic event firing, *) +(* trigger delivery (TriggerCreate / TriggerDestroy) *) +(* - intrinsic poison is ALWAYS ON (a failed/cancelled create resolves *) +(* eCreated(i) poisoned; C2 then propagates into preD[i]). The *) +(* USER_POISON constant gates only client-poisoned ballistic events. *) +(* - Init/Next composition, Done self-loop + Quiescent (DESIGN.md s6), *) +(* fairness + LIVE_NoStuckAllocs for the Liveness config, *) +(* SeqCtrBound state constraint, per-config Size functions. *) +(* *) +(* Quiescence subtlety: a destroy whose precondition fires POISONED is *) +(* silently cancelled (cc:818-825) or removed (cc:1754); the instance *) +(* then stays ALLOCATED with its tag in cur forever - a documented, *) +(* client-caused leak ("POSSIBLE LEAK", ii:87), not a protocol bug. *) +(* Quiescent treats that as a resolved terminal state, and *) +(* INV_QuiescentHeapEmpty permits exactly those tags, so deadlock-checked *) +(* configs stay green on legal traces and the BUG-4 detector stays sharp *) +(* (a BUG-4-stranded tag belongs to a DESTROYED/notified instance and is *) +(* still flagged). *) +(***************************************************************************) +EXTENDS DeferredAlloc + +CONSTANTS + USER_POISON, \* BOOLEAN: client may poison ballistic user events + C1_ENABLED, \* BOOLEAN: dep sets restricted to earlier-requested creates + C2_ENABLED, \* BOOLEAN: i \in preD[i].deps forced + C3_ENABLED, \* BOOLEAN: destroy request only after create attempted + CLIENT_MODE \* "FREE" | "SCRIPTED_EVENTLOOP" | "NO_CROSS_DEPS" + +ASSUME CLIENT_MODE \in {"FREE", "SCRIPTED_EVENTLOOP", "NO_CROSS_DEPS", + "SCRIPTED_COMPOSITE", + "SCRIPTED_GCRIPPLE", "SCRIPTED_INVERSION"} +ASSUME USER_POISON \in BOOLEAN /\ C1_ENABLED \in BOOLEAN + /\ C2_ENABLED \in BOOLEAN /\ C3_ENABLED \in BOOLEAN + +(***************************************************************************) +(* Client / event variables (DESIGN.md s5). *) +(***************************************************************************) +VARIABLES + createRequested, \* [INSTANCES -> BOOLEAN] + destroyRequested, \* [INSTANCES -> BOOLEAN] + createWaiter, \* [INSTANCES -> BOOLEAN] DeferredCreate registered (cc:715) + destroyWaiter, \* [INSTANCES -> BOOLEAN] DeferredDestroy registered (cc:920) + preC, \* [INSTANCES -> Term] create precondition term + preD, \* [INSTANCES -> Term] destroy precondition term + balC, \* [INSTANCES -> BalState] ballistic event of preC[i] + balD \* [INSTANCES -> BalState] ballistic event of preD[i] + +BalStates == {"NONE", "PENDING", "FIRED", "POISONED"} +NoTerm == [deps |-> {}, ballistic |-> FALSE] + +clientVars == <> + +mcVars == <> + +ClientInit == + /\ createRequested = [i \in INSTANCES |-> FALSE] + /\ destroyRequested = [i \in INSTANCES |-> FALSE] + /\ createWaiter = [i \in INSTANCES |-> FALSE] + /\ destroyWaiter = [i \in INSTANCES |-> FALSE] + /\ preC = [i \in INSTANCES |-> NoTerm] + /\ preD = [i \in INSTANCES |-> NoTerm] + /\ balC = [i \in INSTANCES |-> "NONE"] + /\ balD = [i \in INSTANCES |-> "NONE"] + +MCInit == InitProto /\ ClientInit + +(***************************************************************************) +(* The DESIGN.md s5 firing rule over the protocol's explicit eCreated. *) +(***************************************************************************) +ECreatedResolved(j) == eCreated[j] # "UNFIRED" +ECreatedPoisoned(j) == eCreated[j] = "POISONED" + +\* A term fires only when ALL deps have resolved (clean or poisoned) AND, +\* if ballistic, its user event has fired. No early-fire on first poison. +TermFired(t, bal) == + /\ \A j \in t.deps : ECreatedResolved(j) + /\ t.ballistic => bal \in {"FIRED", "POISONED"} + +\* Poisoned iff at least one dep resolved poisoned, or the ballistic user +\* event was user-poisoned. Meaningful only when TermFired holds. +TermPoisoned(t, bal) == + \/ \E j \in t.deps : ECreatedPoisoned(j) + \/ (t.ballistic /\ bal = "POISONED") + +(***************************************************************************) +(* Client choice sets (contract C1/C2/C3 + CLIENT_MODE). *) +(***************************************************************************) +EligibleDeps == + IF C1_ENABLED THEN {j \in INSTANCES : createRequested[j]} ELSE INSTANCES + +AllowedCTerm(i) == + CASE CLIENT_MODE = "SCRIPTED_EVENTLOOP" -> + \* the s5 worked example: I0=1 immediate, I1=2 ballistic-only + IF i = 1 THEN {[deps |-> {}, ballistic |-> FALSE]} + ELSE {[deps |-> {}, ballistic |-> TRUE]} + [] CLIENT_MODE = "SCRIPTED_COMPOSITE" -> + \* Composite4 script (bugs/BUG-6.md item 6): every create is + \* immediate-precondition; the interesting deferral (i2, i4) comes + \* from ADA itself, not from create preconditions. + {[deps |-> {}, ballistic |-> FALSE]} + [] CLIENT_MODE = "SCRIPTED_GCRIPPLE" -> + \* GC-ripple (bugs/BUG-1.md, FIX-REVIEW.md trade-off client 1): + \* I1 fills the heap immediately; I2 (same size) is gated on + \* ballistic B. The environment fires B and the destroy's + \* ballistic D in nondeterministic order. + IF i = 1 THEN {[deps |-> {}, ballistic |-> FALSE]} + ELSE {[deps |-> {}, ballistic |-> TRUE]} + [] CLIENT_MODE = "SCRIPTED_INVERSION" -> + \* Trigger inversion (bugs/BUG-1.md A/R*/B): I1 immediate filler; + \* A=2 and B=3 are both ballistic-gated creates whose triggers can + \* fire in inverted request order. + IF i = 1 THEN {[deps |-> {}, ballistic |-> FALSE]} + ELSE {[deps |-> {}, ballistic |-> TRUE]} + [] OTHER -> + {[deps |-> d, ballistic |-> b] : + d \in SUBSET (INSTANCES \ {i}), b \in BOOLEAN} + +AllowedDTerm(i) == + CASE CLIENT_MODE = "SCRIPTED_EVENTLOOP" -> + IF i = 1 THEN {[deps |-> {1, 2}, ballistic |-> FALSE]} + ELSE {[deps |-> {2}, ballistic |-> FALSE]} + [] CLIENT_MODE = "NO_CROSS_DEPS" -> + \* excludes the BUG-1 shape: no destroy precondition may wait on + \* another instance's eCreated + {[deps |-> d, ballistic |-> b] : d \in SUBSET {i}, b \in BOOLEAN} + [] CLIENT_MODE = "SCRIPTED_COMPOSITE" -> + \* Composite4 script: destroy(1) is held open by a ballistic user + \* event (it is the stranding drain trigger); every other destroy + \* waits only on its own eCreated (C2-minimal). destroy(3) and + \* destroy(5) therefore arrive request-time-TRIGGERED once their + \* instance is ALLOCATED - the cc:884-887 pushback (i3) and the + \* stale-rel ARR invocation (i5) respectively. + IF i = 1 THEN {[deps |-> {1}, ballistic |-> TRUE]} + ELSE {[deps |-> {i}, ballistic |-> FALSE]} + [] CLIENT_MODE = "SCRIPTED_GCRIPPLE" -> + \* destroy(I1) is gated on ballistic D (its dep {1} is resolved by + \* then, so D alone controls firing); destroy(I2) is C2-minimal. + \* DestroyOrderOK forces destroy(1) to be requested AFTER I2's + \* create - the request lands inside I2's request->trigger window. + IF i = 1 THEN {[deps |-> {1}, ballistic |-> TRUE]} + ELSE {[deps |-> {i}, ballistic |-> FALSE]} + [] CLIENT_MODE = "SCRIPTED_INVERSION" -> + \* destroy(I1) = R1 depends on eCreated(A=2) - the cycle edge; the + \* cleanup destroys are C2-minimal. CreateOrderOK forces B=3's + \* create request AFTER destroy(1), so cap(B) >= seq(R1) while + \* cap(A) < seq(R1): inverted triggers (B before A) then exercise + \* the monotone-cap guard. + IF i = 1 THEN {[deps |-> {1, 2}, ballistic |-> FALSE]} + ELSE {[deps |-> {i}, ballistic |-> FALSE]} + [] OTHER -> + {[deps |-> d, ballistic |-> b] : + d \in SUBSET INSTANCES, b \in BOOLEAN} + +\* Scripted-mode request-order constraints (TRUE in every other mode). +CreateOrderOK(i) == + CASE CLIENT_MODE = "SCRIPTED_INVERSION" /\ i = 3 -> + destroyRequested[1] \* B requested after destroy(I1) + [] CLIENT_MODE = "SCRIPTED_GCRIPPLE" /\ i = 2 -> + \* I1 fills the heap FIRST; without this guard TLC found the + \* off-script order (I2 first, destroy(I2), then I1 allocates into + \* the freed heap) violating INV_GCRippleSuccessFunded's intent + \* (traces/GCRipple-orderguard-misfire.txt). + createRequested[1] + [] OTHER -> TRUE + +DestroyOrderOK(i) == + IF CLIENT_MODE = "SCRIPTED_GCRIPPLE" /\ i = 1 + THEN createRequested[2] \* destroy(I1) after I2's create + ELSE TRUE + +(***************************************************************************) +(* Client actions. Each wraps exactly one protocol action (which updates *) +(* all protocol+ghost variables) with the client-side bookkeeping. *) +(* Status at request time (has_triggered_faultaware, cc:703 / cc:819): *) +(* a just-issued ballistic conjunct is PENDING, so any ballistic term is *) +(* untriggered at request; a deps-only term is triggered iff all deps *) +(* have already resolved, poisoned iff one resolved poisoned. *) +(***************************************************************************) +ClientRequestCreate(i) == + /\ ~createRequested[i] + /\ CreateOrderOK(i) + /\ \E t \in AllowedCTerm(i) : + /\ t.deps \subseteq EligibleDeps \* C1 + /\ LET trig == (\A j \in t.deps : ECreatedResolved(j)) /\ ~t.ballistic + pois == trig /\ (\E j \in t.deps : ECreatedPoisoned(j)) + IN /\ RequestCreate(i, trig, pois) + /\ createWaiter' = [createWaiter EXCEPT ![i] = ~trig] \* cc:715 + /\ preC' = [preC EXCEPT ![i] = t] + /\ balC' = [balC EXCEPT ![i] = IF t.ballistic THEN "PENDING" ELSE "NONE"] + /\ createRequested' = [createRequested EXCEPT ![i] = TRUE] + /\ UNCHANGED <> + +ClientRequestDestroy(i) == + /\ createRequested[i] + /\ ~destroyRequested[i] + /\ DestroyOrderOK(i) + /\ C3_ENABLED => + instState[i] \notin {"CREATE_PENDING", "CREATE_PENDING_DESTROY"} + /\ \E t \in AllowedDTerm(i) : + /\ t.deps \subseteq EligibleDeps \* C1 + /\ C2_ENABLED => i \in t.deps \* C2 + /\ LET trig == (\A j \in t.deps : ECreatedResolved(j)) /\ ~t.ballistic + pois == trig /\ (\E j \in t.deps : ECreatedPoisoned(j)) + IN /\ RequestDestroy(i, trig, pois) + \* waiter registered iff deferred (cc:918-921); note the + \* cc:846 structural-flag case (triggered destroy of a + \* CREATE_PENDING instance, C2-off only) registers NO waiter, + \* matching the release-build fallthrough to cc:915-917. + /\ destroyWaiter' = [destroyWaiter EXCEPT ![i] = ~trig] + /\ preD' = [preD EXCEPT ![i] = t] + /\ balD' = [balD EXCEPT ![i] = IF t.ballistic THEN "PENDING" ELSE "NONE"] + /\ destroyRequested' = [destroyRequested EXCEPT ![i] = TRUE] + /\ UNCHANGED <> + +(***************************************************************************) +(* Environment actions. *) +(***************************************************************************) +FireBallisticC(i) == + /\ balC[i] = "PENDING" + /\ \E v \in ({"FIRED"} \cup (IF USER_POISON THEN {"POISONED"} ELSE {})) : + balC' = [balC EXCEPT ![i] = v] + /\ UNCHANGED protoVars + /\ UNCHANGED <> + +FireBallisticD(i) == + /\ balD[i] = "PENDING" + /\ \E v \in ({"FIRED"} \cup (IF USER_POISON THEN {"POISONED"} ELSE {})) : + balD' = [balD EXCEPT ![i] = v] + /\ UNCHANGED protoVars + /\ UNCHANGED <> + +EnvTriggerCreate(i) == + /\ createWaiter[i] + /\ TermFired(preC[i], balC[i]) + /\ TriggerCreate(i, TermPoisoned(preC[i], balC[i])) + /\ createWaiter' = [createWaiter EXCEPT ![i] = FALSE] + /\ UNCHANGED <> + +\* Enabled purely on "waiter registered AND term fired": deliberately NOT +\* guarded on create-side state, so that C2-off configs can deliver a +\* destroy trigger while the create is still pending and reach the +\* cc:1630 / cc:1720-1723 / cc:1548-1551 structural asserts (BUG-2 hunt). +EnvTriggerDestroy(i) == + /\ destroyWaiter[i] + /\ TermFired(preD[i], balD[i]) + /\ TriggerDestroy(i, TermPoisoned(preD[i], balD[i])) + /\ destroyWaiter' = [destroyWaiter EXCEPT ![i] = FALSE] + /\ UNCHANGED <> + +(***************************************************************************) +(* Quiescence, Done self-loop, Next (DESIGN.md s6). *) +(***************************************************************************) +\* A destroy whose precondition fired poisoned was silently cancelled +\* (cc:818-825) or removed (cc:1754-1755): the instance legally stays +\* ALLOCATED, tag in cur, forever ("POSSIBLE LEAK", ii:87). Terminal. +DestroyResolvedLeak(i) == + /\ instState[i] = "ALLOCATED" + /\ destroyRequested[i] + /\ ~destroyWaiter[i] + /\ \A k \in 1..Len(pendingReleases) : pendingReleases[k].inst # i + +Quiescent == + /\ \A i \in INSTANCES : createRequested[i] /\ destroyRequested[i] + /\ pendingAllocs = <<>> + /\ pendingReleases = <<>> + /\ \A i \in INSTANCES : ~createWaiter[i] /\ ~destroyWaiter[i] + /\ \A i \in INSTANCES : + instState[i] \in {"DESTROYED", "FAILED"} \/ DestroyResolvedLeak(i) + /\ \A i \in INSTANCES : balC[i] # "PENDING" /\ balD[i] # "PENDING" + +\* Clean completion self-loops so that a TLC deadlock report fires exactly +\* on stuck NON-quiescent states. +Done == Quiescent /\ UNCHANGED mcVars + +MCNext == + \/ \E i \in INSTANCES : ClientRequestCreate(i) + \/ \E i \in INSTANCES : ClientRequestDestroy(i) + \/ \E i \in INSTANCES : FireBallisticC(i) + \/ \E i \in INSTANCES : FireBallisticD(i) + \/ \E i \in INSTANCES : EnvTriggerCreate(i) + \/ \E i \in INSTANCES : EnvTriggerDestroy(i) + \/ Done + +Spec == MCInit /\ [][MCNext]_mcVars + +(***************************************************************************) +(* Fairness and temporal properties (Liveness config). WF on the client *) +(* request actions encodes the full-cleanup client (DESIGN.md s5): without *) +(* it, a trace where the client simply never issues a destroy makes *) +(* LIVE_NoStuckAllocs fail for a reason that is not a Realm bug. *) +(***************************************************************************) +Fairness == + \A i \in INSTANCES : + /\ WF_mcVars(ClientRequestCreate(i)) + /\ WF_mcVars(ClientRequestDestroy(i)) + /\ WF_mcVars(FireBallisticC(i)) + /\ WF_mcVars(FireBallisticD(i)) + /\ WF_mcVars(EnvTriggerCreate(i)) + /\ WF_mcVars(EnvTriggerDestroy(i)) + +LiveSpec == Spec /\ Fairness + +LIVE_NoStuckAllocs == + \A i \in INSTANCES : + (instState[i] = "ALLOC_DEFERRED") ~> + (instState[i] \in {"ALLOCATED", "DESTROYED", "FAILED"}) + +(***************************************************************************) +(* Harness-side invariants and constraints. *) +(***************************************************************************) +\* Backstop for BUG-4-escalated (DESIGN.md s6/s8): at quiescence every tag +\* still in the heap must be a documented poisoned-destroy leak. A +\* BUG-4-stranded tag belongs to a DESTROYED (already-notified) instance, +\* which DestroyResolvedLeak does not admit, so the detector stays sharp. +INV_QuiescentHeapEmpty == + Quiescent => \A t \in DOMAIN cur : DestroyResolvedLeak(t) + +\* DESIGN.md s7 backstop; naturally bounded (one release per instance v1). +SeqCtrBound == seqCtr <= 2 * Cardinality(INSTANCES) + +(***************************************************************************) +(* Fix-validation intent invariants (v-next; FIX_CAP / FIX_SWEEP are *) +(* declared in DeferredAlloc.tla). Mode-guarded: vacuous elsewhere. *) +(***************************************************************************) +\* GC-ripple under the pure request-time cap: I2's funding set is empty +\* (the destroy of I1 is requested after I2's create), so ADA either +\* succeeds against current (destroy already APPLIED - "completed releases +\* fund via current regardless of cap") or INSTANT-FAILs. Never deferred. +INV_GCRippleNoDefer == + (CLIENT_MODE = "SCRIPTED_GCRIPPLE" /\ FIX_CAP) => + instState[2] # "ALLOC_DEFERRED" + +\* If I2 succeeded, it was funded by I1's completed release: I1's tag was +\* out of cur when I2 was placed, and (sizes = H each) can never return. +INV_GCRippleSuccessFunded == + CLIENT_MODE = "SCRIPTED_GCRIPPLE" => + (instState[2] \in {"ALLOCATED", "DESTROYED"} => ~HasTag(cur, 1)) + +\* Inversion under the cap: A=2 has cap < seq(R1) and, when B=3 is already +\* queued, a lower cap than B - the empty-funding-set test or the monotone- +\* cap guard must INSTANT-FAIL it. A is never admitted as deferred. +INV_InversionCapped == + (CLIENT_MODE = "SCRIPTED_INVERSION" /\ FIX_CAP) => + instState[2] # "ALLOC_DEFERRED" + +(***************************************************************************) +(* Per-config Size functions (cfg files substitute Size <- SizesX). *) +(***************************************************************************) +SizesSmoke == (1 :> 2) @@ (2 :> 2) \* H=3 +SizesEventLoop == (1 :> 3) @@ (2 :> 3) \* H=3 +SizesLiveness == (1 :> 2) @@ (2 :> 2) @@ (3 :> 1) \* H=4 +SizesSafety == (1 :> 2) @@ (2 :> 1) @@ (3 :> 1) @@ (4 :> 2) \* H=4 +SizesBig == (1 :> 2) @@ (2 :> 1) @@ (3 :> 2) @@ (4 :> 1) @@ (5 :> 3) \* H=6 +SizesComposite == (1 :> 2) @@ (2 :> 2) @@ (3 :> 1) @@ (4 :> 1) @@ (5 :> 1) \* H=4 +SizesGCRipple == (1 :> 3) @@ (2 :> 3) \* H=3 +SizesInversion == (1 :> 3) @@ (2 :> 1) @@ (3 :> 2) \* H=3 + +(***************************************************************************) +(* Hand-simulated BUG-1 trace (SCRIPTED_EVENTLOOP, INSTANCES={1,2}, H=3, *) +(* Size=SizesEventLoop), verified against the action definitions above: *) +(* *) +(* 1. ClientRequestCreate(1): t=[deps={},bal=F] -> trig, clean -> *) +(* RequestCreate(1,T,F): ADA INSTANT_SUCCESS, cur={1@[0,3)}, *) +(* eCreated[1]=CLEAN, instState[1]=ALLOCATED. *) +(* 2. ClientRequestCreate(2): t=[deps={},bal=T] -> balC[2]=PENDING, *) +(* untriggered -> RequestCreate(2,F,F): instState[2]=CREATE_PENDING, *) +(* createWaiter[2]=T. *) +(* 3. ClientRequestDestroy(1): t=[deps={1,2},bal=F] (C1 ok: both *) +(* requested; C2 ok: 1 in deps). eCreated[2]=UNFIRED -> untrig -> *) +(* RequestDestroy(1,F,F): pendingAllocs empty -> push release *) +(* [1,ready=F,seq=1] (cc:858-859); destroyWaiter[1]=T. *) +(* 4. ClientRequestDestroy(2): t=[deps={2},bal=F] -> untrig; create(2) *) +(* still pending -> DELAYEDDESTROY (cc:845-849): *) +(* instState[2]=CREATE_PENDING_DESTROY, destroyWaiter[2]=T; NO *) +(* release entry pushed yet. (full cleanup) *) +(* 5. FireBallisticC(2): balC[2]=FIRED -> preC[2] fired clean. *) +(* 6. EnvTriggerCreate(2): TriggerCreate(2,F), dd=TRUE -> ADA(2,3): *) +(* cur full -> rebuild fut = cur - {rel 1} = empty -> fits -> DEF, *) +(* pendingAllocs=<<[2,3,lastSeq=1]>>, rel=cur; then cc:1146-1147 *) +(* pushes [2,ready=F,seq=2] and cc:1150-1153 frees it from fut *) +(* (fut ends empty). instState[2]=ALLOC_DEFERRED, createWaiter[2]=F. *) +(* 7. STUCK: EnvTriggerDestroy(1) needs eCreated[2] resolved; *) +(* eCreated[2] fires only on EVENTUAL_SUCCESS of 2, which needs the *) +(* release of 1. EnvTriggerDestroy(2) likewise. Nothing else *) +(* enabled; state is not Quiescent (pendingAllocs # <<>>) -> TLC *) +(* reports deadlock. This is BUG-1. *) +(***************************************************************************) + +=============================================================================== diff --git a/tla/allocation/Poison4.cfg b/tla/allocation/Poison4.cfg new file mode 100644 index 0000000000..7d6862711b --- /dev/null +++ b/tla/allocation/Poison4.cfg @@ -0,0 +1,52 @@ +\* Poison4: user-poison hunt at 4 instances. Targets (DESIGN.md s8): +\* BUG-4-escalated - remove_pending_release leaves rel := cur without the +\* surviving READY releases (cc:1556-1557 vs cc:1573); a later ARR +\* full-success swap (cc:1239) strands the tag in cur with its dealloc +\* notify already fired -> INV_NoOrphanTags / INV_QuiescentHeapEmpty. +\* BUG-5 - the poison rebuild never replays trailing pending allocs onto +\* fut (no trailing replay after cc:1562-1595) -> future overspending -> +\* INV_FutureOffsetConsistency / INV_InOrderUnblockSucceeds / +\* INV_NoOverlap. +\* BUG-6(b) - poison-path variant of the stranded-ready state -> +\* INV_NoReadyWhenNoPendingAllocs / INV_NoReadyAtRebuild. +\* TLC flags: -deadlock (invariant hunt; deadlock class owned by +\* EventLoop/Liveness). +\* Expected: FAIL - likely first INV_NoReadyWhenNoPendingAllocs (BUG-6); +\* iterate by commenting out confirmed-expected invariants and +\* re-running to reach the BUG-4/BUG-5 detectors. +\* Scale: sapling-likely (try local first; USER_POISON roughly doubles +\* the ballistic branching). +CONSTANTS + HEAP_SIZE = 4 + INSTANCES = {1, 2, 3, 4} + Size <- SizesSafety + USER_POISON = TRUE + C1_ENABLED = TRUE + C2_ENABLED = TRUE + C3_ENABLED = FALSE + CLIENT_MODE = "FREE" + FIX_CAP = FALSE + FIX_SWEEP = FALSE + FIX_RPR = FALSE +INIT MCInit +NEXT MCNext +INVARIANTS + TypeOK + INV_NoOverlap + INV_InBounds + INV_Conservation + INV_CurrentMatchesGround + INV_NoReadyWhenNoPendingAllocs + INV_NoReadyAtRebuild + INV_TriggeredDeallocPresent + INV_StructuralAsserts + INV_NoOrphanTags + INV_InOrderUnblockSucceeds + INV_FutureOffsetConsistency + INV_PoisonReplayOnlyFailsAfterPoint + INV_NoDupAlloc + SAFETY_PromisesKept + PROP_NotifyOnceSafety + INV_QuiescentHeapEmpty +CONSTRAINT + SeqCtrBound diff --git a/tla/allocation/Poison4Fixed.cfg b/tla/allocation/Poison4Fixed.cfg new file mode 100644 index 0000000000..43abb411dc --- /dev/null +++ b/tla/allocation/Poison4Fixed.cfg @@ -0,0 +1,50 @@ +\* Poison4Fixed: Poison4's bounds (4 instances, USER_POISON) with BOTH +\* fixes on. Validates FIX_SWEEP's poison-path coverage at scale: the +\* three-site sweep plus the BUG-4-standalone rel re-apply in +\* remove_pending_release mean the BUG-4/BUG-6 detector family +\* (INV_NoOrphanTags, INV_QuiescentHeapEmpty, INV_NoReadyWhenNoPendingAllocs, +\* INV_NoReadyAtRebuild, INV_CurrentMatchesGround) must now HOLD. +\* TLC flags: -deadlock (invariant hunt at scale; see SafetyFixed4.cfg +\* header - deadlock/drain liveness is owned by the local +\* bundle configs, Inversion deadlock-ON included). +\* Expected: FULLY GREEN, BUG-5 detectors included - FIX_RPR's trailing +\* replay closes BUG-5 (bugs/BUG-5.md), so a BUG-5-detector hit +\* here (INV_FutureOffsetConsistency / INV_InOrderUnblockSucceeds +\* / INV_NoOverlap) now means an FIX_RPR design bug on the +\* poison paths: stop, save the trace, report. +\* Scale: sapling (USER_POISON roughly doubles ballistic branching on +\* top of the Safety-scale space). +CONSTANTS + HEAP_SIZE = 4 + INSTANCES = {1, 2, 3, 4} + Size <- SizesSafety + USER_POISON = TRUE + C1_ENABLED = TRUE + C2_ENABLED = TRUE + C3_ENABLED = FALSE + CLIENT_MODE = "FREE" + FIX_CAP = TRUE + FIX_SWEEP = TRUE + FIX_RPR = TRUE +INIT MCInit +NEXT MCNext +INVARIANTS + TypeOK + INV_NoOverlap + INV_InBounds + INV_Conservation + INV_CurrentMatchesGround + INV_NoReadyWhenNoPendingAllocs + INV_NoReadyAtRebuild + INV_TriggeredDeallocPresent + INV_StructuralAsserts + INV_NoOrphanTags + INV_InOrderUnblockSucceeds + INV_FutureOffsetConsistency + INV_PoisonReplayOnlyFailsAfterPoint + INV_NoDupAlloc + SAFETY_PromisesKept + PROP_NotifyOnceSafety + INV_QuiescentHeapEmpty +CONSTRAINT + SeqCtrBound diff --git a/tla/allocation/PoisonHunt.cfg b/tla/allocation/PoisonHunt.cfg new file mode 100644 index 0000000000..de68eb12b8 --- /dev/null +++ b/tla/allocation/PoisonHunt.cfg @@ -0,0 +1,59 @@ +\* PoisonHunt: the UNPREEMPTED user-poison hunt at 4 instances. +\* Identical to Poison4.cfg except the two expected-FAIL invariants +\* (INV_NoReadyWhenNoPendingAllocs, INV_NoReadyAtRebuild) are removed from +\* the battery: sapling round 1 (slurm-77809-Poison4.out) confirmed BUG-6 on +\* the poison paths (887M generated / 272M distinct, 47min), and since TLC +\* halts at the first violation, that known-expected hit PREEMPTED the deep +\* hunts this config exists for. BUG-6 is deliberately NOT checked here - +\* already witnessed locally and at scale. +\* Purpose (DESIGN.md s8 targets, now reachable): +\* BUG-4-standalone - remove_pending_release leaves rel := cur without the +\* surviving READY releases (cc:1556-1557 vs cc:1573); a later ARR +\* full-success swap (cc:1239) strands the tag in cur with its dealloc +\* notify already fired. A hit on INV_NoOrphanTags / +\* INV_CurrentMatchesGround / INV_QuiescentHeapEmpty here is the FIRST +\* TLC witness of the standalone (poison-reached) variant. +\* BUG-5 (unfixed) - the poison rebuild never replays trailing pending +\* allocs onto fut (cc:1562-1595) -> future overspending -> a hit on +\* INV_FutureOffsetConsistency / INV_InOrderUnblockSucceeds / +\* INV_NoOverlap is the first unfixed-code BUG-5 witness (the composed +\* FIX_CAP witness already exists: traces/Inversion-bug5-deadlock.txt). +\* cc:1587 assert(found) - INV_PoisonReplayOnlyFailsAfterPoint. +\* TLC flags: -deadlock (invariant hunt; deadlock class owned by +\* EventLoop/Liveness). +\* Expected: a hit on any detector above = new witness, save trace + report; +\* GREEN = those bugs absent at these bounds. +\* Scale: sapling; likely longer than round-1 Poison4's 47min since it +\* will not stop early - use TLC -recover on resubmission. +CONSTANTS + HEAP_SIZE = 4 + INSTANCES = {1, 2, 3, 4} + Size <- SizesSafety + USER_POISON = TRUE + C1_ENABLED = TRUE + C2_ENABLED = TRUE + C3_ENABLED = FALSE + CLIENT_MODE = "FREE" + FIX_CAP = FALSE + FIX_SWEEP = FALSE + FIX_RPR = FALSE +INIT MCInit +NEXT MCNext +INVARIANTS + TypeOK + INV_NoOverlap + INV_InBounds + INV_Conservation + INV_CurrentMatchesGround + INV_TriggeredDeallocPresent + INV_StructuralAsserts + INV_NoOrphanTags + INV_InOrderUnblockSucceeds + INV_FutureOffsetConsistency + INV_PoisonReplayOnlyFailsAfterPoint + INV_NoDupAlloc + SAFETY_PromisesKept + PROP_NotifyOnceSafety + INV_QuiescentHeapEmpty +CONSTRAINT + SeqCtrBound diff --git a/tla/allocation/README.md b/tla/allocation/README.md new file mode 100644 index 0000000000..5ec001103e --- /dev/null +++ b/tla/allocation/README.md @@ -0,0 +1,106 @@ +# TLA+ model: Realm deferred instance allocation + +Models `LocalManagedMemory`'s deferred allocation/deletion protocol +(`src/realm/mem_impl.cc`): the three heap states (`current`, `future`, +`release` allocators), the `pending_allocs`/`pending_releases` queues, release +reordering, and poisoned-precondition cancellation, together with a client +bound by Realm's contract (topologically sorted requests, destroy-after-create). +The goal is to model-check the code's own assertions and its two promises — +"never say an allocation will succeed and then fail" and "never get stuck" — +and to adjudicate the bug hypotheses catalogued in `DESIGN.md` §8. + +## Files + +| File | Purpose | +|---|---| +| `DESIGN.md` | Authoritative C++→TLA mapping; every action cites `mem_impl.cc` lines. Read this first. | +| `DeferredAlloc.tla` | Protocol spec: allocator operators, state, actions. | +| `MCDeferredAlloc.tla` | Client/environment model, constants, ghost variables, invariants. | +| `Smoke/Safety/EventLoop/Liveness/Poison4/Big.cfg` | Configurations (constants, checks, expectations — see `DESIGN.md` §7 and the `run.sh` header). | +| `EXPECTED.md` | Expected pass/fail per config with the bug each failure corresponds to (written during verification). | +| `run.sh` | Local runner. | +| `sapling_tlc.sbatch` | Slurm script for the expensive configs on sapling. | +| `FUTURE-VERIFICATION.md` | Options for deeper verification (scale-run tuning, distributed TLC, symmetry/simulation, v2 model debt, C++-side harnesses) if ever wanted beyond the accepted baseline. | + +## Running locally + +```sh +./run.sh sany # parse-check only +./run.sh Smoke # one config +./run.sh # default sweep: Smoke, EventLoop, Safety, Liveness +``` + +Overrides: `JAVA`, `JAR`, `WORKERS`, `HEAP`, `JTMP` (see `run.sh` header). +Default jar: `../barrier/tools/tla2tools.jar`; default java: homebrew openjdk. + +## Running on sapling + +`Poison4` and `Big` (and `Safety` at larger bounds) are projected > 1 hour — +run them on sapling: + +```sh +# on sapling, with the repo cloned anywhere: +cd /tla/allocation +sbatch sapling_tlc.sbatch Poison4 +sbatch sapling_tlc.sbatch Big +``` + +Resource defaults (partition `cpu`, 24 h, 40 cpus, 128 GB) are guesses — +override on the command line (`sbatch -p ... -t ... -c ... --mem=...`). +TLC checkpoints hourly; see the sbatch header for `-recover` resume +instructions. + +## Status + +**TrailingRPR correction re-validated locally; sapling round 2 fully +unlocked.** The corrected bundle passed the entire hardened local matrix +(2026-08-26): fast set green incl. Inversion deadlock-ON (478/255), both +SafetyMini-scale full exhaustions green (SafetyMiniFixed 64.68M gen / +23.54M distinct; SweepOnly 53.30M / 20.55M), LivenessFixed green, and the +toggles-off regressions exact (EventLoop deadlock@7, SafetyMini +violation@9). Round 1 had confirmed BUG-6 at scale (4-instance Safety and +poison-path Poison4); the SafetyFixed4 INV_NoDupAlloc violation was +triaged as a **spec artifact** (FIX_RPR call-site wiring fed TrailingRPR +the full survivor list instead of the trailing remainder — two-line +correction applied, fix design unaffected; `bugs/DUPALLOC-TRIAGE.md`). +`INV_NoDupAlloc` is now checked in every bundle config. See +`SAPLING_JOBS.md`: ALL round-2 jobs submittable — SafetyHunt/PoisonHunt/ +Big (resume) plus the bundle jobs (SafetyFixed4/Poison4Fixed/BigFixed, +fresh starts). + +Prior milestone (still true modulo the triage): fix bundle (CAP+SWEEP+RPR) +verified locally. + +The candidate fixes are modeled as three spec toggles — `FIX_CAP` (BUG-1 +capped admission), `FIX_SWEEP` (BUG-6/BUG-4 stranded-ready sweep), and +`FIX_RPR` (BUG-5 trailing-alloc replay in remove_pending_release) — and +validated by the 2026-08-26 local matrix (EXPECTED.md "Fix validation" +section): the full bundle is green on every local config, including +full-exhaustion passes at the exact bounds where the base model fails and +**Inversion green with deadlock checking ON**. The bundle is indivisible: +the two-toggle round's Inversion deadlock +(`traces/Inversion-bug5-deadlock.txt`) proved pre-existing BUG-5 is +load-bearing for drain liveness under FIX_CAP — **FIX_CAP must not land in +C++ without FIX_RPR** (bugs/BUG-5.md). Regression: all toggles-FALSE +baselines reproduce exactly. + +Machine-confirmed Realm bug candidates, each with a saved trace and a +written report: + +- **BUG-1** (`bugs/BUG-1.md`) — deferred-create ordered at trigger time, not + request time → silent event-loop deadlock. Confirmed three ways: EventLoop + deadlock (7 states, `traces/EventLoop.trace.txt`), Smoke deadlock + (`traces/Smoke-run1.txt`), and a temporal-property lasso + (`traces/Liveness-bug1.txt`) whose same-bounds control + (`LivenessNoCross.cfg`, `traces/LivenessNoCross-pass.txt`) passes — + isolating the cross-instance destroy dependency as the cause. +- **BUG-6** (`bugs/BUG-6.md`) — the `assert(!it->is_ready)` at + mem_impl.cc:772 is reachable by a legal poison-free client + (`traces/SafetyMini.trace.txt`), and its **composite with the BUG-4 stale-release + mechanism is confirmed** (`Composite4.cfg`, `traces/Composite4.txt`): + permanent heap-range leak with the dealloc notify already fired (#442 + class), no poison involved. + +Per-config outcomes and iteration protocol live in `EXPECTED.md`. No Realm +source changes have been made — findings feed the bug reports first +(`DESIGN.md` §8). diff --git a/tla/allocation/SAPLING_JOBS.md b/tla/allocation/SAPLING_JOBS.md new file mode 100644 index 0000000000..2e84691761 --- /dev/null +++ b/tla/allocation/SAPLING_JOBS.md @@ -0,0 +1,229 @@ +# Sapling job list — deferred-allocation TLA+ campaign — ROUND 2 + +Round 1 (jobs 77808-77813, 2026-08-26) is complete; results are summarized +at the bottom of this file and recorded per-config in EXPECTED.md. Round 2 +exists because: + +1. **Both Group-A hunts were preempted by the expected BUG-6 violation** — + TLC halts at the first violation, so Safety and Poison4 confirmed BUG-6 + at scale (good) but never reached the bug classes they were submitted to + hunt. New checked-in configs **SafetyHunt.cfg** and **PoisonHunt.cfg** + are identical minus the two expected-FAIL invariants + (`INV_NoReadyWhenNoPendingAllocs`, `INV_NoReadyAtRebuild`), so the deep + detectors are now reachable. +2. **Poison4Fixed never ran** — round-1 submission typo. The config is + **`Poison4Fixed`** (job 77811 was submitted as `PoisonFixed4` and exited + immediately with "no such config"). Copy-paste the command below. +3. **Big / BigFixed were still clean but unfinished** at snapshot time — + Big (toggles off) is checkpointed and resumable; BigFixed is now HELD + (see below). +4. **Bundle jobs were gated on a local re-validation — the gate is now + CLEARED (2026-08-26):** the corrected bundle re-validated green across + the full local matrix (fast set + both SafetyMini-scale full + exhaustions + LivenessFixed) with exact toggles-off regressions. + §2(iii) is submittable. History below. + +## Bundle jobs (SafetyFixed4 / Poison4Fixed / BigFixed): re-validation history (gate CLEARED) + +Round 1's SafetyFixed4 violated **INV_NoDupAlloc** at depth 12 (12.8B +generated / 4.0B distinct, 14h10m). **Triage verdict +(`bugs/DUPALLOC-TRIAGE.md`): spec artifact** — the FIX_RPR call-site +wiring fed `TrailingRPR` the full survivor list instead of the trailing +remainder; a two-line DeferredAlloc.tla correction has been applied (the +fix DESIGN is unaffected; a second in-model flavor of the same wiring bug +— a kept alloc spuriously EVENTUAL_FAILED with its placement stranded in +`fut` — is covered by the same correction). Consequences for sapling: + +- **Local re-validation on the corrected spec: COMPLETE and green + (2026-08-26)** — SmokeFixed/EventLoopFixed/EventLoopCapOnly/ + Composite4Fixed/GCRipple/Inversion all pass (Inversion 478/255, + deadlock-ON), SafetyMiniFixed exhausts 64.68M gen / 23.54M distinct and + SafetyMiniSweepOnly 53.30M / 20.55M with zero violations, LivenessFixed + passes, and the toggles-off regressions reproduce their exact baselines + (EventLoop deadlock@7, SafetyMini violation@9). `INV_NoDupAlloc` is now + in every local bundle battery, closing the coverage gap that let + sapling catch this first. +- **Bundle checkpoints from round 1 are INVALID** (the spec changed): + SafetyFixed4 and BigFixed must restart FRESH — no `-recover`, and + delete `states/SafetyFixed4` / `states/BigFixed` before resubmitting. + BigFixed's round-1 progress in particular checked the dup detector + against the STALE wiring, so its "clean through 470M distinct" tells us + nothing about the corrected bundle. +- **Toggles-off jobs are unaffected** (`FIX_RPR = FALSE` behavior is + untouched by the correction): SafetyHunt / PoisonHunt / Big submit or + resume freely now. + +## 1. Get the tree onto sapling + +Same as round 1 — the new/changed files are `SafetyHunt.cfg`, +`PoisonHunt.cfg`, `EXPECTED.md`, this file: + +```sh +# from the laptop +rsync -av --exclude states --exclude jtmp --exclude 'slurm-*' \ + ~/realm/tla/ sapling:realm-tla/ +``` + +Note: rsyncing does NOT touch `states/` on sapling. **However: +DeferredAlloc.tla changed since round 1 (the TrailingRPR wiring +correction), and TLC checkpoint recovery requires the BYTE-IDENTICAL spec +that wrote the checkpoint** — the serialized state stream references the +string-intern table built at parse time, so ANY spec edit (even a +semantically inert one under `FIX_RPR = FALSE`) shifts the table and +recovery fails with `ValueInputStream: Can not unpickle a value of kind +` (observed 2026-08-27 attempting the Big resume). Consequence: ALL +round-1 checkpoints are invalid, Big's included. Rule for future rounds: +never edit the `.tla` files while a checkpointed run you intend to resume +is outstanding. + +## 2. Submit (round 2) + +```sh +cd realm-tla/allocation # submit FROM this dir (SLURM_SUBMIT_DIR) + +# (i) SUBMITTABLE NOW - the unpreempted Group-A hunts (toggles off, +# unaffected by the TrailingRPR correction) +sbatch -t 48:00:00 sapling_tlc.sbatch SafetyHunt +sbatch -t 24:00:00 sapling_tlc.sbatch PoisonHunt + +# (ii) Big (toggles off): round-1 checkpoint is UNRECOVERABLE (spec changed; +# see the note in §1 — recovery was attempted 2026-08-27 and failed +# with the unpickle error). Options: +# (a) RECOMMENDED: skip Big this round. Its expected outcome was an +# eventual BUG-6-family hit, already confirmed at scale twice; +# SafetyHunt/PoisonHunt are the informative toggles-off runs. +# (b) If queue time is cheap, restart fresh: +# rm -rf states/Big +# sbatch -t 48:00:00 sapling_tlc.sbatch Big +# (For reference, a VALID resume of an unchanged spec passes the +# checkpoint dir's ABSOLUTE path: -recover is a filesystem path +# resolved from the JVM's cwd, not an id looked up under the metadir.) + +# (iii) SUBMITTABLE NOW - the corrected bundle re-validated green locally +# on 2026-08-26 (full matrix incl. both SafetyMini-scale exhaustions, +# LivenessFixed, and exact toggles-off regressions; see EXPECTED.md). +# FRESH starts, no -recover - the spec changed: +rm -rf states/SafetyFixed4 states/BigFixed +sbatch sapling_tlc.sbatch SafetyFixed4 +sbatch sapling_tlc.sbatch Poison4Fixed # CORRECT NAME (round 1 + # typo'd it as "PoisonFixed4") +sbatch -t 48:00:00 sapling_tlc.sbatch BigFixed +``` + +## 2b. Round-2 outcome and round 3 (2026-08-27) + +All five round-2 jobs (77823-77827) died simultaneously at ~05:18 after +~4.6-4.9 h — no violations, no completions. Cause: TLC's disk state queues +were on shared `/scratch2` (a design mistake in this script's round-1/2 +version), which exhausted the shared filesystem and impacted other users. +The states were deleted to unblock the cluster, so round-2 checkpoints are +GONE — round 3 restarts from zero. Clean-so-far bounds from round 2 (still +valid as bounded-verification evidence, all no-violation): SafetyFixed4 +989M distinct @ depth 10, Poison4Fixed 1.21B @ 10, SafetyHunt 1.02B @ 10, +PoisonHunt 1.16B @ 11, BigFixed 900M @ 9. + +**HARD RULE (recorded 2026-08-27): never put TLC data on /scratch on +sapling — this or any other work.** The script now keeps the metadir on +node-local /tmp ($SLURM_TMPDIR when set) and checkpoint sync-back to the +submit dir is DEFAULT OFF (opt in per-run with SYNC_CHECKPOINT=1 only if a +single bounded checkpoint write to shared storage is acceptable). With sync +off, an interrupted run restarts from zero — size time limits so runs +finish in one shot. + +Round 3: fresh starts. The jobs are INDEPENDENT — the `-d afterany` chain +below is optional: it guarantees the gate job (SafetyFixed4) runs first, +limits the footprint to one node at a time, and prevents two jobs from +sharing one node's /tmp if nodes are wider than 40 cores. If the queue is +quiet, submitting all four flat (no `-d`, add `--exclusive` on wide nodes) +is equally correct and finishes ~4x sooner in wall-clock. Re-sync the tree +first (the sbatch script changed): + +```sh +cd /scratch2/mebauer/tla/allocation +# --parsable makes sbatch print just the job id, so chaining is automatic +J1=$(sbatch --parsable -t 48:00:00 sapling_tlc.sbatch SafetyFixed4) # the C++ gate +J2=$(sbatch --parsable -t 48:00:00 -d afterany:$J1 sapling_tlc.sbatch Poison4Fixed) +J3=$(sbatch --parsable -t 48:00:00 -d afterany:$J2 sapling_tlc.sbatch SafetyHunt) +J4=$(sbatch --parsable -t 48:00:00 -d afterany:$J3 sapling_tlc.sbatch PoisonHunt) +echo "queued: $J1 -> $J2 -> $J3 -> $J4" +# BigFixed: dropped (largest state space, least marginal info vs SafetyFixed4) +``` + +Gate criterion (BFS ⇒ "clean through depth D" = complete coverage of all +behaviors of length ≤ D): the only scale-level failure ever observed was at +depth 12 (round-1 SafetyFixed4, stale wiring). **SafetyFixed4 clean through +completed depth 13 (log shows `Progress(14)`) passes the round-1 failure +point and opens the C++ gate**, with the local full exhaustions as the +semantic backbone — full exhaustion of these spaces is a bonus, not a +requirement. + +## 2c. Round-3 outcome (2026-08-27, jobs 77843/77853/77854/77855) + +All four died on node-local disk exhaustion — they ran the PRE-gzip script +(the tree was not re-synced before submission). All violation-free at +death: Poison4Fixed clean through depth 10 @ 1.90B distinct (4h25m, best +bundle+poison coverage yet); SafetyFixed4 depth 9 @ 1.10B (node had only +261G free at start); SafetyHunt depth 10 @ 1.67B; PoisonHunt depth 10 @ +1.42B. Measured queue cost ~210-270 bytes/state uncompressed → with -gzip +(now default in the script) a clean 733G node holds roughly 15-30B queued +states, which should reach the depth-13 gate. Round-4 prep: (1) RSYNC THE +TREE (the missed step), (2) sweep leftover /tmp/mebauer-tlc-* dirs off the +compute nodes (scancel'ed jobs can't finish their cleanup trap in the kill +grace window), (3) resubmit the serial chain with --exclusive; verify the +job header shows `-gzip` in extra and "checkpoint-sync=OFF". + +Site defaults (partition `cpu`, 40 cpus, 128 G, java discovery) are at the +top of `sapling_tlc.sbatch`, overridable per submission. The script +auto-appends `-deadlock` for every config named here (deadlock checking +stays on only for the small local Smoke/EventLoop-family runs); SafetyHunt +and PoisonHunt fall under the same default. + +## 3. What each round-2 job hunts + +| Job | Toggles | Hunts | Expected outcome | +|---|---|---|---| +| SafetyHunt (Safety minus BUG-6 invariants) | off | **BUG-3 class**: in-order unblock soundness (`INV_InOrderUnblockSucceeds`, cc:1668-1670) and future-offset determinism (`INV_FutureOffsetConsistency`, cc:1674-1691) after ARR partial-path history rewrites; plus the full remaining battery | **GREEN = BUG-3 absent at these bounds.** A detector hit = first BUG-3 witness → save trace, Phase 5. | +| PoisonHunt (Poison4 minus BUG-6 invariants) | off | **BUG-4-standalone** (`INV_NoOrphanTags`/`INV_CurrentMatchesGround`/`INV_QuiescentHeapEmpty`), **unfixed BUG-5** (`INV_FutureOffsetConsistency`/`INV_InOrderUnblockSucceeds`/`INV_NoOverlap`), **cc:1587** (`INV_PoisonReplayOnlyFailsAfterPoint`) | Any hit = first TLC witness of that variant → save trace, Phase 5. GREEN = absent at these bounds. | +| Poison4Fixed | bundle | FIX_SWEEP poison-path coverage + FIX_RPR trailing replay under USER_POISON | FULLY GREEN (incl. INV_NoDupAlloc). **Gated**: submit only after the corrected bundle re-validates locally; fresh start. | +| Big (resume) | off | Open hunt continuation (clean through 2.87B distinct, depth 9) | **Submittable now** (toggles-off, unaffected by the correction). Known markers first (BUG-6 family will eventually fire here too and end the run — when it does, treat as confirmation and stop; a BigHunt variant is only worth creating if Big's BUG-6 hit comes early). | +| SafetyFixed4 (fresh) | bundle | Fixed-model soundness at ARR-partial scale, now with the CORRECTED TrailingRPR wiring | FULLY GREEN (incl. INV_NoDupAlloc — the round-1 violation was a spec artifact, corrected). **Gated** on local re-validation; fresh start, delete `states/SafetyFixed4` first. | +| BigFixed (fresh) | bundle | Largest fixed-model sweep | **Gated + fresh start** — the round-1 checkpoint ran the stale wiring and is invalid; delete `states/BigFixed` first. | + +## 4. Expected wall-times (round-1 calibrated) + +Round-1 sapling throughput at 40 workers: ~14-18M generated/min (Safety), +~19M/min (Poison4), ~10M/min at depth 9+ (Big-sized states). + +- **SafetyHunt**: Safety hit BUG-6 at 4.4B generated / 5h04m *while + stopping early*; SafetyHunt explores the same space to exhaustion or a + BUG-3 hit — plan for **>5h, possibly 24h+**; submit with `-t 48:00:00` + and expect a `-recover` cycle. +- **PoisonHunt**: Poison4 hit BUG-6 at 887M / 47min; the unpreempted space + is larger — **several hours**, `-t 24:00:00` should suffice. +- **Big resume**: unknown total; keep `-t 48:00:00` and `-recover` cycles. +- **BigFixed fresh start** (when un-gated): budget the full run again + (round-1's ~2h reached 470M distinct on the stale wiring); `-t 48:00:00`. + +## 5. Bring back + +- `slurm--.out` for every job (full TLC output incl. any + counterexample trace and the summary lines). +- On any **detector hit** in SafetyHunt/PoisonHunt: that log contains a + first-of-its-kind witness — bring it back immediately, don't wait for the + other jobs. + +## 6. Round-1 results (2026-08-26, jobs 77808-77813) + +| Job | Config | Outcome | +|---|---|---| +| 77808 | Safety | **BUG-6(a) CONFIRMED at 4-instance scale** — INV_NoReadyWhenNoPendingAllocs at depth 10; 4.4B gen / 1.44B distinct, 5h04m. Expected; deep hunt preempted → SafetyHunt. | +| 77809 | Poison4 | **BUG-6 CONFIRMED on poison paths** — same invariant; 887M gen / 272M distinct, 47min. Expected; deep hunts preempted → PoisonHunt. | +| 77810 | SafetyFixed4 | **UNEXPECTED: INV_NoDupAlloc violated** at depth 12; 12.8B gen / 4.0B distinct, 14h10m. Trace: `slurm-77810-SafetyFixed4.out` line 901 on. **Triaged: spec artifact** (TrailingRPR call-site wiring; corrected in DeferredAlloc.tla — fix design unaffected). | +| 77811 | — | Submission typo (`PoisonFixed4`); Poison4Fixed never ran. | +| 77812 | Big | Clean through 6.26B gen / 2.87B distinct, depth 9 (~12h); checkpointed, resumable. | +| 77813 | BigFixed | Clean through 1.06B gen / 470M distinct, depth 9; checkpointed, resumable. | + +Round-1 positives worth keeping in mind: the two expected BUG-6 +confirmations extend the local witnesses to 4-instance scale and to the +poison paths, and neither Big run found anything new in ~3B combined +distinct states. diff --git a/tla/allocation/Safety.cfg b/tla/allocation/Safety.cfg new file mode 100644 index 0000000000..bf0b8b45f0 --- /dev/null +++ b/tla/allocation/Safety.cfg @@ -0,0 +1,56 @@ +\* Safety: main local safety hunt at the 4-instance scale that makes the +\* attempt_release_reordering PARTIAL path (cc:1253-1310) and the cc:768 +\* future-rebuild reachable (needs >=2 pending allocs + >=2 pending +\* releases with one ready). +\* Sizes (2,1,1,2) on H=4 cover the BUG-6(a) 4-instance recipe from +\* DESIGN.md s8; NOTE that TLC (BFS) will likely find the SHORTER canonical +\* 3-instance witness first (see EXPECTED.md - Phase 2 already produced it): +\* destroy(i1) deferred = R1; create i2 -> DEFERRED lastSeq=seq(R1); +\* destroy(i2) deferred = R2 (legal: request != trigger); +\* destroy(i3) triggered -> ARR gate fails -> R3 pushed READY; +\* trigger destroy(i1) -> oldest drain places i2, pendingAllocs empties, +\* do-while stops at non-ready R2 -> ready R3 stranded. +\* TLC flags: -deadlock (deadlock checking OFF). Rationale: BUG-1 deadlock +\* traces are short and would preempt the deeper invariant hunts +\* (TLC halts at the first violation); EventLoop/Liveness own the +\* deadlock class. Deviation from the DESIGN.md s7 "dlk" mark, +\* documented in EXPECTED.md. +\* Expected: FAIL - INV_NoReadyWhenNoPendingAllocs (BUG-6). After recording +\* that trace, comment it AND INV_NoReadyAtRebuild out and re-run +\* to continue hunting (BUG-3 territory: INV_InOrderUnblockSucceeds, +\* INV_FutureOffsetConsistency). +\* Scale: local, minutes-hours, ~10^7-10^9 states; sapling fallback. +CONSTANTS + HEAP_SIZE = 4 + INSTANCES = {1, 2, 3, 4} + Size <- SizesSafety + USER_POISON = FALSE + C1_ENABLED = TRUE + C2_ENABLED = TRUE + C3_ENABLED = FALSE + CLIENT_MODE = "FREE" + FIX_CAP = FALSE + FIX_SWEEP = FALSE + FIX_RPR = FALSE +INIT MCInit +NEXT MCNext +INVARIANTS + TypeOK + INV_NoOverlap + INV_InBounds + INV_Conservation + INV_CurrentMatchesGround + INV_NoReadyWhenNoPendingAllocs + INV_NoReadyAtRebuild + INV_TriggeredDeallocPresent + INV_StructuralAsserts + INV_NoOrphanTags + INV_InOrderUnblockSucceeds + INV_FutureOffsetConsistency + INV_PoisonReplayOnlyFailsAfterPoint + INV_NoDupAlloc + SAFETY_PromisesKept + PROP_NotifyOnceSafety + INV_QuiescentHeapEmpty +CONSTRAINT + SeqCtrBound diff --git a/tla/allocation/SafetyFixed4.cfg b/tla/allocation/SafetyFixed4.cfg new file mode 100644 index 0000000000..f6c95b3828 --- /dev/null +++ b/tla/allocation/SafetyFixed4.cfg @@ -0,0 +1,53 @@ +\* SafetyFixed4: Safety's 4-instance bounds with the full fix bundle (CAP+SWEEP+RPR) on - the +\* fixed-model soundness run at the scale where the ARR partial path +\* (cc:1253-1310) and the cc:768 rebuild are reachable. The BUG-6 pair +\* (INV_NoReadyWhenNoPendingAllocs / INV_NoReadyAtRebuild) is IN the +\* battery and must now HOLD (FIX_SWEEP). +\* TLC flags: -deadlock (invariant soundness hunt at scale). Deadlock/ +\* drain liveness for the bundle is owned locally by SmokeFixed/ +\* EventLoopFixed/GCRipple/Inversion (Inversion runs deadlock-ON +\* and is green under FIX_RPR); keeping the flag off here keeps +\* the scale run focused on the invariant battery and avoids +\* halting on any not-yet-registered liveness shape without its +\* invariant fingerprint. +\* Expected: FULLY GREEN, BUG-5 detectors included - the three-toggle +\* bundle (FIX_RPR trailing replay) now addresses BUG-5. Any +\* violation is a fix-design flaw at scale or a new candidate +\* (highest interest - report with trace, do not iterate past +\* it). +\* Scale: sapling (base Safety exceeded 110M distinct locally; the +\* fixed space is the same order). +CONSTANTS + HEAP_SIZE = 4 + INSTANCES = {1, 2, 3, 4} + Size <- SizesSafety + USER_POISON = FALSE + C1_ENABLED = TRUE + C2_ENABLED = TRUE + C3_ENABLED = FALSE + CLIENT_MODE = "FREE" + FIX_CAP = TRUE + FIX_SWEEP = TRUE + FIX_RPR = TRUE +INIT MCInit +NEXT MCNext +INVARIANTS + TypeOK + INV_NoOverlap + INV_InBounds + INV_Conservation + INV_CurrentMatchesGround + INV_NoReadyWhenNoPendingAllocs + INV_NoReadyAtRebuild + INV_TriggeredDeallocPresent + INV_StructuralAsserts + INV_NoOrphanTags + INV_InOrderUnblockSucceeds + INV_FutureOffsetConsistency + INV_PoisonReplayOnlyFailsAfterPoint + INV_NoDupAlloc + SAFETY_PromisesKept + PROP_NotifyOnceSafety + INV_QuiescentHeapEmpty +CONSTRAINT + SeqCtrBound diff --git a/tla/allocation/SafetyHunt.cfg b/tla/allocation/SafetyHunt.cfg new file mode 100644 index 0000000000..3a7cc98b09 --- /dev/null +++ b/tla/allocation/SafetyHunt.cfg @@ -0,0 +1,53 @@ +\* SafetyHunt: the UNPREEMPTED BUG-3 hunt at the 4-instance scale. +\* Identical to Safety.cfg except the two expected-FAIL invariants +\* (INV_NoReadyWhenNoPendingAllocs, INV_NoReadyAtRebuild) are removed from +\* the battery: sapling round 1 (slurm-77808-Safety.out) confirmed BUG-6(a) +\* at scale (4.4B generated / 1.44B distinct, 5h04m, trace in that log), and +\* since TLC halts at the first violation, that known-expected hit PREEMPTED +\* the deep hunt this config exists for. BUG-6 is deliberately NOT checked +\* here - it is already witnessed (SafetyMini.trace.txt locally, slurm-77808 +\* at scale). +\* Purpose: BUG-3 territory - soundness of the in-order unblock assert(ok) +\* (cc:1668-1670 -> INV_InOrderUnblockSucceeds) and the future- +\* offset cross-check (cc:1674-1691 -> INV_FutureOffsetConsistency) +\* after attempt_release_reordering partial-success rewrites +\* history; plus the full remaining safety battery. +\* TLC flags: -deadlock (deadlock checking OFF; deadlock class owned by +\* EventLoop/Liveness). +\* Expected: GREEN = BUG-3 absent at these bounds. A hit on +\* INV_InOrderUnblockSucceeds or INV_FutureOffsetConsistency is +\* the FIRST BUG-3 witness - save the trace and report. +\* Scale: sapling; likely LONGER than round-1 Safety's 5h04m since it +\* will not stop early - use TLC -recover on resubmission. +CONSTANTS + HEAP_SIZE = 4 + INSTANCES = {1, 2, 3, 4} + Size <- SizesSafety + USER_POISON = FALSE + C1_ENABLED = TRUE + C2_ENABLED = TRUE + C3_ENABLED = FALSE + CLIENT_MODE = "FREE" + FIX_CAP = FALSE + FIX_SWEEP = FALSE + FIX_RPR = FALSE +INIT MCInit +NEXT MCNext +INVARIANTS + TypeOK + INV_NoOverlap + INV_InBounds + INV_Conservation + INV_CurrentMatchesGround + INV_TriggeredDeallocPresent + INV_StructuralAsserts + INV_NoOrphanTags + INV_InOrderUnblockSucceeds + INV_FutureOffsetConsistency + INV_PoisonReplayOnlyFailsAfterPoint + INV_NoDupAlloc + SAFETY_PromisesKept + PROP_NotifyOnceSafety + INV_QuiescentHeapEmpty +CONSTRAINT + SeqCtrBound diff --git a/tla/allocation/SafetyMini.cfg b/tla/allocation/SafetyMini.cfg new file mode 100644 index 0000000000..96185abde7 --- /dev/null +++ b/tla/allocation/SafetyMini.cfg @@ -0,0 +1,46 @@ +\* SafetyMini: local reproducer for BUG-6 at the canonical 3-instance scale +\* (added during Phase 2 validation - the full Safety.cfg at 4 instances is +\* ~10^9 states and needs sapling). +\* Shape: H=3, sizes (2,2,1). i1(2)+i3(1) fill the heap; i2(2) can only be +\* planned against a pending release; freeing i3 alone leaves a size-1 gap so +\* the ARR front-gate fails and a triggered destroy(i3) is pushed READY; +\* draining destroy(i1) then places i2 and empties pendingAllocs with the +\* ready entry stranded behind the non-ready destroy(i2) entry. +\* TLC flags: -deadlock (deadlock class owned by EventLoop/Liveness). +\* Expected: FAIL - INV_NoReadyWhenNoPendingAllocs (BUG-6), the canonical +\* witness from EXPECTED.md. +\* Scale: local, minutes. +CONSTANTS + HEAP_SIZE = 3 + INSTANCES = {1, 2, 3} + Size <- SizesLiveness + USER_POISON = FALSE + C1_ENABLED = TRUE + C2_ENABLED = TRUE + C3_ENABLED = FALSE + CLIENT_MODE = "FREE" + FIX_CAP = FALSE + FIX_SWEEP = FALSE + FIX_RPR = FALSE +INIT MCInit +NEXT MCNext +INVARIANTS + TypeOK + INV_NoOverlap + INV_InBounds + INV_Conservation + INV_CurrentMatchesGround + INV_NoReadyWhenNoPendingAllocs + INV_NoReadyAtRebuild + INV_TriggeredDeallocPresent + INV_StructuralAsserts + INV_NoOrphanTags + INV_InOrderUnblockSucceeds + INV_FutureOffsetConsistency + INV_PoisonReplayOnlyFailsAfterPoint + INV_NoDupAlloc + SAFETY_PromisesKept + PROP_NotifyOnceSafety + INV_QuiescentHeapEmpty +CONSTRAINT + SeqCtrBound diff --git a/tla/allocation/SafetyMiniFixed.cfg b/tla/allocation/SafetyMiniFixed.cfg new file mode 100644 index 0000000000..064250361d --- /dev/null +++ b/tla/allocation/SafetyMiniFixed.cfg @@ -0,0 +1,47 @@ +\* SafetyMiniFixed: the canonical BUG-6 reproducer bounds with the full fix bundle (CAP+SWEEP+RPR) +\* on. FIX_SWEEP performs the stranded-ready sweep at the +\* pendingAllocs->empty transitions, so the ready entry can no longer be +\* stranded behind a non-ready one. +\* TLC flags: -deadlock (matched to SafetyMini.cfg for a controlled +\* comparison; the deadlock class is covered by SmokeFixed / +\* EventLoopFixed). +\* Expected: FULLY GREEN - in particular INV_NoReadyWhenNoPendingAllocs +\* and INV_NoReadyAtRebuild (the base config's expected-FAIL +\* pair) now PASS. A violation of either => FIX_SWEEP design +\* bug; any other violation => fix regression - Phase 5. +\* Scale: local, tens of seconds (base found BUG-6 in 10s at 5.5M +\* states; the fixed space is comparable). +CONSTANTS + HEAP_SIZE = 3 + INSTANCES = {1, 2, 3} + Size <- SizesLiveness + USER_POISON = FALSE + C1_ENABLED = TRUE + C2_ENABLED = TRUE + C3_ENABLED = FALSE + CLIENT_MODE = "FREE" + FIX_CAP = TRUE + FIX_SWEEP = TRUE + FIX_RPR = TRUE +INIT MCInit +NEXT MCNext +INVARIANTS + TypeOK + INV_NoOverlap + INV_InBounds + INV_Conservation + INV_CurrentMatchesGround + INV_NoReadyWhenNoPendingAllocs + INV_NoReadyAtRebuild + INV_TriggeredDeallocPresent + INV_StructuralAsserts + INV_NoOrphanTags + INV_InOrderUnblockSucceeds + INV_FutureOffsetConsistency + INV_PoisonReplayOnlyFailsAfterPoint + INV_NoDupAlloc + SAFETY_PromisesKept + PROP_NotifyOnceSafety + INV_QuiescentHeapEmpty +CONSTRAINT + SeqCtrBound diff --git a/tla/allocation/SafetyMiniSweepOnly.cfg b/tla/allocation/SafetyMiniSweepOnly.cfg new file mode 100644 index 0000000000..9b87051417 --- /dev/null +++ b/tla/allocation/SafetyMiniSweepOnly.cfg @@ -0,0 +1,46 @@ +\* SafetyMiniSweepOnly: attribution isolation - the canonical BUG-6 +\* reproducer with ONLY FIX_SWEEP enabled (FIX_CAP off). BUG-6 must be +\* fixed by the sweep alone; the cap addresses the independent BUG-1 +\* mechanism. Note BUG-1 deadlocks still exist in this mode, which is why +\* the flags stay -deadlock (invariant hunt only). +\* TLC flags: -deadlock. +\* Expected: GREEN - in particular INV_NoReadyWhenNoPendingAllocs and +\* INV_NoReadyAtRebuild now PASS at the exact bounds where the +\* base config violates them in 10s. A violation here but not +\* in SafetyMiniFixed would mean the cap is (wrongly) +\* load-bearing for BUG-6 - an attribution error. +\* Scale: local, tens of seconds. +CONSTANTS + HEAP_SIZE = 3 + INSTANCES = {1, 2, 3} + Size <- SizesLiveness + USER_POISON = FALSE + C1_ENABLED = TRUE + C2_ENABLED = TRUE + C3_ENABLED = FALSE + CLIENT_MODE = "FREE" + FIX_CAP = FALSE + FIX_SWEEP = TRUE + FIX_RPR = FALSE +INIT MCInit +NEXT MCNext +INVARIANTS + TypeOK + INV_NoOverlap + INV_InBounds + INV_Conservation + INV_CurrentMatchesGround + INV_NoReadyWhenNoPendingAllocs + INV_NoReadyAtRebuild + INV_TriggeredDeallocPresent + INV_StructuralAsserts + INV_NoOrphanTags + INV_InOrderUnblockSucceeds + INV_FutureOffsetConsistency + INV_PoisonReplayOnlyFailsAfterPoint + INV_NoDupAlloc + SAFETY_PromisesKept + PROP_NotifyOnceSafety + INV_QuiescentHeapEmpty +CONSTRAINT + SeqCtrBound diff --git a/tla/allocation/Smoke.cfg b/tla/allocation/Smoke.cfg new file mode 100644 index 0000000000..347e172331 --- /dev/null +++ b/tla/allocation/Smoke.cfg @@ -0,0 +1,43 @@ +\* Smoke: fast parse/typecheck/sanity gate - GREEN BASELINE. +\* Purpose: everything parses, all checked invariants hold on a tiny space. +\* NOTE: the expected-FAIL invariants (INV_NoReadyWhenNoPendingAllocs, +\* INV_NoReadyAtRebuild = BUG-6) are deliberately NOT checked here +\* so Smoke stays green; Safety owns them. +\* TLC flags: NONE (deadlock checking ON). +\* Expected: clean pass, OR a deadlock counterexample of the BUG-1 shape +\* (a wait cycle rooted in an ALLOC_DEFERRED instance - reachable +\* even at this scale; both outcomes are non-alarming, see +\* EXPECTED.md). Any INVARIANT violation here means a spec bug, +\* not a Realm bug. +\* Scale: local, seconds-minutes, < 10^6 states. +CONSTANTS + HEAP_SIZE = 3 + INSTANCES = {1, 2} + Size <- SizesSmoke + USER_POISON = FALSE + C1_ENABLED = TRUE + C2_ENABLED = TRUE + C3_ENABLED = FALSE + CLIENT_MODE = "FREE" + FIX_CAP = FALSE + FIX_SWEEP = FALSE + FIX_RPR = FALSE +INIT MCInit +NEXT MCNext +INVARIANTS + TypeOK + INV_NoOverlap + INV_InBounds + INV_Conservation + INV_CurrentMatchesGround + INV_TriggeredDeallocPresent + INV_StructuralAsserts + INV_NoOrphanTags + INV_InOrderUnblockSucceeds + INV_FutureOffsetConsistency + INV_PoisonReplayOnlyFailsAfterPoint + SAFETY_PromisesKept + PROP_NotifyOnceSafety + INV_QuiescentHeapEmpty +CONSTRAINT + SeqCtrBound diff --git a/tla/allocation/SmokeFixed.cfg b/tla/allocation/SmokeFixed.cfg new file mode 100644 index 0000000000..52468c028b --- /dev/null +++ b/tla/allocation/SmokeFixed.cfg @@ -0,0 +1,46 @@ +\* SmokeFixed: Smoke's bounds and FREE client with the full fix bundle (CAP+SWEEP+RPR) on +\* (FIX_CAP = BUG-1 capped admission, FIX_SWEEP = BUG-6/BUG-4 sweep). +\* The BUG-6 expected-FAIL invariants are ADDED here (they must now hold). +\* TLC flags: NONE (deadlock checking ON - the whole point: the BUG-1 +\* deadlock class must be gone; capped rejections cascade via +\* poison and every run drains to Quiescent or a documented +\* poisoned-destroy leak terminal). +\* Expected: FULLY GREEN, including the deadlock check. +\* A deadlock => FIX_CAP design bug (wait-graph not a DAG). +\* A BUG-6-invariant violation => FIX_SWEEP design bug. +\* Any other violation => fix regression or spec bug - Phase 5. +\* Scale: local, seconds-minutes. +CONSTANTS + HEAP_SIZE = 3 + INSTANCES = {1, 2} + Size <- SizesSmoke + USER_POISON = FALSE + C1_ENABLED = TRUE + C2_ENABLED = TRUE + C3_ENABLED = FALSE + CLIENT_MODE = "FREE" + FIX_CAP = TRUE + FIX_SWEEP = TRUE + FIX_RPR = TRUE +INIT MCInit +NEXT MCNext +INVARIANTS + TypeOK + INV_NoOverlap + INV_InBounds + INV_Conservation + INV_CurrentMatchesGround + INV_NoReadyWhenNoPendingAllocs + INV_NoReadyAtRebuild + INV_TriggeredDeallocPresent + INV_StructuralAsserts + INV_NoOrphanTags + INV_InOrderUnblockSucceeds + INV_FutureOffsetConsistency + INV_PoisonReplayOnlyFailsAfterPoint + INV_NoDupAlloc + SAFETY_PromisesKept + PROP_NotifyOnceSafety + INV_QuiescentHeapEmpty +CONSTRAINT + SeqCtrBound diff --git a/tla/allocation/bugs/BLUEPRINT-REVIEW.md b/tla/allocation/bugs/BLUEPRINT-REVIEW.md new file mode 100644 index 0000000000..b94e95fc52 --- /dev/null +++ b/tla/allocation/bugs/BLUEPRINT-REVIEW.md @@ -0,0 +1,303 @@ +# Blueprint review: FIX_CAP / FIX_SWEEP as landed in DeferredAlloc.tla + +Reviewer: blueprint-review fork. Scope: (1) adversarial soundness review of the +landed fix semantics (DeferredAlloc.tla ~1152 lines: `ApplyCappedFrees`, +`CanonReplay`, `ADAResCap`, `ADAResv`, `Sweep`, `reqCap`) against the agreed +design (bugs/BUG-1.md, bugs/BUG-6.md, bugs/FIX-REVIEW.md, and Mike's +accepted-behavior-change decision); (2) the definitive C++ change list the spec +corresponds to. No TLC runs were performed here (verification fork owns that); +all soundness arguments are on-paper against the spec text and mem_impl.{h,cc,inl}. + +## 0. Headline caveat (read first): the spec models the PURE cap, not BUG-1.md's union rule + +The landed `ADAResCap` funds an admission from **{surviving releases with +`seq <= cap`} only** (spec line ~299: "NO ready-fold in v1"). bugs/BUG-1.md's +recommendation section still describes the fuller repaired rule — cap **∪ +releases whose precondition has already triggered clean** — including an +implementation item ("PendingRelease stores its precondition Event") that the +pure cap does **not** need. Both are sound; the pure cap is strictly more +conservative and was chosen as v1 after Mike accepted spurious instant-fails +("I would rather be correct... even if Realm's behavior might change"). + +Consequences: + +- **What TLC validates is the pure cap.** The union term is UNVERIFIED. The + C++ v1 must implement the pure cap exactly as modeled. If the union term is + ever wanted (it saves the GC-ripple pattern under memory pressure when + funding destroys are stuck as ready entries behind other pending work), + extend `ApplyCappedFrees`/`CanonReplay` to also free `is_ready` survivors + with `seq > cap` and re-run the whole matrix FIRST. +- v1 does NOT need `PendingRelease` to store its precondition `Event` — drop + that item from the v1 change list (a real simplification vs BUG-1.md). +- BUG-1.md should gain a short "v1 as modeled/verified = pure cap" note + (documentation action for the report owner; not edited here per scope). +- Note the pure cap still gets most of ARR's opportunism for free: ARR funds + only from `current` + already-triggered (ready) releases, which are + dependency sinks — see §1.9. + +Verdict for §0: **NEEDS-DOCUMENTATION, not a spec issue.** No blocker. + +## 1. Soundness verdicts on the eight design decisions (+ ARR interaction) + +### 1.1 cur-first fast path kept under FIX_CAP — SOUND +`ADAResCap` guards the fast path with `paA = <<>> /\ CanAlloc(curA, sz)` +(spec ~286), preserving the legacy pending-allocs-empty gate (cc:754-757). +With pending allocs it falls through to the monotone guard + capped test — +never a direct-cur admission, so queue monotonicity cannot be bypassed. +Cur-funded allocations consume only space free *now*; no dependency on any +pending release, hence no cycle risk. The legacy `prA = <<>> -> IF` special +case (cc:762-765) is subsumed: an empty capped funding set degenerates the +test to cur-only, which already failed. + +### 1.2 missing-ok frees in the capped/canonical replay — SOUND +Mirrors cc:778. A missing tag can only make the replay state *less* free +(the free no-ops), i.e. the capped test can only under-admit — the safe +direction (spurious IF, accepted). Over-admission via missing-ok is +impossible; v1's one-release-per-instance rule excludes double-entry masking. +The DELAYEDDESTROY self-release (`seq = cap + k > cap`) is excluded from the +admission bound and applied after the alloc in the fut rebuild — matching +cc:1146-1153 semantics and guaranteeing an instance never funds itself. + +### 1.3 queued-alloc placement failure in canonical replay -> flag + IF — SOUND (TLC is the adjudicator) +The determinism claim: every queued alloc re-places successfully in a +canonical replay from the current `cur`. The inductive argument: (i) at its +own admission each alloc was verified against the canonical state; (ii) the +drain applies operations in exactly canonical order, and placement agreement +between drain and canonical fut is `INV_FutureOffsetConsistency` — checked by +TLC under FIX_CAP; (iii) after an ARR swap, `cur' = g.test` and the remaining +survivors are all non-ready, so `CanonReplay(cur', ...)` replays exactly +ARR's own successful `Replay` (same gate `seq <= lastSeq`, same base) — the +directive's erased-ready concern resolves because erased ready entries' +space is already inside `cur'` and they are gone from the list, while +ready-but-unerased entries still hold their tags in `cur`, which is exactly +the state `CanonReplay` frees them from. I could not construct a divergence. +The spec surfaces any residual hole via `capAssert -> structuralAssertFailed` +so TLC hunts it; **the C++ must carry the same assert** (§2.3). + +### 1.4 fut unchanged on capped-test failure — SOUND +Every `fut` read in the module is guarded by `pendingAllocs # <<>>` (the +validity convention): the ADA nonempty test, RequestDestroy's fut frees, the +TriggerCreate dd-free, UnblockScan's cross-check, ARR's pass-through. On a +failed admission with the queue empty, fut is invalid-by-convention and the +next successful admission rewrites it from scratch (`ADAResCap` never reads +the stale value). Legacy's cc:790-791 stale write is dead state in legacy +too; not writing it is safe and cleaner. C++: do not clobber +`future_allocator` on failed admissions. + +### 1.5 ready survivors with seq <= cap DO fund — SOUND +`seq <= cap` iff the entry was *pushed* before the create request (seqids are +assigned monotonically under the mutex at the push sites cc:859/885/895/1147). +A destroy pushed before the create request had its precondition fixed before +`e_created(i)` existed, so it cannot depend on it — ready or not, it is +cycle-safe funding. The one wrinkle: a destroy *requested* early but pushed +late (DELAYEDDESTROY, push at create-trigger cc:1146-1147) gets `seq > cap` +and is excluded — an error in the conservative direction only. Redistrict / +reuse pushbacks (cc:1037-1041) are v2 scope; the same push-time argument will +apply but must be re-checked when redistricts enter the model. + +### 1.6 RPR sweep rel handling + poisoned ack suppression — SOUND +When the queue empties in RPR, `rel'` keeps the pre-sweep `cur` (legacy +cc:1557 value) while `cur'` is swept — `rel` is invalid when the queue is +empty and the next first admission overwrites it (`rel := cur`, cc:787 +analog), so the unswept value is dead state; C++ need not touch +`release_allocator` in that branch. When the queue stays nonempty, `rel' = +cur + surviving ready frees` (strict, cc:1713-1714 pattern) — exactly +h:399-405 restored; this is the BUG-4-standalone fix. The poisoned instance's +own ack stays suppressed (its entry is erased by the walk, never swept; +matches the cc:1789 guard); swept defNote acks fire exactly when their tags +leave `cur` — the h:427-436 contract. + +### 1.7 strict sweep frees — SOUND for v1, note for v2 +Ready survivors' tags are in `cur` by the deferred-dealloc invariant, so +strict is correct and doubles as a corruption detector (`missingFree`). +v2 caveat: duplicate releases from network delays could legally present two +ready entries for one instance; the second strict free would false-trip. +Keep strict in the v1 C++; revisit under the v2 duplicate-release model. +Separately: the C++ sweep must be redistrict-aware (BUG-6.md fix A) even +though the v1 model has no redistricts — see §2.4. + +### 1.8 append-free ≡ canonical-interleave for new releases — SOUND +A new release's seq (`seqCtr + 1`) exceeds every queued watermark (watermarks +are caps `<=` the seqCtr at their admission), so it sorts after every queued +alloc in canonical order — appending its free to a canonical `fut` is the +canonical result. Post-ARR-partial `fut := rp.tf` is canonical because ARR's +`Replay` uses the same gate (`seq <= lastSeq`) as `CanonReplay`, the same +base (`g.test = cur'`), and post-swap survivors are all non-ready; under +FIX_CAP `lastSeq == cap == watermark`, so ARR's boundaries and the canonical +watermark rule coincide *by construction*. The drain preserves canonicity by +the prefix property of deterministic replay. ARR full-success leaves `fut` +stale — harmless: the queue is empty, fut invalid-by-convention. + +### 1.9 ARR × FIX_CAP interaction — SOUND +ARR funds placements only from `release_allocator` = `current` + ready +(already-triggered) releases. Triggered releases are dependency sinks — they +can never be waiting on any `e_created` — so ARR needs no cap awareness at +all: it is precisely the safe opportunistic half of the funding story, kept +verbatim. After an ARR partial swap the remaining queue is a suffix, so the +non-decreasing-watermark (monotone-cap) invariant is preserved, and §1.8 +gives fut-canonicity for the next `ADAResCap` admission. + +### Also verified +- Request-triggered creates pass `cap = seqCtr`: the monotone guard can never + fire for them and the capped test equals the legacy behavior given + fut-canonicity — the dominant path is behavior-identical (and §2.2 notes + the O(1) C++ fast path this licenses). +- The sweep runs whenever the queue is empty (not only on the transition) — + idempotent, establishes `INV_NoReadyWhenNoPendingAllocs` inductively; all + four queue-emptying transitions are covered (two sweeps + two ARR + full-success paths that erase every ready entry by construction); ready + entries are only ever created on paths requiring a nonempty queue, closing + the induction. +- `reqCap` reset at TriggerCreate is model-state canonicalization only (the + C++ field simply dies with the `DeferredCreate` use). +- BUG-5 (RPR trailing-alloc replay omission) was untouched by FIX_CAP and + FIX_SWEEP; it is now covered by the third toggle FIX_RPR (§2.7), added to + the bundle after the Inversion matrix showed FIX_CAP composes badly with + unfixed BUG-5 (traces/Inversion-bug5-deadlock.txt). + +**No blocking issue found. FIX_CAP and FIX_SWEEP as landed are sound +transcriptions of the agreed v1 design; the only action item is §0's +documentation alignment.** + +## 2. C++ implementation blueprint (v1 = the verified pure cap + sweep) + +All line numbers refer to the current tree (mem_impl.cc as read in this +campaign). No code has been changed; this is the transcription target. + +### 2.1 New state +- `inst_impl.h`, `RegionInstanceImpl::DeferredCreate` (inst_impl.h:61-74): + add `unsigned seqid_cap;`, set via a new parameter on `defer()` (called at + mem_impl.cc:715). +- `mem_impl.h`, `LocalManagedMemory` (h:396-447): `cur_release_seqid` stays a + plain `unsigned`; take `allocator_mutex` briefly in the deferral path to + snapshot it (**recommended over an atomic**: preconditioned creates are not + hot, and the mutex version needs no memory-order argument. If an atomic is + preferred later, a relaxed load is safe because a stale/smaller read only + shrinks the funding set — conservative direction). +- `PendingAlloc` (h:413-419): unchanged; `last_release_seqid` now stores the + **cap** at admission instead of the admission-time watermark. Every + downstream consumer (unblock scan cc:1654-1662, ARR replay cc:1258-1277, + RPR inner loop cc:1579) keys off `last_release_seqid` and needs no change — + this identification (lastSeq := cap) is the heart of the fix. +- `PendingRelease`: **no new fields in v1** (the precondition-Event storage in + BUG-1.md's sketch belongs to the unverified union term — omit). + +### 2.2 allocate_storage_deferrable + attempt_deferrable_allocation +- cc:712-717 (deferral path): snapshot `seqid_cap = cur_release_seqid` under + the mutex; pass into `deferred_create.defer(...)`. +- `attempt_deferrable_allocation` (cc:749-807) gains `unsigned seqid_cap`; + callers: cc:734 passes `cur_release_seqid` (request == trigger), cc:1136 + passes the stored snapshot. +- Body, in order: + 1. Keep the cc:754-757 current-allocator fast path verbatim (pending_allocs + empty only). + 2. If `!pending_allocs.empty() && seqid_cap < + pending_allocs.back().last_release_seqid` -> `ALLOC_INSTANT_FAILURE` + (monotone-cap guard; back() suffices because caps are non-decreasing). + 3. Capped canonical test on a scratch allocator seeded from + `current_allocator`: walk `pending_releases` (list order == seq order) + and `pending_allocs` interleaved by watermark; free survivors with + `seq <= min(next alloc's last_release_seqid, seqid_cap)` using + missing_ok=true (cc:778 form); place each queued alloc + (**assert placement succeeds** — §1.3); after the last queued alloc, + free remaining survivors with `seq <= seqid_cap`; finally test the new + instance. Failure -> `ALLOC_INSTANT_FAILURE`, and **do not write + `future_allocator`** (§1.4). + 4. On success: push `PendingAlloc(..., last_release_seqid = seqid_cap)`; + rebuild `future_allocator` canonically (same walk, bound = max seq, + new alloc placed at its cap watermark); `release_allocator = + current_allocator` when this is the first pending alloc (cc:787). +- Optimization licensed by §1.8 (optional): when `seqid_cap == + cur_release_seqid` and the queue is nonempty, the capped test is provably + equal to the legacy O(1) `future_allocator.allocate()` because fut is + maintained canonically — keep the O(1) path for the dominant + request-triggered case. The uniform slow path is what the model verifies; + add the fast path only with the equivalence comment. +- Shared helper suggestion: one `rebuild_future_canonical()` used by step 3/4 + subsumes the cc:768-779 replay; unification with the cc:1439-1447 and + cc:1706-1717 rebuilds is a follow-on cleanup, not required. +- The cc:772 `assert(!it->is_ready)` site disappears with the replaced + rebuild; carry its intent as `assert(pending_allocs.empty() => no ready + entries)` at ADA entry — valid again because of the sweep (BUG-6 fix). + +### 2.3 Asserts to carry (all TLC-backed) +- Canonical-replay queued-alloc placement succeeds (§1.3; `capAssert`). +- Sweep and rel-re-apply frees find their tags (strict; `missingFree`). +- The DEBUG cross-check cc:1674-1691 is *expected to hold* under the fix + (INV_FutureOffsetConsistency) — keep it enabled; it now guards the + fut-canonicity invariant the whole design leans on. + +### 2.4 sweep_ready_releases() — three sites + the rel re-apply +Signature sketch: walks `pending_releases` in list order while +`pending_allocs.empty()`; for each `is_ready` entry: apply to +`current_allocator` (strict), collect its deferred_dealloc_notify instance, +erase. Notifies fire after the mutex is released (existing +`deferred_dealloc_notifies` vector pattern). +1. `release_storage_immediate`, oldest path: after the cc:1702 prefix erase + and the cc:1751-1753 tail ARR, whenever `pending_allocs.empty()`. +2. `remove_pending_release`: after the cc:1562-1595 walk, whenever the walk + left `pending_allocs` empty. Additionally (BUG-4-standalone), when the + queue stays NONEMPTY: after `release_allocator = current_allocator` + (cc:1556-1557), re-apply surviving ready entries to `release_allocator` + (cc:1713-1714 pattern, strict). +3. `reuse_storage_immediate` mirror (~cc:1433-1448): same sweep — but the + sweep body must be **redistrict-aware** here and at site 1 (a ready entry + with `redistrict_tags` takes the `split_range` flavor with child-offset + collection and EVENTUAL_SUCCESS/FAILURE notifies, cc:1372/1459 form). + **Confidence caveat:** the v1 model contains no redistricts; treat the + redistrict arm of the sweep as unverified until the v2 model covers + `reuse_storage_*` — implement it, but flag it in review. +ARR full-success needs no sweep (erases every ready entry, cc:1241-1250). + +### 2.5 Behavior changes to document in the commit +- Allocations that previously deferred against untriggered deletions + requested after the create can now return `ALLOC_INSTANT_FAILURE` + (poisoning `e_created`, cascading per existing poison semantics). This is + deliberate: those DEFERRED answers were unsound promises that could + deadlock (BUG-1). Includes the GC-ripple fired-but-undelivered window. +- Dealloc notifications for previously-stranded ready releases now fire at + the sweep instead of at the next unrelated trigger — earlier, and now + bounded (BUG-6). +- The debug assert at cc:772 (previously reachable on legal input, BUG-6) is + restored as a true invariant in its new form. + +### 2.6 The eight design decisions as normative notes +Carry §1.1-1.8 verbatim into the patch discussion: (1) cur fast path only +when queue empty; (2) missing-ok in replays; (3) placement-assert in the +canonical replay; (4) never write fut on failed admission; (5) `seq <= cap` +membership is push-order, DELAYEDDESTROY self-excludes; (6) RPR: rel dead +when queue empties, rel re-apply when it doesn't; (7) strict sweep frees +(v2 duplicates caveat); (8) append-free of a fresh release preserves +fut-canonicity — preserve the seqid-assignment sites exactly. + +### 2.7 FIX_RPR — trailing alloc replay in remove_pending_release (BUG-5) +After the outer walk ends at cc:1595, run the cc:1579-1594 inner loop once +more with no seqid bound: place each remaining queued alloc onto the rebuilt +`future_allocator` in order (success stays queued, `last_release_seqid` +unchanged; failure gets `ALLOC_EVENTUAL_FAILURE` exactly like the in-walk +path, downstream poison cascades included). **Normative: the trailing pass +must CONTINUE from the walk's final `it2` cursor and must never restart from +`pending_allocs.begin()` — the SafetyFixed4 trace (slurm-77810, +bugs/DUPALLOC-TRIAGE.md) is a live demonstration that a restart-from-begin +variant re-runs `allocate()` on an already-placed tag, double-allocating it +(#442 class: second range inserted, `allocated[tag]` overwritten, old range +leaked).** Sequencing within remove_pending_release: walk → trailing replay +→ sweep / rel re-apply (§2.4 site 2), with the sweep condition evaluated on +the POST-trailing queue. + +## 3. Verdict summary + +| Item | Verdict | +|---|---| +| §0 spec-vs-BUG-1.md rule mismatch (pure cap modeled, union rule documented) | NEEDS-DOCUMENTATION (no spec change; C++ v1 = pure cap; union term must not ship unverified) | +| 1.1 cur fast path | SOUND | +| 1.2 missing-ok replay frees | SOUND | +| 1.3 replay placement assert | SOUND (TLC adjudicates; C++ carries assert) | +| 1.4 fut on failed admission | SOUND | +| 1.5 ready `seq <= cap` funding | SOUND (v2 redistrict re-check noted) | +| 1.6 RPR rel handling / acks | SOUND | +| 1.7 strict sweep frees | SOUND (v2 duplicate-release caveat) | +| 1.8 append ≡ canonical | SOUND | +| 1.9 ARR interaction | SOUND | + +No blocker for C++ work once the local fix-validation matrix is green. diff --git a/tla/allocation/bugs/BUG-1.md b/tla/allocation/bugs/BUG-1.md new file mode 100644 index 0000000000..7a2d8afbe8 --- /dev/null +++ b/tla/allocation/bugs/BUG-1.md @@ -0,0 +1,229 @@ +# BUG-1: Deferred creates are ordered at precondition-trigger time, not request time — event-loop deadlock + +**Status:** machine-confirmed by TLC (EventLoop config: deadlock in 7 states; Smoke config independently, 4,029 states). +**Traces:** `EventLoop.trace.txt` (primary), `traces/Smoke-run1.txt` (same shape at Smoke's constants). +**Fix review:** recommended fix revised per adversarial review — see `FIX-REVIEW.md` §1. +**Class:** liveness — silent permanent hang. No poison, no OOM abort, no error of any kind is reported. + +## Summary + +A deferred instance creation is inserted into the memory's release/alloc total order when its +**precondition triggers**, not when it was **requested**. `attempt_deferrable_allocation` associates +the new pending alloc with every pending release currently in the list (`last_release_seqid := +cur_release_seqid`, mem_impl.cc:784/801), and tests fit against a future heap that applies all of +them (mem_impl.cc:768-781). But Realm handed out the instance's ready event `e_created` at *request* +time, so a release *requested after the create but before its trigger* may legally carry a +precondition that depends on `e_created` of this very instance. Realm then plans the allocation out +of a release that can only happen after the allocation completes: a cycle through the event graph +that Realm can never resolve. This is the bug Sean suspected in the deferred-allocation talk +("an instance that is created with a precondition doesn't actually necessarily get put in the right +spot in the overall ordering ... can cause an event loop", transcript ~24:47-25:35). The +conservatism rule he describes — "we don't ever want to associate an allocation with a release that +came chronologically after it" (~34:20) — is enforced against the wrong clock: trigger order instead +of request order. + +## Concrete execution (from the TLC counterexample, EventLoop config: heap = 3 units, two instances of size 3) + +Client call order (all from one control-plane context, in program order): + +| # | Client call | Realm path | Heap state after | +|---|------------|-----------|------------------| +| 1 | create **I1** (size 3, no precondition) | `allocate_storage_deferrable` (cc:693) -> precondition already triggered (cc:703) -> `attempt_deferrable_allocation` (cc:734): `pending_allocs` empty, `current_allocator.allocate` succeeds at offset 0 (cc:754-757) -> **INSTANT_SUCCESS**; `e_created(I1)` fires clean (inst_impl.cc:1201-1202) | current = {I1@[0,3)} | +| 2 | create **I2** (size 3, precondition **P** = user event, untriggered) | cc:712-717: `inst_offset := INSTOFFSET_DELAYEDALLOC`, `deferred_create.defer`, return `ALLOC_DEFERRED`. **Note: the ready event `e_created(I2)` was already handed to the client by this call** (transcript ~15:56: "we immediately hand back an instance ID and an event"). Nothing is recorded in the allocator's ordering. | unchanged | +| 3 | destroy **I1** (precondition = merge(`e_created(I1)`, `e_created(I2)`)) | `release_storage_deferrable` (cc:810): precondition untriggered -> `pending_allocs` empty -> push `PendingRelease(I1, !ready, seqid=1)` (cc:857-859); `deferred_destroy.defer` (cc:920) | pending_releases = [R1(I1, seq 1)] | +| 4 | destroy **I2** (precondition = `e_created(I2)`) | cc:845-849: I2 is `INSTOFFSET_DELAYEDALLOC` -> marked `INSTOFFSET_DELAYEDDESTROY`, deferred. (Incidental to the cycle — this is just the client's eventual cleanup of I2.) | unchanged | +| 5 | client triggers **P** (ballistic: depends on nothing) | `DeferredCreate::event_triggered` (inst_impl.cc:49-56) -> `allocate_storage_immediate(I2)` (cc:1088): sees DELAYEDDESTROY (cc:1106), not poisoned -> `attempt_deferrable_allocation` (cc:1136): `current` full (cc:755 fails), releases exist (cc:762) -> **future := current − R1 = empty heap** (cc:768-779) -> `future.allocate(I2)` succeeds (cc:781) -> `pending_allocs = [PendingAlloc(I2, lastSeq = cur_release_seqid = 1)]` (cc:783-784), `release_allocator := current` (cc:787) -> **ALLOC_DEFERRED**. Then the queued destroy is pushed: `PendingRelease(I2, !ready, seqid=2)` (cc:1146-1147) and applied to future (cc:1150-1153). `e_created(I2)` stays unfired (inst_impl.cc:1126-1157). | pending_allocs = [A(I2, lastSeq=1)]; pending_releases = [R1(I1,1), R2(I2,2)] | + +**Terminal state (the deadlock):** + +- A(I2) is granted only when R1 drains (`release_storage_immediate` oldest-drain, cc:1631-1702, or a reorder — R1 is the only release that frees enough space). +- R1 drains only when merge(`e_created(I1)`, `e_created(I2)`) triggers. `e_created(I1)` is fired; **`e_created(I2)` is not**. +- `e_created(I2)` fires only when A(I2) is granted (`notify_allocation(ALLOC_EVENTUAL_SUCCESS)`, inst_impl.cc:1201-1202). + +Cycle: A(I2) -> R1 -> `e_created(I2)` -> A(I2). Every waiter is parked; nothing times out. Worse +than a mere hang with no answer: at DEFERRED admission the `InstanceAllocResult` profiling response +**is delivered with `success = true`** (inst_impl.cc:1140-1142) — a Legion mapper blocked on it is +affirmatively told the allocation will succeed, unblocks, builds downstream work on that promise, +and *then* the system hangs. The eventual completion (`ALLOC_EVENTUAL_SUCCESS` / the `e_created` +trigger) is what never comes. The Smoke trace is the identical construction at Smoke's constants +(HEAP_SIZE = 3, two instances of size 2). + +## Why the client is legal + +- **Topological sort / no back edges:** the calls occur in the order create(I1), create(I2), + destroy(I1), destroy(I2), trigger(P). Every event dependency points at an event handed out by an + *earlier* call (`e_created(I1)`, `e_created(I2)`); P is triggered unconditionally, depending on no + Realm result. No user event is ever triggered based on a deferred-allocation outcome. +- **Destroy-after-create:** both destroy preconditions include the respective instance's own + `e_created`, exactly the discipline Legion enforces today. +- **The idiom is canonical:** "free the old instance only after the new one exists (and the copy + from old to new has run)" — destroy(I1) gated on `e_created(I2)` is the standard copy/migration + chain. Combine it with a create precondition (Sean/Mike's own example: creates gated on garbage- + collection ripple, transcript ~24:03-24:39) and this is exactly the shape above. + +The talk's protection (~30:40-34:45) is that each pending alloc remembers "the newest pending +release when it *showed up*" so that an alloc is never satisfied by a release that might depend on +its post-condition. But "showed up" is implemented as ADA time = trigger time. Between request and +trigger, Realm has already published `e_created`, and any release requested in that window — R1 here +— may depend on it. ADA then happily counts R1's space. The rule "never associate an allocation with +a release that came chronologically after it" is sound only if "chronologically" means request +order; the code implements trigger order. + +## Impact + +- Silent, permanent, distributed hang: the memory's pending lists never drain, every event + downstream of the instance never triggers, and no error path fires (not the poisoned case, not an + OOM `abort()`, not `ALLOC_INSTANT_FAILURE`). There are no autonomous diagnostics for stuck + deferred allocations: the only related machinery is `deadlock_catch` (runtime_impl.cc:2119), + which dumps state only when an operator externally sends SIGTERM/SIGINT. +- The severity is compounded by the affirmative promise: the mapper receives + `InstanceAllocResult{success=true}` at DEFERRED admission (inst_impl.cc:1140-1142) before the + hang, so the client has already committed downstream work to an allocation that will never + materialize. +- Exposure requires a create with an untriggered precondition. Legion's current mapper path blocks + on `InstanceAllocResult` before exposing the instance, which narrows the single-context window, + but (a) Legion does issue preconditioned creates for GC ripple today, (b) destroy requests can + arrive from other contexts/nodes during the window, and (c) Realm's contract does not require + clients to serialize this way. Severity: **high** (hard hang, legal client, no diagnostics), with + moderate likelihood today and growing likelihood as preconditioned creates get more use. + +## The central trade-off (read this before the fix) + +Any fix keyed on **arrival order alone** faces an impossibility: construct two clients with +*identical* arrival sequences at the memory — create(I2, pre=P), destroy(I1, pre=Q), trigger(P). In +the **GC-ripple client**, Q is a last-use event of I1, independent of I2: the whole point of P +(transcript ~24:03-24:39) is that the funding destroy *requests* land in the create's +request->trigger window, and the create must then use them. In the **cycle client**, Q depends on +`e_created(I2)`. Realm sees the same inputs in the same order; the difference lives entirely in the +event graph Realm cannot inspect. Any deterministic arrival-order-only policy must answer both the +same way — so it either hangs the cycle client (today's behavior) or **false-OOMs the canonical +GC-ripple pattern, the primary motivating use of preconditioned creates**, in exactly the +near-capacity workloads deferred allocation exists to serve. A pure request-time cap (our first +draft) makes the second choice silently. The repaired rule below uses one extra bit of information +that *is* visible to Realm — whether a release's precondition has already triggered — to save the +GC pattern in its natural wiring; whether that covers Legion's actual wiring is an open question +for Mike (below). + +## Candidate fixes (no code changed yet) + +**(a) = (c), REPAIRED FORM — recommended: request-time seqid cap, with clean-triggered releases +always fundable.** Record `cur_release_seqid` at request time in the deferral path (cc:712-717, +e.g. on `DeferredCreate` or the instance) as `seqid_cap`. At trigger time, ADA tests the fit +against a **capped funding set**: + +> releases with `seqid <= seqid_cap` **UNION** releases whose own destroy precondition has +> **already triggered clean** by ADA time (checked via `has_triggered_faultaware` on the stored +> precondition event; poisoned-triggered excluded — those get cancelled/removed). + +replayed in list order on a scratch allocator, instead of testing the full `future_allocator` +(cc:768-779). Admission also requires the **monotone-cap rule** on the cap term: admit only if +`seqid_cap >=` the cap of every alloc already in `pending_allocs` (caps monotone => compare the +newest entry only). If either check fails -> `ALLOC_INSTANT_FAILURE`. On success the alloc is +placed in the full `future_allocator` exactly as today, so the drain determinism machinery +(cc:1668-1670, cc:1674-1691, ARR replay) is untouched; first-fit `can_allocate` is monotone in +free-set inclusion, so capped-yes implies full-yes — "never say yes then no" is preserved. + +*Soundness of the union term:* the alloc's `e_created` has not fired (it is only now being +attempted), so a release whose precondition has already triggered clean cannot be waiting on it — +counting it can never close a cycle through this alloc, regardless of request order. Excluding +poisoned-triggered adds no new failure mode: a counted release that fires poisoned already +EVENTUAL_FAILs its dependents today (cc:1587-1592). + +*Why no hang survives (induction sketch, repaired form):* clean-triggered funding releases are +dependency **sinks** — they wait on nothing. Every *untriggered* funding release R used by a queued +alloc has `seq(R) <= cap` of that alloc, i.e. R was requested before that alloc's request; under +the monotone-cap rule the caps are nondecreasing in queue order, so any alloc whose cap does *not* +admit R (requested before R) sits strictly earlier in the queue than every alloc whose cap does. +Hence every wait edge — alloc-to-untriggered-release, release-to-`e_created`, alloc-to-queue- +predecessor — points strictly earlier in (queue order, request order), and the wait graph is a +DAG. **This prevents hangs; it does not make every outcome clean:** a capped rejection poisons +`e_created(A)`, which can poison a dependent release, which can EVENTUAL_FAIL an already-admitted +alloc via `remove_pending_release` — a poison-mediated failure *cascade*, within existing poison +semantics, and strictly better than the hang today's code produces on the same input — but a +cascade nonetheless. **Caveat (validation matrix): the cascade drains cleanly only with the BUG-5 +fix also in place.** If a dependent alloc is *trailing* in `remove_pending_release`'s replay (its +`last_release_seqid` exceeds every surviving walked seqid), the pre-existing BUG-5 hole strands it +forever instead of failing it — the Inversion client deadlocked with FIX_CAP on and FIX_RPR off +(witness: `traces/Inversion-bug5-deadlock.txt`). FIX_RPR = `remove_pending_release` processes +trailing allocs after its walk (place-or-EVENTUAL_FAIL); see `bugs/BUG-5.md` for the full analysis. + +*GC-ripple coverage, honestly stated:* the repaired rule saves the GC pattern **iff** by the time +`e_pre` fires, the victims' destroy preconditions have themselves already triggered (the natural +wiring: `e_pre` downstream of, or merged with, the collection's completion conditions). If Legion's +`e_pre` only guarantees the destroy *requests* have landed while their preconditions are still +unfired, the pattern still instant-fails under this rule. + +> **OPEN QUESTION for Mike:** which wiring does Legion use for the GC-ripple `e_pre` — does it fire +> only after the victims' destroy preconditions have triggered (rule saves the pattern), or merely +> after the destroy *requests* have been issued (rule false-fails it)? + +*Fallbacks if the wiring is unfavorable:* (i) client-side retry — Legion already blocks on +`InstanceAllocResult`, so on a capped `ALLOC_INSTANT_FAILURE` it can re-issue the create after +collection completes; (ii) **ballistic-lite**: a flag on `release_storage_deferrable` by which the +client *declares* the destroy precondition independent of unfired allocator-output events. Flagged +window releases are always fundable; unflagged ones fall under the cap. This is the transcript's +ballistic direction (~35:34) as a contract declaration rather than an inference — consistent with +Realm's existing unverified-contract style (no-back-edges is already such a contract), backward +compatible (default = unflagged = capped), and Legion's GC destroys qualify trivially. + +*Implementation sketch:* (i) store `seqid_cap` at cc:712-717; (ii) **`PendingRelease` stores its +precondition `Event`** (captured at the push sites cc:859/885/895/1147) so ADA can evaluate the +union term — today only the instance's `deferred_destroy` holds it; (iii) ADA gains the capped +admission: on the `pending_allocs`-empty path a scratch rebuild mirroring cc:768-779 restricted to +the funding set; on the `pending_allocs`-**nonempty** path (cc:798) the test must be an +**interleaved scratch replay** — current + funding-set releases + already-queued allocs' placements +applied in seqid/queue order (the cc:1258-1277 shape) — not a bare release-only rebuild, otherwise +older queued allocs' space consumption is unaccounted; this is well-defined precisely because the +monotone rule makes older allocs' funding sets subsets of the newer alloc's. Cost: O(pending list) +once per *deferred-create trigger* only; triggered-precondition creates (the dominant case) are +untouched. Edge cases: a DELAYEDDESTROY self-release gets `seqid = cap + k > cap` and an unfired +precondition by construction, so an instance still never funds itself; ARR's erasures only remove +already-triggered releases, which are in the universally-fundable class, so no cap drift across +swaps. + +**(b) Full ballistic-event tracking** (talk, ~35:34): only count releases whose preconditions are +known "scheduled" by inspecting event provenance. Strictly stronger — but requires event-graph +visibility Realm does not have; ballistic-lite above captures its value as a contract bit. Long-term +direction, not a near-term fix. + +**(d) Cycle detection over the event graph at destroy-request time.** Rejected for the reasons the +talk itself gives (~33:38): Realm cannot see whether one event derives from another, and walking +the distributed graph is impractical. + +**Recommendation: (a)/(c) in the repaired form (cap ∪ clean-triggered), with the monotone-cap +admission rule, plus the ballistic-lite flag if Legion's GC wiring turns out unfavorable.** + +> **v1 scope as verified (see BLUEPRINT-REVIEW.md §0/§3).** The TLA+ fix that was actually landed +> and validated (`FIX_CAP`) implements the **pure request-time cap**: funding = surviving releases +> with `seqid <= seqid_cap` only. The "∪ clean-triggered releases" union term above is **not +> modeled and therefore not verified**. Per Mike's decision, spurious instant-failures are +> acceptable (correctness over compatibility), so: +> - **C++ v1 must implement the pure cap exactly as modeled** — no union term, and no +> precondition-`Event` storage on `PendingRelease` (a cost v1 thereby avoids). +> - The union term is demoted to an optional later optimization. **Do not implement it without +> first extending the spec's funding gate and re-running the full validation matrix.** +> - Practical consequence (accepted behavior change): the GC-ripple pattern succeeds when the +> funding destroys have been *applied* before the create's trigger, and honestly +> `ALLOC_INSTANT_FAILURE`s otherwise — instead of risking a hang. +> - **Shipping constraint: FIX_CAP must not land in C++ without FIX_RPR** (the BUG-5 fix) — the +> capped rejection cascade only drains cleanly with FIX_RPR in place; without it a trailing +> dependent alloc in `remove_pending_release` is stranded forever +> (witness: `traces/Inversion-bug5-deadlock.txt`; full analysis in `bugs/BUG-5.md`). + +## Verification plan (model v-next) + +1. Add `seqid_cap` to the spec's deferred-create state; change ADA to the **repaired** rule: capped + funding set = {seq <= cap} ∪ {releases whose precondition has fired clean}, scratch-replay + admission (interleaved form on the nonempty path), monotone-cap check. Keep everything else + identical. +2. Expected flips: EventLoop — deadlock disappears; I2's create resolves `ALLOC_INSTANT_FAILURE`, + both destroys fire poisoned, run drains to Quiescent (green). Smoke — returns to a clean pass. +3. New inversion client (the A/R*/B shape): expected to **instant-fail cleanly** (possibly as a + poison-mediated cascade) rather than hang; also run it against the naive pure cap to document + that the naive form is insufficient. +4. New GC-ripple client: model `e_pre` firing only after the victims' destroy preconditions have + resolved clean; expected to still **SUCCEED** under the repaired rule (and to false-fail under + the pure cap — documenting the regression the review caught). +5. Expected non-flips: SafetyMini's BUG-6 violation persists (independent mechanism); all currently + green invariants stay green (the funding set only restricts admissions; the union term only + adds releases that are sinks). diff --git a/tla/allocation/bugs/BUG-5.md b/tla/allocation/bugs/BUG-5.md new file mode 100644 index 0000000000..04ef4be994 --- /dev/null +++ b/tla/allocation/bugs/BUG-5.md @@ -0,0 +1,129 @@ +# BUG-5 — `remove_pending_release` never revisits trailing pending allocs + +**Status: TLC-confirmed in composition with FIX_CAP (Inversion witness, +`traces/Inversion-bug5-deadlock.txt`); latent-variant hunt in unfixed code +pending at scale (sapling Poison4). SHIPPING CONSTRAINT: FIX_CAP MUST NOT +LAND WITHOUT THE FIX BELOW (FIX_RPR).** + +## 1. Summary + +When a pending release is poison-cancelled, `remove_pending_release` +(mem_impl.cc:1538-1597) rewrites future history: it resets +`future_allocator := current_allocator` (cc:1556), walks the surviving +`pending_releases` replaying each onto the rebuilt future (cc:1562-1595), +and — interleaved with that walk — retries pending allocs whose +`last_release_seqid` is ≤ the seqid being walked (cc:1579-1594), keeping +the ones that fit and EVENTUAL_FAILing the ones that don't (cc:1587-1592). + +The walk ends when the release list is exhausted. Any pending alloc whose +`last_release_seqid` **exceeds every walked seqid** is never reached by the +inner loop: it stays in `pending_allocs`, but it is **absent from the +rebuilt `future_allocator`** and is **never failed**. Such *trailing* +allocs are possible because `last_release_seqid` is a counter snapshot +(cc:784/801), not a reference to a surviving entry — the entry that raised +the counter may itself have been erased by an earlier poison cancellation. + +Two distinct consequences: + +- **Planning soundness (unfixed code, latent):** later admissions test + against a future state missing the trailing alloc's reservation and can + plan overlapping space — surfacing downstream as the cc:1668 + `assert(ok)` failing, the cc:1674-1691 offset cross-check failing, or + overlapping placements. +- **Drain liveness (composition with the BUG-1 capped-admission fix):** a + trailing alloc whose only funding release was poison-erased is never + failed, so nothing downstream of its `e_created` can ever run — + permanent hang. Under FIX_CAP this shape is **common, not exotic**: + capped rejections poison `e_created` as the designed failure path + (ii:1121-1122), C2 makes dependent destroy preconditions fire poisoned + as a matter of course, and every such poisoned release runs + `remove_pending_release`. + +## 2. The TLC witness (Inversion, FIX_CAP+FIX_SWEEP, no user poison) + +H=3; I1 size 3, A=I2 size 1, B=I3 size 2. Client is contract-clean (C1 +topological order, C2 destroy-after-create). Model trace mapped to the C++: + +| # | Call / event | Code path | State after | +|---|---|---|---| +| 1 | create(I1), no pre | cc:755 INSTANT_SUCCESS | I1 @ [0,3); heap full | +| 2 | create(A), pre = user event BA | cc:712-717 deferral; **cap(A) := seqid 0** (the fix's request-time snapshot) | A CREATE_PENDING | +| 3 | destroy(I1)=R1, pre deps {eCr(1), eCr(A)} | untriggered → push (cc:857-860), waiter (cc:918-921) | pending_releases=[R1 seq 1] | +| 4 | destroy(A), pre ⊇ eCr(A) | A still pending → DELAYEDDESTROY (cc:845-849) | marker only | +| 5 | BA fires → attempt A | capped test: funding ∅ (cap 0), current full → **INSTANT_FAILURE**; eCr(A) POISONED (ii:1121-1122); A's destroy entry pushed **seq 2** (cc:1146-1147) | list=[R1(1), RA(2)] | +| 6 | create(B), pre = user event BB | deferral; **cap(B) := seqid 2** | B CREATE_PENDING | +| 7 | destroy(A)'s pre fires POISONED | `remove_pending_release` fast path (pending_allocs empty, cc:1547-1553) erases RA | list=[R1(1)] | +| 8 | BB fires → attempt B | cap 2 ≥ seq(R1)=1 → funded by R1 → **ALLOC_DEFERRED, lastSeq = 2** | pending_allocs=[B(lastSeq 2)] | +| 9 | R1's pre fires POISONED (eCr(A) poisoned) | RPR rebuild path: fut/rel := cur (cc:1556-1557); walk erases R1 (saved seqid **1**, cc:1564-1570); inner loop bound: allocs with lastSeq ≤ 1 — **B (lastSeq 2) is trailing, never processed** | B still queued; fut = full heap, no B | +| 10 | destroy(B) requested, pre ⊇ eCr(B) | untriggered (eCr(B) UNFIRED) → push seq 3 | list=[RB(3)] | +| 11 | — | B unblocks only via a completing release (cc:1631-1753 drain / cc:1207 ARR / RPR); the only release waits on eCr(B), which waits on B | **permanent hang** (TLC deadlock, depth 14, 469/254 states) | + +The guard itself is not at fault: the same config run invariant-only +(`-deadlock`) exhausts the full space with INV_InversionCapped and +SAFETY_PromisesKept holding everywhere. The deadlock is purely the +trailing-alloc omission. Every trigger ordering of this client ends in the +same stranding — the config cannot drain until BUG-5 is fixed. + +## 3. Reachability in current, unfixed code + +The same shape exists without FIX_CAP because `last_release_seqid` is the +counter value at admission, which can already exceed every *surviving* +seqid at admission time: + +1. destroy(X) queued untriggered (seq 1); create(Y) with pre, destroy(Y) + queued (DELAYEDDESTROY); Y's create INSTANT_FAILs at trigger → + eCr(Y) poisoned intrinsically, Y's entry (seq 2) pushed, then erased by + its poisoned destroy (fast path) — no user poison needed. +2. create(Z) too big for current → cc:768 rebuild admits Z **with + lastSeq = cur_release_seqid = 2** funded by R1 (cc:781-784). +3. destroy(X)'s precondition fires poisoned → RPR rebuild: walked seqids = + {1} < 2 → **Z is trailing**: not failed (though its only funding is + gone → hang if no further releases), absent from the rebuilt future → + a later release + admission can overspend Z's space + (INV_NoOverlap / INV_FutureOffsetConsistency / + INV_InOrderUnblockSucceeds — reviewer A's original shape). + +**Honesty note:** we have a TLC witness only for the FIX_CAP composition +(§2). The unfixed variant above is a paper construction; the sapling +Poison4 run (toggles off, full battery) hunts it at scale, and a targeted +scripted config is easy to add if we want a local witness. The difference +in urgency stands regardless: in unfixed code poisoned releases are the +exception; under FIX_CAP they are the designed failure path. + +## 4. Fix (modeled as the FIX_RPR toggle) + +After the outer walk of cc:1562-1595 completes, run the cc:1579-1594 inner +loop **once more with no seqid bound** over the remaining `pending_allocs` +in order: + +- `future_allocator.allocate(...)` succeeds → keep the alloc (its + reservation is now refunded into the rebuilt future); +- fails → EVENTUAL_FAILURE + erase + poison `e_created` — the standard + cascade, identical to the existing cc:1587-1592 arm (the `assert(found)` + is trivially satisfied there since the erased target was already seen). + +The cascade terminates: each poisoned-destroy RPR erases at least one +release entry. **Sequencing with FIX_SWEEP:** if the trailing pass empties +`pending_allocs`, the RPR-tail stranded-ready sweep (FIX_SWEEP site 3) +must run on the **post-trailing-pass** state — otherwise surviving ready +entries strand exactly as in BUG-6. + +## 5. Shipping constraint + +**FIX_CAP must not land in C++ without FIX_RPR** (and FIX_SWEEP, already +established). The capped fix converts BUG-1's silent hang into honest +failures *only if* the poison cascade actually drains; BUG-5's hole makes +the cascade strand any trailing dependent alloc, reintroducing a permanent +silent hang on a legal client — the exact defect class FIX_CAP exists to +eliminate. The three changes are one bundle. + +## 6. Verification status + +| Run | Toggles | Status / result | +|---|---|---| +| Inversion, deadlock ON | CAP+SWEEP (no RPR) | **FAIL = the witness** — deadlock depth 14, `traces/Inversion-bug5-deadlock.txt` | +| Inversion, `-deadlock` | CAP+SWEEP (no RPR) | PASS full space — isolates the deadlock to the stranding (guard sound) | +| Inversion, deadlock ON | CAP+SWEEP+**RPR** | **PENDING** (spec fork landing FIX_RPR) — must flip GREEN: B EVENTUAL_FAILs, cascade drains | +| Local fixed matrix re-run | full bundle | PENDING — all nine configs must stay green | +| Poison4 (sapling) | none | PENDING — hunts the unfixed overspend variant (§3) | +| SafetyFixed4 / Poison4Fixed / BigFixed (sapling) | CAP+SWEEP | BUG-5 detectors expected-possible until FIX_RPR joins; re-run with the full bundle flips expectation to green | diff --git a/tla/allocation/bugs/BUG-6.md b/tla/allocation/bugs/BUG-6.md new file mode 100644 index 0000000000..70b9792ef3 --- /dev/null +++ b/tla/allocation/bugs/BUG-6.md @@ -0,0 +1,175 @@ +# BUG-6: `assert(!it->is_ready)` at mem_impl.cc:772 is reachable by a legal client (stranded ready release) + +**Status:** machine-confirmed by TLC (SafetyMini.cfg, 3 instances, heap = 3 units, sizes 2/2/1; +invariant `INV_NoReadyWhenNoPendingAllocs` violated at depth 9; 5.5M states, 10 s). +Trace: `traces/SafetyMini.trace.txt`. Independently constructed on paper by two Phase-1 reviewers before TLC confirmed it. + +## Summary + +A `PendingRelease` entry that was pushed *already-ready* (the `attempt_release_reordering` +failure path, mem_impl.cc:884-887) can be **stranded behind a non-ready entry** while the +oldest-release drain empties `pending_allocs`. Both cleanup sites that would normally retire +ready entries are skipped in that final drain iteration: + +- the `release_allocator` rebuild at cc:1706-1717 requires + `!successful_allocs.empty() && !pending_allocs.empty()` — the drain just emptied `pending_allocs`; +- the tail `attempt_release_reordering` at cc:1751-1753 requires `!pending_allocs.empty()` — same. + +The system is left with `pending_allocs` empty and `pending_releases = [R_nonready, R_ready]`. +(A poison-path variant reaches the same stranded state through `remove_pending_release`, whose +inner replay loop can erase the last pending alloc while ready entries survive the walk, +cc:1587-1592 — BUG-6 variant (b), reachable at 3 instances with one poisoned event. Any fix must +cover that transition too; see fix (A).) +The *next* allocation request that misses `current_allocator` and needs the future rebuild walks +`pending_releases` at cc:768-779 and fires `assert(!it->is_ready)` at cc:772. **Debug builds abort +on a fully legal workload.** Release-build consequences are analyzed in "Impact" below — the +rebuild itself survives, but the stranded entry breaks the documented `release_allocator` +invariant (mem_impl.h:399-405) and composes with the BUG-4 mechanism into a **TLC-confirmed +permanent-leak path** (`Composite4.cfg`) that does **not** require poison. + +## Concrete C++ execution (from the TLC trace) + +Memory: 3 units. Instances: i1 (size 2), i2 (size 2), i3 (size 1). Every call below is +contract-clean (see "Legality"). `R#(inst, ready?, seq)` denotes a `PendingRelease`. + +| # | Client call | Code path | State after | +|---|---|---|---| +| 1 | `create(i1)`, no precondition | `attempt_deferrable_allocation` cc:754-757: fits current | cur = {i1:[0,2)} | +| 2 | `create(i3)`, no precondition | same, cc:754-757 | cur = {i1:[0,2), i3:[2,3)} — heap full | +| 3 | `destroy(i1)`, precondition pending (user event) | cc:857-860: push, no future yet | releases = [R1(i1,¬rdy,1)] | +| 4 | `create(i2)`, no precondition | cc:762-788: misses current; future := current − i1; i2 fits at [0,2) → **ALLOC_DEFERRED**, `lastSeq = 1`; `release_allocator := current` (cc:787) | allocs = [A(i2, lastSeq 1)]; fut = {i2:[0,2), i3:[2,3)}; rel = cur | +| 5 | `destroy(i2)`, precondition = eCreated(i2) (unfired) | cc:892-895: future-free i2 (missing_ok), push | releases = [R1, R2(i2,¬rdy,2)]; fut = {i3:[2,3)} | +| 6 | `destroy(i3)`, precondition already triggered | cc:871-872: free i3 from rel and fut; ARR gate cc:1211-1215: front alloc i2 (size 2) cannot fit rel's 1-unit hole → **reordering fails**; cc:884-887: push **ready**, `deferred_dealloc_notify = true`; i3's tag stays in `current` | releases = [R1, R2, **R3(i3, READY, 3)**]; rel = {i1:[0,2)}; fut = {} | +| 7 | i1's destroy precondition triggers → `release_storage_immediate(i1)` | oldest path cc:1634-1702: rel-free i1 (cc:1636); drain R1 into current (cc:1641); unblock scan: A(i2).lastSeq = 1 < R2.seq = 2 → no break (cc:1653-1658); `current.allocate(i2)` at [0,2) — `assert(ok)` holds (cc:1668-1670); future cross-check takes the lookup-miss branch, finds R2 ¬ready (cc:1674-1691) — passes; **`pending_allocs` empties**; do-while stops at R2 (¬ready, cc:1700); prefix erase leaves **[R2(¬rdy), R3(READY)]**; rebuild skipped (`pending_allocs` empty, cc:1706); tail ARR skipped (cc:1751) | cur = {i2:[0,2), i3:[2,3)}; allocs = []; **stranded state** — `INV_NoReadyWhenNoPendingAllocs` violated | +| 8 | any `create(i4)` (size ≥ 1) | current is full → cc:762 falls into the rebuild loop cc:768-779; iteration reaches R3 | **`assert(!it->is_ready)` fires at cc:772** | + +Step 8 needs a fourth allocation request, which SafetyMini's 3-instance client cannot issue — +the invariant at step 7 is the precursor state and is exactly what the assert guards against. + +Note also: R3's `deferred_dealloc_notify` means i3's `notify_deallocation()` (profiling ack + +instance-slot recycle) is delayed until R3 drains — which now cannot happen until R2's +precondition (eCreated-i2-derived) triggers. Bounded under normal progress, but the latency is +inherited by whatever the stranding delays. + +## Legality of the client + +- Every destroy is requested after its instance's create request (topологically sorted; no back edges). +- `destroy(i2)` while i2 is still ALLOC_DEFERRED is legal: the *request* arrives early, but its + precondition includes eCreated(i2), so it cannot *trigger* before creation (contract C2). This is + the standard "run ahead" pattern the deferred allocator exists to support. +- `destroy(i3)` with an already-triggered precondition after i3's successful creation is plain usage. +- No poison, no failures, no user-event tricks anywhere in the trace. + +## Impact + +**(a) Debug builds (DEBUG_REALM / any build with asserts): hard abort.** Any workload that +reaches the stranded state and then requests one more allocation that needs deferral kills the +process at cc:772. The trace is short and un-exotic; debug CI and developer runs are exposed. + +**(b) Release builds — the rebuild itself is functionally correct.** The stranded entry's tag is +still in `current_allocator` (that is precisely why its ack was deferred, mem_impl.h:427-436), +so the `missing_ok=true` replay at cc:778 actually *succeeds* in freeing it from the rebuilt +future state; the future picture and any resulting admission decisions are sound. + +**(c) Release builds — the documented `release_allocator` invariant is broken, with two +consequences:** + +1. *Lost reordering (conservatism).* On the next deferred admission, cc:787 sets + `release_allocator = current_allocator` **without re-applying the stranded ready entry** + (nothing after cc:787 does either; the only sites that re-apply ready entries, + cc:1438-1447/1707-1717, require a later oldest-drain with both queues nonempty — site + enumeration independently re-verified by the adversarial fix review, FIX-REVIEW.md §3.4). + `release_allocator` therefore + under-reports free space versus its definition "current + ready releases" (mem_impl.h:399-405), + so `attempt_release_reordering`'s gate can falsely fail and allocations wait longer than needed. + +2. *Confirmed composite with BUG-4 — permanent leak without poison.* Continue past step 8 in a + release build: `create(i4)` is admitted DEFERRED (future rebuild is correct), and cc:787 resets + `release_allocator := current` — **with the stranded i3 tag still allocated in it**. Now let any + later instance's destroy arrive with a triggered precondition: cc:871-872 free it from + rel/fut and call ARR. If ARR reaches **full success** (cc:1236-1252): `current := test` + (derived from the stale rel, i.e. *still containing i3's tag*), and the ready-entry sweep at + cc:1241-1250 **erases R3 and fires i3's deferred dealloc notify**. Result: i3's range is + permanently allocated in `current_allocator` with no `pending_releases` entry referencing it, + while the instance slot has been recycled — if the recycled `RegionInstance` ID ever re-enters + this allocator, `allocated[tag] = idx` (mem_impl.inl:500) silently double-tracks. This is the + same corruption class as issue #442, reached **without any poisoned event** (BUG-4 proper + needs poison; this composite does not). **TLC-confirmed:** `Composite4.cfg` (5 instances, + H=4, sizes 2,2,1,1,1, `SCRIPTED_COMPOSITE` client mode) violates `INV_CurrentMatchesGround` + with `INV_NoOrphanTags` violated in the same state — 12-step trace at `traces/Composite4.txt`, + 93,795 states generated / 51,904 distinct, ~5 s, **zero poison**. Final state: tag 3 still in + `current_allocator` with `instState[3] = DESTROYED` and `notifyCount[3] = 1` (dealloc notify + already fired), no `pendingReleases` entry referencing it, and `readyAtRebuild = TRUE` — + proving the path went through the BUG-6 stranding. Four instances are provably insufficient + (the stranding consumes two instances, the rel-resurrection needs a fresh deferred create, and + the ARR invocation needs a fresh request-time-triggered destroy), hence 5. The config is named + after BUG-4, whose mechanism it completes. + +**(d)** The cc:772 assert is load-bearing documentation: the rebuild, the cc:787 reset, and the +cc:871-872 strict frees all *assume* no ready entry exists outside a pending-allocs regime. The +assumption is false; every consumer of it should be re-audited once a fix direction is chosen. + +## Candidate fixes (no code changed yet) + +**(A) Recommended: a shared helper — `sweep_stranded_ready_releases()` — invoked at every +`pending_allocs` → empty transition.** There are three such transitions (the fourth emptying +site, ARR full-success cc:1236-1252, already erases all ready entries by construction and needs +nothing): + +1. `release_storage_immediate`, oldest-drain tail (after the prefix erase, ~cc:1702); +2. `reuse_storage_immediate`, the mirrored oldest-drain tail (~cc:1433-1448); +3. `remove_pending_release`, after its replay loop (~cc:1596) — the inner loop can erase the + last pending alloc while ready entries survive the walk (cc:1587-1592, variant (b)); a sweep + placed only at the drain tails misses this poison-path stranding entirely. + +The helper: if `pending_allocs.empty()`, walk the remaining `pending_releases` **in list order**; +for every `is_ready` entry, apply it to `current_allocator`, collect its +`deferred_dealloc_notify` ack, and erase it. The body must be **redistrict-aware**: a stranded +ready entry can carry `redistrict_tags` (the reuse path pushes ready-with-defNote at +cc:1037-1041), and such an entry requires the `split_range` flavor with child-offset collection +and `ALLOC_EVENTUAL_SUCCESS/FAILURE` notifications for the new instances (the +`it->release(current_allocator, offsets)` form used at cc:1372/1459) — not just +deallocate-and-ack. + +*Correctness:* with `pending_allocs` empty there are no admitted futures to invalidate and +`future_allocator` is invalid-by-convention (rebuilt from `current` on next use, so it inherits +the sweep automatically). Plain frees are tag-keyed and commute — the final free-set is +order-independent; order only matters against interleaved *allocations*, and there are none. +Redistrict entries are also order-safe because `split_range` carves children inside the old +instance's own range (mem_impl.inl:200-262), independent of other swept frees — and sweeping in +list order makes the outcome identical to the eventual in-order drain the entries would have +received. *Notify timing is safe by construction* (confirmed by the adversarial fix review): the +sweep fires each `notify_deallocation` at the moment the tag leaves `current_allocator`, which is +exactly what the `deferred_dealloc_notify` contract demands (mem_impl.h:427-436 — the sweep *is* +a drain of the entry); the #442 guard condition (tag out of current before slot recycle) holds. +The sweep bounds i3-style ack latency, removes the stranded-entry precondition of the composite +leak in (c)(2) at the root, and makes the cc:772 assert a true invariant again, unchanged. + +**(B) Safety-equivalent alternative: tolerate ready entries.** Weaken cc:772 to +`assert(!it->is_ready || tag-still-in-current)` (the rebuild replay already handles ready entries +via `missing_ok`), and re-apply all ready entries to `release_allocator` after **both** resets: +cc:787 (`attempt_deferrable_allocation`) **and** cc:1557 (`remove_pending_release` — the BUG-4 +site, same stale-rel shape; cc:1309 needs nothing since ARR's partial path erases ready entries +before assigning). The fix review confirmed this **also closes the composite leak**: ARR then +builds `test` from a rel that already excludes the stranded tag, so `current := test` drops the +tag exactly when the entry is erased and its notify fired. The A-vs-B choice is therefore +**latency and hygiene, not safety**: (A) restores the documented invariant, bounds dealloc-ack +latency, and keeps "no ready entries outside a pending-allocs regime" true for every future +reader of `pending_releases`; (B) is the smaller diff but leaves stranded entries live longer +(ack latency remains, and all readers must stay ready-aware). + +**(C) Recommendation:** (A) as primary, with (B)'s strengthened assert kept as belt-and-suspenders +documentation; (B) is an acceptable fallback if the sweep is judged too invasive, provided its +re-apply lands at *both* reset sites. + +## Verification plan + +- Model fix (A) — all three sweep sites — in a spec branch: `INV_NoReadyWhenNoPendingAllocs` + flips from expected-FAIL to expected-HOLD; SafetyMini goes green; Poison4's expected variant-(b) + violation (the `remove_pending_release` stranding) also flips green, which specifically + validates sweep site 3; Smoke's acceptable-deadlock (BUG-1, unrelated) and EventLoop unchanged. +- `Composite4.cfg` is now an expected-FAIL config (the confirmed (c)(2) leak); the three-site + sweep fix (A) — or (B) with both re-apply sites — must flip it green. `INV_NoOrphanTags` in + no-poison Big / sapling Safety remains the open-hunt backstop. +- The `reuse_storage_immediate` mirror (sweep site 2, redistrict-aware body) gets model coverage + when v2 adds redistricting. diff --git a/tla/allocation/bugs/BUG-7.md b/tla/allocation/bugs/BUG-7.md new file mode 100644 index 0000000000..c3037cd833 --- /dev/null +++ b/tla/allocation/bugs/BUG-7.md @@ -0,0 +1,141 @@ +# BUG-7 — `reuse_storage_immediate` drains prefix followers with the offsets-flavor release + +**Status: TLC-unverified (the v1 model excludes redistricting) — confirmed +by code review with a hand-constructed trace, adversarially re-verified +twice. Pre-existing on `main` (mem_impl.cc:1372), independent of the +CAP/SWEEP/RPR fix bundle; unchanged on `mbauer-deferred-alloc-fixes`. +Found during the C++ fidelity review of the fix branch (2026-08-30). +One manifestation is an out-of-bounds write — memory-safety severity.** + +## STATUS UPDATE (2026-08-31): fixed, test-first, detection confirmed + +Fixed on branch **`mbauer-bug7-reuse-drain`** (worktree +`/Users/mebauer/realm-bug7`, changes left **uncommitted** for review). The +fix is the §5 direction verbatim: the oldest-drain do-while applies the +offsets-flavor release only when `it->inst == old_inst` and the void flavor +to every follower (~10 lines in `reuse_storage_immediate`). + +Two regression tests added to the `DeferredAllocBadPathTest` suite +(`tests/unit_tests/deferred_alloc_test.cc`), one per drainable follower +kind, both constructed exactly in the §2 shape: + +- `ReuseOldestDrainSweepsPlainFollower` — on unmodified `main` (Debug): + `Assertion failed: (!redistrict_tags.empty()), function release, file + mem_impl.cc, line 1855.` (manifestation 1, exactly as predicted). +- `ReuseOldestDrainSweepsRedistrictFollower` — on unmodified `main` + (Debug): `Assertion failed: (offsets.size() == redistrict_tags.size()), + function release, file mem_impl.cc, line 1856.` (the debug face of + manifestation 3, the OOB write). + +With the fix applied both pass, alongside the four pre-existing +`DeferredAllocBadPathTest` units; the tests also assert the release-build +obligations (children keep their promised offsets in `current_allocator`, +parents' slots recycle exactly once, no stranded tags). The fix does not +textually or semantically conflict with `mbauer-deferred-alloc-fixes` +(disjoint hunks in `reuse_storage_immediate`; the bundle's sweep and this +drain fix are complementary). v2 model confirmation remains pre-registered +per §6. + +## 1. Summary + +When a deferred redistrict's precondition fires, `reuse_storage_immediate` +takes the oldest-entry drain path and catches up `current_allocator` with a +do-while over the ready prefix (main mem_impl.cc:1365-1431). Every iteration +applies the **offsets-flavor** `PendingRelease::release(allocator, offsets)` +(main:1372) — but only the *first* entry is guaranteed to be `old_inst`'s +redistrict entry. The do-while continues through any **ready followers** +(main:1431), and those can be: + +- a **plain destroy** (empty `redistrict_tags`), or +- a **different redistrict** with a different child count. + +The offsets flavor asserts `!redistrict_tags.empty()` (main:1855) and +`offsets.size() == redistrict_tags.size()` (main:1856), and it overwrites +the caller's `allocated`/`offsets` — which the function tail then uses to +notify `old_inst`'s children (main:1527-1535). The sibling drain in +`release_storage_immediate` uses the void flavor (main:1641) and is immune; +only the reuse drain has the defect. + +## 2. Concrete reachable scenario (legal client) + +Heap of 6 units. All calls contract-clean: destroys after creates, forward +edges only, two independent user events `eA`, `eP` triggered in order. + +| # | Call | Effect | +|---|------|--------| +| 1 | create P (size 4), create A (size 2) | both INSTANT_SUCCESS; heap full: P@[0,4), A@[4,6) | +| 2 | redistrict P → child C1 (size 1), precondition `eP` (untriggered) | `pending_releases = [R_P(redistrict, !ready, seq 1)]` (cc:1049-1051) | +| 3 | create D (size 3), no precondition | current full; future = current + split(P→C1) has hole [1,4) → **ALLOC_DEFERRED**, `last_release_seqid = 1` (cc:781-788); `pending_allocs = [D]` | +| 4 | destroy A, precondition `eA` (untriggered) | `pending_releases = [R_P, R_A(plain, !ready, seq 2)]` (cc:894-895) | +| 5 | trigger `eA` → `release_storage_immediate(A)` | non-oldest path (R_P is front): R_A **marked ready**, applied to `release_allocator` only, `deferred_dealloc_notify = true` (main:1724-1740); tail ARR gate fails (D=3 > the 2-unit hole in release) → R_A stays ready in the list | +| 6 | trigger `eP` → `reuse_storage_immediate(P)` | oldest path. Iteration 1: offsets-flavor on R_P — correct; C1 placed, `allocated=1`, `offsets` filled; unblock scan places D@[1,4) and empties `pending_allocs`. `++it` → R_A is ready → do-while continues (main:1431). Iteration 2: `allocated = it->release(current_allocator, offsets)` on the **plain** R_A → **BUG** | + +## 3. Manifestations and severity + +1. **Debug builds: hard abort on legal input.** Iteration 2 fires + `assert(!redistrict_tags.empty())` (main:1855). With a redistrict + follower of different child count, `assert(offsets.size() == + redistrict_tags.size())` (main:1856) fires instead. Real exposure for + any DEBUG_REALM CI running redistricts concurrently with plain deferred + destroys. +2. **Release builds, plain follower: child mis-notification + leak + (#442 class).** The asserts compile out; `split_range` with zero new + tags deallocates A correctly but returns 0, overwriting `allocated = 0`. + The tail (main:1527-1535) then notifies **every child of P** + `ALLOC_EVENTUAL_FAILURE` — while their tags were already placed in + `current_allocator` by iteration 1. The children are failure-notified + yet their ranges stay allocated forever: a permanent leak plus + dead-slot/live-tag divergence, the instance-ID-reuse (#442) + double-tracking class. (In debug, the consistency assert at + main:1528-1529 fires first.) +3. **Release builds, redistrict follower with more children than + `old_inst`: out-of-bounds write.** The follower's `split_range` writes + `allocs_first[i]` for every child i (mem_impl.inl:210) into the caller's + `offsets` vector sized for `old_inst`'s children — the inl:180 size + assert is compiled out — a heap-buffer overflow. **This is + memory-safety severity**, not just a bookkeeping error. Additionally + `allocated`/`offsets` then describe the follower's children, so + `old_inst`'s children are notified with another instance's counts and + offsets. + +## 4. Why it is pre-existing, and the fix branch's effect + +The defect is on `main` and does not involve the CAP/SWEEP/RPR bundle: it +needs only [redistrict entry at the front of `pending_releases`] + +[ready follower behind it], reachable since the reuse paths landed. The +branch's new `sweep_ready_releases()` **reduces** exposure — stragglers +that previously waited for a later drain are now retired with the void +flavor (which handles both entry kinds) whenever `pending_allocs` drains — +but the do-while window itself is untouched: a follower that is ready at +the moment the redistrict's precondition fires is still drained through +main:1372's offsets flavor. + +## 5. Proposed fix direction (no code changes yet) + +Small and contained, mirroring `release_storage_immediate`'s drain: inside +the do-while, apply the **void flavor** `it->release(current_allocator)` to +every entry, and capture `allocated`/`offsets` via the offsets flavor +**only when `it->inst == old_inst`** (the first iteration by construction). +Followers' own obligations are already handled elsewhere: a ready plain +follower's dealloc ack flows through `deferred_dealloc_notify` +(main:1374-1376), and a ready redistrict follower's children were notified +at its mark-ready time with offsets that are intrinsic to the parent's +interval (see the sweep's offset-match argument, branch mem_impl.cc:1734). + +## 6. Verification plan + +This is redistrict territory — the v1 model deliberately excludes it +(DESIGN.md §1), which is exactly why TLC never saw it. It becomes the **v2 +model's first pre-registered expected-FAIL**, the role BUG-6 played for v1: +model the reuse paths and `split_range` (already first on the v2 roadmap, +FUTURE-VERIFICATION.md §4), give the offsets flavor its size/emptiness +preconditions as ghost flags, and add a child-notify consistency invariant +(children notified success ⇔ tag live in current — an `INV_NoOrphanTags` +sibling). A 3-instance scripted config in the shape of §2 should violate it +in seconds; the §5 fix should flip it green. + +## 7. Provenance + +Found while adjudicating the fix branch's sweep design (choice-1 review of +child-notification timing); constructed and re-verified twice against +`main` (`git show main:src/realm/mem_impl.cc`), lines cited throughout. diff --git a/tla/allocation/bugs/DUPALLOC-TRIAGE.md b/tla/allocation/bugs/DUPALLOC-TRIAGE.md new file mode 100644 index 0000000000..bd11def5d4 --- /dev/null +++ b/tla/allocation/bugs/DUPALLOC-TRIAGE.md @@ -0,0 +1,131 @@ +# Triage: INV_NoDupAlloc violation in SafetyFixed4 (sapling job 77810) + +**Verdict: (a) SPEC ARTIFACT — a wiring bug in the FIX_RPR toggle's call +site, not a fix-design flaw and not a Realm bug.** The C++ blueprint is +unaffected in substance, but gains one normative sentence (below). The C++ +gate re-opens after a two-line spec correction and re-verification. + +Run: SafetyFixed4 (bundle CAP+SWEEP+RPR, 4 inst, H=4, sizes 2,1,1,2, +USER_POISON off, `-deadlock`), violated after 12.8B generated / 4.02B +distinct, trace depth 10 (graph depth 12), 14h10m. +Source: `slurm-77810-SafetyFixed4.out:900-1289`. + +## Trace reconstruction (10 states) + +| # | Action | Effect | +|---|--------|--------| +| 1 | Init | all empty | +| 2 | RequestCreate(**I2**, sz 1, ballistic pre) | CREATE_PENDING, `reqCap[2] = 0` | +| 3 | RequestCreate(**I1**, sz 2, triggered) | placed [0,2) | +| 4 | RequestCreate(**I4**, sz 2, triggered) | placed [2,4) — **heap full** | +| 5 | RequestDestroy(**I1**, untriggered) | `R1 = [inst 1, ¬ready, seq 1]` | +| 6 | RequestCreate(**I3**, sz 1, triggered) | cur full → capped path, cap = seqCtr = 1, funding {R1} → **DEFERRED**, lastSeq 1; canonical `fut = {3@[0,1), 4@[2,4)}`; `rel := cur` | +| 7 | RequestDestroy(**I2**) while CREATE_PENDING | DELAYEDDESTROY (`CREATE_PENDING_DESTROY`), preD = {eCreated(2)} (C2) | +| 8 | FireBallisticC(2) | I2's create precondition fires | +| 9 | TriggerCreate(**I2**) | FIX_CAP monotone guard: cap 0 < queue tail lastSeq 1 → **INSTANT_FAILURE**; eCreated(2) POISONED; dd-push `R2 = [inst 2, ¬ready, seq 2]` (cc:1146-1147, not applied to fut) | +| 10 | TriggerDestroy(**I2**) fires POISONED → `TriggerDestroyPoisoned(2)` | RPR rebuild — **dupAlloc = TRUE** (see below); `fut` bit-identical before/after | + +State-10 internals (`remove_pending_release` model, queue = [R1, R2], +allocs = [I3 lastSeq 1]): rebuild `fut := cur = {1,4}`; walk R1 (survivor) +→ free I1 → inner loop places I3 @ [0,1) (kept); walk R2 (target) → erased. +Walk result: `L.fut = {3@0, 4@2}`, `L.paOut = <>`, `L.dup = FALSE`. + +## Pinned DoAlloc site + +`DeferredAlloc.tla:1018`: `tr == IF FIX_RPR THEN TrailingRPR(L.fut, L.paOut)`. + +`RPRLoop` (line 533) returns `paOut = inner.kept \o r.paOut` — the **full +surviving queue**, i.e. walk-KEPT allocs concatenated with the true trailing +(never-examined) remainder (base case, line 522: `paOut |-> paA`). The +FIX_RPR call site feeds that whole list to `TrailingRPR` (lines 550-562), +which re-runs `CanAlloc`/`DoAlloc` on the kept prefix. In this trace the +kept I3 is already in `L.fut`, `CanAlloc` succeeds on the residual gap +[1,2), and `DoAlloc(fut, 3, 1)` sees `HasTag(fut, 3) = TRUE` → +`dup = TRUE`. The left-biased `@@` in `DoAlloc` keeps the old placement, so +`fut` is unchanged — exactly the observed state delta (only `dupAlloc`, +`pendingReleases`, `destroyWaiter` change). + +Reconstruction re-verified twice against the trace and the operator text. + +## Why (a) and not (b)/(c) + +- **Not a fix-design flaw (b):** the agreed design and the C++ blueprint + specify "after the outer walk ends at cc:1595, run the cc:1579-1594 inner + loop once more with no seqid bound." In C++ the alloc cursor `it2` has + already advanced **past** every kept alloc during the walk; the trailing + pass continues from that cursor and can only see the never-examined + remainder. The re-examination of kept allocs exists only in the spec's + functional reconstruction, which lost the cursor position by reusing + `paOut` (full survivors) instead of the trailing remainder. +- **Not a residual code bug (c):** current C++ has no trailing pass at all + (that omission IS BUG-5), and in the base model `paOut` is only ever used + to rebuild `pendingAllocs'` — it is never re-fed to `DoAlloc`. Consistent + with every toggles-off run being dup-clean. + +Model-only severity note: the witnessed flavor is benign (ghost flag only). +A second, worse flavor is reachable in-model: if `CanAlloc` FAILS for a +kept alloc (its own placement consumed the last fitting gap), `TrailingRPR` +spuriously EVENTUAL_FAILs an alloc the walk had successfully re-placed, +erasing it from the queue while its placement stays in `fut` — bogus poison +cascades and a stale fut tag. Both flavors vanish with the correction. + +## Proposed spec correction (two edits) + +1. `RPRLoop`: expose the trailing remainder separately — base case gains + `trail |-> paA`, recursive case `trail |-> r.trail` (all other fields + unchanged; `paOut` keeps its full-survivors meaning for the + FIX_RPR = FALSE path). +2. Call site (`TriggerDestroyPoisoned`): + `tr == IF FIX_RPR THEN TrailingRPR(L.fut, L.trail) ELSE ...` and + `paF == IF FIX_RPR THEN KeptPrefix \o tr.kept ELSE L.paOut`, with + `KeptPrefix == SubSeq(L.paOut, 1, Len(L.paOut) - Len(L.trail))` + (exact, since `paOut = kept \o trail` by construction). + +## Normative addition to the C++ blueprint (BLUEPRINT-REVIEW.md) + +The trailing pass must **continue from the walk's final `it2` cursor** — +never restart from `pending_allocs.begin()`. This trace is the live +demonstration of the mis-scoped variant: in C++ a restart-from-begin would +re-run `allocate()` on an already-placed tag, inserting a second range and +overwriting `allocated[tag]` — a real #442-class leak/corruption, not a +ghost flag. + +## Why the local matrix missed it + +The shape needs **4 instances** (two triggered fillers to fill the heap, a +third create that defers with a kept-through-RPR placement, and a fourth +whose DELAYEDDESTROY entry poisons — SafetyMiniFixed's 3-instance space +provably cannot build it), AND `INV_NoDupAlloc` is absent from every local +bundle battery (only Safety/SafetyMini/Poison4/Big and the sapling Fixed +configs check it — notably NOT Inversion, SafetyMiniFixed, SmokeFixed, +EventLoopFixed, GCRipple, LivenessFixed). Recommendation for re-validation: +add `INV_NoDupAlloc` to all bundle configs so the corrected `TrailingRPR` +is actually observed locally. + +## Sanity confirmation of the other two sapling violations (expected) + +- **Safety (77808)**, 9-state trace: final state `pendingAllocs = <<>>`, + `pendingReleases = [inst 4 ¬ready seq 2, inst 2 READY+defNote seq 3]`, no + poison anywhere — the known **BUG-6 variant (a)** stranded-ready shape at + 4-instance scale. As expected. +- **Poison4 (77809)**, 9-state trace: final state `pendingAllocs = <<>>`, + `pendingReleases = [inst 2 READY+defNote seq 2]`, `failedVia[4] = "RPR"`, + `eCreated[4] = POISONED` — the queue was emptied by an in-walk RPR + failure with a ready survivor: the known **BUG-6 variant (b)** poison + stranding, i.e. precisely the third sweep site's shape. As expected. +- Both runs halted on these first (expected) violations at 1.44B / 273M + distinct states, so the deep hunts (BUG-3; BUG-4-standalone/BUG-5-unfixed) + were preempted — the already-created SafetyHunt/PoisonHunt configs (out of + this triage's scope) are the vehicle for those re-runs. + +## Implications for the C++ gate + +The fix design itself took no damage: nothing in this violation touches the +capped-admission, sweep, or trailing-replay semantics as intended for C++. +Gate sequence: apply the two-line spec correction → SANY + local bundle +matrix re-run with `INV_NoDupAlloc` added to all bundle batteries (regression +toggles-off must stay exact) → resubmit SafetyFixed4 fresh on sapling (spec +changed; the checkpoint is not reusable). Big/BigFixed submissions are +unaffected by the correction only if resubmitted after it lands (BigFixed +checks `INV_NoDupAlloc` and runs FIX_RPR — a stale-spec run could hit the +same artifact). diff --git a/tla/allocation/bugs/FIX-REVIEW.md b/tla/allocation/bugs/FIX-REVIEW.md new file mode 100644 index 0000000000..a2b1ccd6b0 --- /dev/null +++ b/tla/allocation/bugs/FIX-REVIEW.md @@ -0,0 +1,188 @@ +# FIX-REVIEW: adversarial review of the recommended fixes in BUG-1.md and BUG-6.md + +Scope: the bug **existence** claims are TLC-confirmed and are not in question here. This review +attacks the **recommended fixes** before they are proposed to the Realm maintainers. All reasoning +is on paper against the code (no TLC runs; a concurrent fork owns the model). + +| Fix | Verdict | +|---|---| +| BUG-1 (a/c): request-time seqid cap + monotone-cap admission | **NEEDS-REFINEMENT** — as stated it breaks the canonical GC-ripple pattern; repaired rule below | +| BUG-6 (A): sweep stranded ready entries when the drain empties `pending_allocs` | **NEEDS-REFINEMENT** — mechanism sound, but coverage misses a third stranding site and the sweep body must be redistrict-aware | +| BUG-6 (B): tolerate-ready + re-apply at reset | SOUND as an alternative, **also closes the composite leak** (so A-vs-B is latency/complexity, not safety); B's re-apply must also cover cc:1557 | + +--- + +## 1. BUG-1 fix: the request-time cap instant-fails the GC-ripple pattern (CRITICAL) + +**The regression.** The pure request-time cap excludes every release requested in the +request→trigger window. But the *canonical legal* use of a create precondition — Sean/Mike's own +GC-ripple example (transcript ~24:03-24:39: "I don't want Realm to try to do the instance creation +until I know that the effects of the garbage collection have rippled and **all the destroy calls +have actually been done**") — puts the funding destroy *requests* in exactly that window. That is +the *point* of `e_pre` there: the create request may arrive at the memory before the victims' +destroy requests do (multi-node ripple, network reordering), and `e_pre` orders the create's +*consideration* after they land. Under the cap as written in BUG-1.md, ADA at trigger sees the +funding releases with `seqid > seqid_cap`, the capped test fails, and the create returns +`ALLOC_INSTANT_FAILURE` — a **false out-of-memory on the primary motivating pattern for +preconditioned creates**, in workloads near capacity, which are exactly the workloads deferred +allocation exists to serve. Legion (which blocks on `InstanceAllocResult`) would report OOM to the +mapper where today it correctly reports success. + +**Why no arrival-order-only rule can do better.** Construct two clients with *identical* arrival +sequences at the memory: create(I2, pre=P) → destroy(I1, pre=Q) → trigger(P). In client 1 +(GC-ripple), Q is a last-use event of I1, independent of I2. In client 2 (the BUG-1 cycle), Q +depends on `e_created(I2)`. Realm sees the same inputs in the same order; the difference lives +entirely in the event graph Realm cannot inspect. Any deterministic policy keyed on arrival order +alone must give the same answer to both — so it either hangs client 2 (today's behavior) or fails +client 1 (the proposed cap). **The fix as stated chooses to fail client 1 and BUG-1.md does not +say so.** This must be surfaced to the maintainers as the central trade-off, not discovered by +them. + +**Repaired rule (strongest implementable form found).** At trigger time, the capped funding set is + +> releases with `seqid <= seqid_cap` **UNION** releases whose own destroy precondition **has +> already triggered clean** (checked via `has_triggered_faultaware` on the stored precondition +> event at ADA time; poisoned-triggered excluded — those will be cancelled). + +Soundness of the union term: the alloc's `e_created` has not fired (the alloc is only now being +attempted), so a release whose precondition has *already triggered* cannot be waiting on it — +counting it can never close a cycle through this alloc, regardless of request order. It is also +exactly as strong as existing semantics on the poison side: today a counted release that later +fires poisoned already EVENTUAL_FAILs its dependents (cc:1591), so excluding poisoned-triggered +and counting clean-triggered adds no new "yes then no" mode. + +Re-attack of the repaired rule: +- **Queue-inversion induction still holds.** With the monotone-cap rule applied to the *cap* term + only: caps are nondecreasing in queue order, so any *untriggered* funding release R with + `seq(R) <= cap(A_j)` for some queued alloc was requested before every queued alloc whose cap + admits it — `cap(A_m) < seq(R)` for exactly the allocs requested before R, and those sit + *strictly earlier* in the monotone queue. Hence every dependence edge from an untriggered + funding release points strictly earlier in queue order. Clean-triggered releases are dependency + *sinks* (they wait on nothing). All edges point strictly earlier ⇒ the wait graph is a DAG ⇒ no + hang. The union term does not participate in and does not weaken the monotone comparison. +- **GC coverage, honestly stated:** the repaired rule saves the GC pattern **iff** by the time + `e_pre` fires the victims' destroy preconditions have themselves triggered (the natural wiring: + `e_pre` downstream of, or merged with, the collection's completion conditions). If Legion's + `e_pre` only guarantees the destroy *requests* landed while their preconditions are still + unfired, the pattern still instant-fails. **Open question for Mike: which wiring does Legion + use?** If the weak wiring exists, the client-side mitigation is a retry after collection + completes (Legion already blocks on `InstanceAllocResult`, so it has the hook), or the API + extension below. +- **Ballistic-lite as the complete answer (recommend mentioning):** a flag on + `release_storage_deferrable` by which the client *declares* the precondition independent of + unfired allocator-output events ("ballistic"). Realm counts flagged window releases without any + event-graph visibility; unflagged ones fall under the cap. This is the transcript's ballistic + direction (~35:34) implemented as a contract declaration rather than an inference, consistent + with Realm's existing unverified-contract style (no-back-edges is already such a contract). + Legion's GC destroys qualify trivially. Backward compatible (default = unflagged = capped). + +**Soundness of the capped gate itself (question 1): CONFIRMED.** The capped test is an +*additional* admission guard; on success the alloc is still placed in the full `future_allocator` +as today, so every downstream determinism argument (drain `assert(ok)` cc:1668-1670, future +cross-check cc:1674-1691, ARR replay) is untouched. `can_allocate` is monotone in free-set +inclusion for first-fit, so capped-yes ⇒ full-yes. Caps are static per-alloc; ARR's erasure of +ready releases only ever removes *already-triggered* releases, which are in the universally-safe +class, so no "effective cap" drift arises across swaps. No "yes then no" scenario found. + +**Failure cascades — reword the report's claim.** "Monotone-cap kills all cycles of this family" +overstates: it prevents *hangs*. A monotone-rule rejection poisons `e_created(A)`, which can +poison a dependent release, which can EVENTUAL_FAIL an already-admitted alloc B via +`remove_pending_release` (a chain today's code would have *hung* on instead). Within existing +poison semantics, and strictly better than a hang, but it is a failure *cascade*, not a clean +single failure — say so. + +**Implementability (question 3): workable, with two deltas beyond BUG-1.md's sketch.** +(i) `PendingRelease` must store its precondition `Event` (captured at the push sites cc:859/885/ +895/1147) to support the union term's `has_triggered_faultaware` check — today only the +instance's `deferred_destroy` holds it. (ii) The capped test on the `pending_allocs`-nonempty +path (cc:798) must be an *interleaved scratch replay*: current + capped-set releases + already- +queued allocs, applied in seqid/queue order (the cc:1258-1277 shape), not a bare +release-only rebuild — otherwise older queued allocs' space consumption is unaccounted. This is +well-defined precisely *because* the monotone rule guarantees older allocs' funding sets are +subsets of the newer alloc's capped set. Cost: O(pending list) once per *deferred-create trigger* +only; triggered-precondition creates (the dominant case) are untouched. + +--- + +## 2. BUG-6 fix A: sound mechanism, incomplete coverage + +**(4) Sweep safety: CONFIRMED, with a redistrict proviso.** Tag-keyed frees commute (final +free-set is order-independent; order only matters against interleaved *allocations*, and +`pending_allocs` is empty by hypothesis). Redistrict entries are also safe to sweep **because +`split_range` places children inside the old instance's own range** (mem_impl.inl:200-262), so +their placement is independent of other swept frees — but the sweep body must then replicate the +*full* drain side-effects for a redistrict entry: collect child offsets and fire +`ALLOC_EVENTUAL_SUCCESS/FAILURE` for the children (the `it->release(current_allocator, offsets)` +form, cc:1372/1459), not just `deallocate` + ack. BUG-6.md's one-line fix description covers only +the plain-destroy shape. A stranded ready entry *can* be a redistrict (the reuse path pushes +ready-with-defNote at cc:1037-1041), so this is required, not optional. + +**Coverage gap (the real finding): a third stranding site is missed.** `pending_allocs` can also +transition to empty inside `remove_pending_release` — the inner replay loop erases allocs that no +longer fit (cc:1587-1592) and can drain the queue while *ready* entries survive in the walked +list (this is exactly BUG-6 variant (b) / Phase-1 reviewer-B finding 5, reachable at 3 instances +with one poison). Fix A as located in BUG-6.md (tails of the two oldest-drain paths) does not run +there, so the poison-path stranding — and its identical downstream consequences, including the +cc:772 abort and the composite-leak precondition — survives the fix. **Refinement: make the sweep +a shared helper (`sweep_stranded_ready_releases()`), invoked at every `pending_allocs` → +empty transition:** (1) `release_storage_immediate` oldest-drain tail (~cc:1702), (2) +`reuse_storage_immediate` oldest-drain tail (~cc:1433), (3) `remove_pending_release` tail +(~cc:1596). The fourth emptying site, ARR full-success (cc:1236-1252), already erases all ready +entries by construction and needs nothing. + +**(5) Notify timing: CONFIRMED SAFE.** The sweep fires `notify_deallocation` at the moment the +tag leaves `current_allocator`, which is precisely the condition the `deferred_dealloc_notify` +contract demands (mem_impl.h:427-436: delay "until this entry is drained from pending_releases" — +the sweep *is* a drain). Earlier firing than status quo only shortens the ack latency the report +already flags; the #442 double-tracking guard is the tag-out-of-current condition, which holds. + +**(6) Fix B closes the composite too: CONFIRMED.** With ready entries re-applied after the +`release_allocator := current` reset, an ARR full-success builds `test` from a rel that already +excludes the stranded tag; `current := test` drops the tag exactly when the entry is erased and +its notify fired (cc:1239-1250) — no leak. BUG-6.md already states this; the framing "A removes +the root cause, B keeps stranded entries alive" is fair. One addition: **B's re-apply must also +be added after the reset at cc:1557** (`remove_pending_release`), not only cc:787 — cc:1557 is +the BUG-4 site and has the same stale-rel shape. (cc:1309 needs nothing: ARR's partial path +erases all ready entries before assigning `release := current`.) With that, B is a complete +alternative; the A-vs-B choice is ack latency + "no ready entries at quiescence" hygiene vs. +smaller diff, **not** safety. + +--- + +## 3. Overclaims found (maintainer-falsifiable statements) + +1. **BUG-1.md ("Terminal state" paragraph): "InstanceAllocResult at trigger time — which never + comes."** False as written: the `InstanceAllocResult` *does* come at DEFERRED admission with + `success = true` (inst_impl.cc:1140-1142) — that is what unblocks Legion's mapper wait. What + never comes is the eventual completion (`ALLOC_EVENTUAL_SUCCESS` / the `e_created` trigger). + Reword; as written it also understates the bug (the mapper is affirmatively told "will + succeed" and then hung, which is worse than "never told"). +2. **BUG-1.md ("Concrete execution" tail): "the Smoke trace is the identical construction at heap + size 2."** The Smoke config is HEAP_SIZE = 3 with two size-2 instances (Smoke.cfg:13-16, + SizesSmoke). Same shape, wrong constants. +3. **BUG-1.md "no error of any kind is reported": stands, minor softening available.** Verified: + no autonomous watchdog exists for stuck deferred allocations; the only related machinery is + `deadlock_catch` (runtime_impl.cc:2119-2120), a SIGTERM/SIGINT handler that produces + diagnostics only when an operator kills the process. Optionally say "no autonomous + diagnostics; state is only dumped on external SIGTERM/SIGINT." +4. **BUG-6.md "nothing after cc:787 re-applies the stranded ready entry": VERIFIED CORRECT** by + site enumeration (cc:787/1309/1557 assign without re-apply; cc:1438-1447/1707-1717 re-apply + but require both queues nonempty; cc:871/1024/1367/1468/1636/1738 touch only the incoming + entry). Not an overclaim — noting it here so the maintainers know it was independently + re-checked. +5. **BUG-6.md composite ("#442 class"): stands and is if anything understated** — the + `deferred_dealloc_notify` machinery (mem_impl.h:427-436) *is* the #442 guard, and the + composite fires the notify while the tag is live, i.e. it bypasses that guard by construction; + the "if the recycled ID re-enters" step is the documented #442 failure mode, not speculation. + +## 4. Bottom line + +- **BUG-1:** keep the monotone-cap architecture, but propose it in the **repaired form** + (cap ∪ clean-triggered releases), present the GC-ripple trade-off explicitly, ask Mike which + `e_pre` wiring Legion uses, and offer the ballistic-lite declaration flag as the complete + long-term answer. Reword "kills all cycles" → "prevents hangs; rejections can cascade as + poison-mediated failures." +- **BUG-6:** adopt fix A **as a shared helper at all three `pending_allocs`→empty transitions**, + with a redistrict-aware sweep body; keep B's strengthened assert as documentation; note B is a + safety-equivalent fallback if the sweep is judged too invasive, provided its re-apply also + lands at cc:1557. diff --git a/tla/allocation/run.sh b/tla/allocation/run.sh new file mode 100755 index 0000000000..b5e1d7471a --- /dev/null +++ b/tla/allocation/run.sh @@ -0,0 +1,131 @@ +#!/bin/sh +# --------------------------------------------------------------------------- +# Run TLC on the Realm deferred-instance-allocation specification. +# +# ./run.sh run the default local sweep (increasing cost) +# ./run.sh Safety run one configuration by name +# ./run.sh sany parse-check DeferredAlloc.tla / MCDeferredAlloc.tla +# +# Module is always MCDeferredAlloc; each .cfg selects constants, +# invariants, and client shape (see DESIGN.md section 7). +# +# Configurations and expected outcomes (DESIGN.md sections 7-8): +# +# Smoke H=3, 2 insts all INV_*, deadlock check expect PASS +# (BUG-1-shaped deadlock traces possible) +# EventLoop worked example, 2 insts deadlock check expect FAIL (BUG-1) +# Safety H=4, 4 insts (2,1,1,2) all INV_*/SAFETY_*, dlk expect FAIL: +# INV_NoReadyWhenNoPendingAllocs (BUG-6a) +# Liveness H=4, 3 insts, WF LIVE_NoStuckAllocs expect FAIL (BUG-1) +# +# Poison4 4 insts + USER_POISON hunts BUG-4esc/5/6b sapling-targeted +# Big H=5-6, 4-5 insts open hunt sapling-targeted +# +# Poison4 and Big are excluded from the default sweep (projected > 1h locally); +# run them by name here at your own risk, or submit sapling_tlc.sbatch. +# +# Deadlock semantics: deadlock-check configs rely on TLC's built-in check plus +# the spec's Done self-loop (DESIGN.md section 6), so they must NOT pass +# -deadlock. Temporal-liveness configs (Liveness) MUST pass -deadlock to +# suppress the check (clean termination would otherwise be reported). +# +# Requires tla2tools.jar (default: the copy in ../barrier/tools). +# Environment overrides: JAVA, JAR, WORKERS, HEAP, JTMP. +# +# TLC unpacks the TLA+ standard modules into java.io.tmpdir; this script +# points it at $JTMP (default ./jtmp). Under a sandbox, set JTMP to a +# writable scratch directory. +# --------------------------------------------------------------------------- + +HERE=$(cd "$(dirname "$0")" && pwd) + +# prefer homebrew openjdk (the system /usr/bin/java stub has no runtime) +if [ -z "$JAVA" ]; then + if [ -x /opt/homebrew/opt/openjdk/bin/java ]; then + JAVA=/opt/homebrew/opt/openjdk/bin/java + else + JAVA=java + fi +fi +JAR=${JAR:-$HERE/../barrier/tools/tla2tools.jar} +WORKERS=${WORKERS:-8} +HEAP=${HEAP:-4g} +JTMP=${JTMP:-$HERE/jtmp} + +mkdir -p "$JTMP" "$HERE/states" + +# Per-config extra TLC flags. Deadlock checking stays ON only for +# Smoke/EventLoop (they own the BUG-1 deadlock class); every other config +# passes -deadlock so short deadlock traces don't preempt the deeper +# invariant/temporal hunts (EXPECTED.md). NOTE: temporal configs +# (Liveness, LivenessNoCross) also require an UNSANDBOXED JVM - TLC's +# liveness checker binds a local RMI socket at startup. +extra_flags_for() { + case $1 in + Smoke|EventLoop) echo "" ;; + # Fixed-model configs whose whole point is "the deadlock class is + # gone" keep deadlock checking ON. With the full bundle + # (FIX_CAP+FIX_SWEEP+FIX_RPR) Inversion is GREEN deadlock-ON; the + # historical two-toggle deadlock is kept as the BUG-5 witness in + # traces/Inversion-bug5-deadlock.txt (see the cfg header). + SmokeFixed|EventLoopFixed|EventLoopCapOnly|GCRipple|Inversion) echo "" ;; + *) echo "-deadlock" ;; + esac +} + +sany_check() { + cd "$HERE" || exit 1 # SANY resolves EXTENDS relative to the cwd + for m in DeferredAlloc MCDeferredAlloc; do + echo "=== SANY $m.tla" + "$JAVA" -Djava.io.tmpdir="$JTMP" -cp "$JAR" tla2sany.SANY "$m.tla" \ + || exit 1 + done +} + +run_cfg() { + cfg=$1 + if [ ! -f "$HERE/$cfg.cfg" ]; then + echo "error: no such config: $HERE/$cfg.cfg" >&2 + exit 1 + fi + case $cfg in + Safety|Poison4|Big|SafetyFixed4|Poison4Fixed|BigFixed) + echo "note: $cfg is sapling-targeted (see sapling_tlc.sbatch); running locally anyway." ;; + esac + echo "===========================================================" + echo "=== $cfg (module MCDeferredAlloc)" + echo "===========================================================" + rm -rf "$HERE/states/$cfg" + # shellcheck disable=SC2046 + "$JAVA" -XX:+UseParallelGC -Xmx"$HEAP" \ + -Djava.io.tmpdir="$JTMP" \ + -cp "$JAR" tlc2.TLC \ + -config "$HERE/$cfg.cfg" \ + -workers "$WORKERS" \ + -metadir "$HERE/states/$cfg" \ + $(extra_flags_for "$cfg") \ + "$HERE/MCDeferredAlloc.tla" + echo +} + +if [ $# -ge 1 ]; then + if [ "$1" = "sany" ]; then + sany_check + exit 0 + fi + for c in "$@"; do run_cfg "$c"; done +else + echo "note: Safety, Poison4, Big and their Fixed variants (SafetyFixed4," + echo " Poison4Fixed, BigFixed) are sapling-targeted and excluded from this" + echo " sweep (see SAPLING_JOBS.md); run them by name or via" + echo " sapling_tlc.sbatch. SafetyMini/Composite4 are the local" + echo " reproducers for BUG-6 and the BUG-6->BUG-4 composite; the" + echo " *Fixed/GCRipple/Inversion configs validate the CAP+SWEEP+RPR" + echo " fix bundle (all green, Inversion deadlock-ON included)." + # increasing cost order; base model first, then the fix-validation matrix + for c in Smoke EventLoop SafetyMini Composite4 Liveness LivenessNoCross \ + SmokeFixed EventLoopFixed EventLoopCapOnly GCRipple Inversion \ + Composite4Fixed SafetyMiniSweepOnly SafetyMiniFixed LivenessFixed; do + run_cfg "$c" + done +fi diff --git a/tla/allocation/sapling_tlc.sbatch b/tla/allocation/sapling_tlc.sbatch new file mode 100755 index 0000000000..3b579173ad --- /dev/null +++ b/tla/allocation/sapling_tlc.sbatch @@ -0,0 +1,233 @@ +#!/bin/bash +# --------------------------------------------------------------------------- +# Slurm batch script: run TLC on the Realm deferred-allocation spec on sapling. +# +# Usage (submit FROM the tla/allocation directory so SLURM_SUBMIT_DIR is right): +# +# cd /tla/allocation +# sbatch sapling_tlc.sbatch Poison4 # config name as $1, default Big +# sbatch sapling_tlc.sbatch Big +# sbatch -t 48:00:00 --mem=256G sapling_tlc.sbatch Big # override resources +# +# STORAGE LAYOUT: TLC's metadir (disk state queue + fingerprint set + +# checkpoints) lives on NODE-LOCAL /tmp (or $SLURM_TMPDIR when set), NOT on +# the shared filesystem. The state queue is a huge, constantly-churning +# write load; putting it on shared storage is slow and can fill the +# filesystem for everyone (this killed all five round-2 jobs simultaneously +# when /scratch2 pressure hit). To keep resumability, the newest COMPLETED +# checkpoint is periodically synced back to states// in the submit +# directory (see SYNC below). +# +# Resuming an interrupted run: point RECOVER_FROM at the synced checkpoint +# directory on shared storage (it must contain vars.chkpt); the script stages +# it onto node-local disk and passes TLC the local path: +# RECOVER_FROM=$PWD/states// \ +# sbatch sapling_tlc.sbatch +# (Raw TLC_EXTRA="-recover " still works but the path must then be +# reachable AND fast from the compute node — prefer RECOVER_FROM.) +# +# CHECKPOINT VALIDITY: recovery requires the BYTE-IDENTICAL .tla files that +# wrote the checkpoint. Any spec edit — even one that provably does not +# change the next-state relation for this config's constants — shifts the +# string-intern table TLC serializes states against, and recovery fails with +# "ValueInputStream: Can not unpickle a value of kind ". If the spec has +# changed since the checkpoint: delete states/ and start fresh. +# +# SYNC: OFF BY DEFAULT — hard rule: never put TLC data on /scratch on +# sapling. With SYNC_CHECKPOINT=1 (explicit opt-in only), a background loop +# every CHECKPOINT_MIN/2 minutes rsyncs the newest quiescent checkpoint to +# states// in the submit dir (atomic .partial rename, older +# checkpoints pruned) so an interrupted run can be resumed via RECOVER_FROM. +# With the default (off), an interrupted run restarts from zero — size the +# time limit so runs finish in one shot. +# +# ------------------------- SITE-SPECIFIC DEFAULTS --------------------------- +# CHECK THESE BEFORE FIRST SUBMISSION - they are guesses, not pinned to +# sapling's actual layout. All are overridable on the sbatch command line +# (partition/time/cpus/mem) or via environment (JAVA, JAR, ALLOC_DIR, HEAP). +#SBATCH --job-name=tlc-deferred-alloc +#SBATCH --partition=cpu +#SBATCH --time=24:00:00 +#SBATCH --cpus-per-task=40 +#SBATCH --mem=128G +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --output=slurm-%j.out +# --------------------------------------------------------------------------- + +set -u + +CONFIG=${1:-Big} + +# locate the spec: $ALLOC_DIR override, else the directory we submitted from +ALLOC_DIR=${ALLOC_DIR:-${SLURM_SUBMIT_DIR:-$(pwd)}} +if [ ! -f "$ALLOC_DIR/MCDeferredAlloc.tla" ]; then + echo "error: MCDeferredAlloc.tla not found in $ALLOC_DIR" >&2 + echo " submit from tla/allocation or set ALLOC_DIR=" >&2 + exit 1 +fi +if [ ! -f "$ALLOC_DIR/$CONFIG.cfg" ]; then + echo "error: no such config: $ALLOC_DIR/$CONFIG.cfg" >&2 + exit 1 +fi + +# redirect output to a per-config log next to the catch-all slurm-%j.out +exec > "$ALLOC_DIR/slurm-${SLURM_JOB_ID:-nojob}-${CONFIG}.out" 2>&1 + +# ------------------------------ java discovery ------------------------------ +# order: $JAVA env var, environment-modules java, PATH java +if [ -z "${JAVA:-}" ]; then + if command -v module >/dev/null 2>&1; then + module load java 2>/dev/null || module load openjdk 2>/dev/null || true + fi + JAVA=$(command -v java || true) +fi +if [ -z "${JAVA:-}" ] || ! "$JAVA" -version >/dev/null 2>&1; then + echo "error: no working java found; set JAVA= in the environment" >&2 + exit 1 +fi + +JAR=${JAR:-$ALLOC_DIR/../barrier/tools/tla2tools.jar} +if [ ! -f "$JAR" ]; then + echo "error: tla2tools.jar not found at $JAR (set JAR=)" >&2 + exit 1 +fi + +# ------------------------------ resources ----------------------------------- +WORKERS=${SLURM_CPUS_PER_TASK:-40} + +# heap ~80% of the job's memory; SLURM_MEM_PER_NODE is in MB when --mem is used +if [ -z "${HEAP:-}" ]; then + if [ -n "${SLURM_MEM_PER_NODE:-}" ]; then + HEAP=$(( SLURM_MEM_PER_NODE * 80 / 100 ))m + else + HEAP=100g + fi +fi + +CHECKPOINT_MIN=${CHECKPOINT_MIN:-60} +# HARD RULE (Mike, 2026-08-27): never put TLC data on /scratch on sapling. +# Checkpoint sync-back to the submit dir therefore defaults OFF; opt in +# per-run with SYNC_CHECKPOINT=1 only if the single-checkpoint write to +# shared storage is judged acceptable for that run. +SYNC_CHECKPOINT=${SYNC_CHECKPOINT:-0} +TLC_EXTRA=${TLC_EXTRA:-} +RECOVER_FROM=${RECOVER_FROM:-} + +# Deadlock checking is ON only for Smoke/EventLoop (they own the BUG-1 +# deadlock class); every other config passes -deadlock so short deadlock +# traces don't preempt the deeper invariant/temporal hunts (EXPECTED.md). +case $CONFIG in + Smoke|EventLoop) : ;; + SmokeFixed|EventLoopFixed|EventLoopCapOnly|GCRipple|Inversion) : ;; + *) TLC_EXTRA="$TLC_EXTRA -deadlock" ;; +esac + +# -gzip compresses the on-disk state queue (typically 5-10x): the queue is +# the dominant disk consumer at billion-state scale, and node-local disks +# are limited, so this is what makes big runs fit. Disable with TLC_GZIP=0. +# (Appended HERE, before the header echo, so the printed extra= is truthful.) +TLC_GZIP=${TLC_GZIP:-1} +[ "$TLC_GZIP" = "1" ] && TLC_EXTRA="$TLC_EXTRA -gzip" + +# --------------------------- node-local storage ------------------------------ +LOCAL_ROOT=${SLURM_TMPDIR:-/tmp} +LOCAL_DIR="$LOCAL_ROOT/${USER}-tlc-${SLURM_JOB_ID:-nojob}-$CONFIG" +LOCAL_STATES="$LOCAL_DIR/states" +LOCAL_JTMP="$LOCAL_DIR/jtmp" +SHARED_STATES="$ALLOC_DIR/states/$CONFIG" +mkdir -p "$LOCAL_STATES" "$LOCAL_JTMP" +# shared-storage dir only exists if checkpoint sync is explicitly opted in +[ "$SYNC_CHECKPOINT" = "1" ] && mkdir -p "$SHARED_STATES" + +echo "=== TLC on sapling: config=$CONFIG" +echo " java=$JAVA workers=$WORKERS heap=$HEAP extra='$TLC_EXTRA'" +echo " spec=$ALLOC_DIR/MCDeferredAlloc.tla" +if [ "$SYNC_CHECKPOINT" = "1" ]; then + echo " metadir=$LOCAL_STATES (node-local) checkpoint-sync=$SHARED_STATES (every ~$(( CHECKPOINT_MIN / 2 ))min)" +else + echo " metadir=$LOCAL_STATES (node-local) checkpoint-sync=OFF (nothing is written to shared storage)" +fi +df -h "$LOCAL_ROOT" | tail -1 | awk '{print " node-local disk: size="$2" used="$3" avail="$4}' +AVAIL_G=$(df -BG --output=avail "$LOCAL_ROOT" 2>/dev/null | tail -1 | tr -dc '0-9') +if [ -n "$AVAIL_G" ] && [ "$AVAIL_G" -lt 100 ]; then + echo " WARNING: only ${AVAIL_G}G free on $LOCAL_ROOT - large runs WILL die with" + echo " 'No space left on device'. Check for co-scheduled TLC jobs or" + echo " leftover /tmp junk (du -sh $LOCAL_ROOT/* | sort -h | tail);" + echo " consider sbatch --exclusive so no two jobs share a node." +fi +"$JAVA" -version 2>&1 | head -1 + +# stage a shared-storage checkpoint onto local disk for recovery +if [ -n "$RECOVER_FROM" ]; then + if [ ! -f "$RECOVER_FROM/vars.chkpt" ]; then + echo "error: RECOVER_FROM=$RECOVER_FROM has no vars.chkpt" >&2 + exit 1 + fi + ts=$(basename "$RECOVER_FROM") + echo " staging checkpoint $ts to node-local disk..." + rsync -a "$RECOVER_FROM/" "$LOCAL_STATES/$ts/" || { echo "error: checkpoint staging failed" >&2; exit 1; } + TLC_EXTRA="$TLC_EXTRA -recover $LOCAL_STATES/$ts" + echo " staged; recovering from $LOCAL_STATES/$ts" +fi + +# background checkpoint sync: newest quiescent checkpoint -> shared storage +sync_loop() { + while sleep $(( CHECKPOINT_MIN * 30 )); do # CHECKPOINT_MIN/2 in seconds + newest=$(ls -dt "$LOCAL_STATES"/*/ 2>/dev/null | head -1) + [ -n "$newest" ] || continue + [ -f "$newest/vars.chkpt" ] || continue + # quiescent = nothing written in the last minute (not mid-checkpoint) + [ -z "$(find "$newest" -mmin -1 -print -quit 2>/dev/null)" ] || continue + ts=$(basename "$newest") + [ -d "$SHARED_STATES/$ts" ] && continue # this checkpoint already synced + rsync -a "$newest/" "$SHARED_STATES/$ts.partial/" || continue + rm -rf "$SHARED_STATES/$ts" && mv "$SHARED_STATES/$ts.partial" "$SHARED_STATES/$ts" + # prune older synced checkpoints, keep the newest one only + for d in "$SHARED_STATES"/*/; do + [ "$(basename "$d")" = "$ts" ] || rm -rf "$d" + done + echo " [sync] checkpoint $ts synced to shared storage at $(date '+%F %T')" + done +} +SYNC_PID="" +if [ "$SYNC_CHECKPOINT" = "1" ]; then + sync_loop & + SYNC_PID=$! +fi + +cleanup() { + [ -n "$SYNC_PID" ] && kill "$SYNC_PID" 2>/dev/null + rm -rf "$LOCAL_DIR" +} +trap cleanup EXIT + +# shellcheck disable=SC2086 +"$JAVA" -XX:+UseParallelGC -Xmx"$HEAP" \ + -Djava.io.tmpdir="$LOCAL_JTMP" \ + -cp "$JAR" tlc2.TLC \ + -config "$ALLOC_DIR/$CONFIG.cfg" \ + -workers "$WORKERS" \ + -checkpoint "$CHECKPOINT_MIN" \ + -metadir "$LOCAL_STATES" \ + $TLC_EXTRA \ + "$ALLOC_DIR/MCDeferredAlloc.tla" +rc=$? + +# final sync attempt: if TLC exited on its own (not killed), the last +# checkpoint is quiescent and worth keeping for a later resume +if [ "$SYNC_CHECKPOINT" = "1" ]; then + newest=$(ls -dt "$LOCAL_STATES"/*/ 2>/dev/null | head -1) + if [ -n "$newest" ] && [ -f "$newest/vars.chkpt" ]; then + ts=$(basename "$newest") + if [ ! -d "$SHARED_STATES/$ts" ]; then + rsync -a "$newest/" "$SHARED_STATES/$ts.partial/" \ + && rm -rf "$SHARED_STATES/$ts" \ + && mv "$SHARED_STATES/$ts.partial" "$SHARED_STATES/$ts" \ + && echo " [sync] final checkpoint $ts synced" + fi + fi +fi + +echo "=== TLC exit code: $rc" +exit $rc diff --git a/tla/allocation/traces/Composite4.txt b/tla/allocation/traces/Composite4.txt new file mode 100644 index 0000000000..25c882d731 --- /dev/null +++ b/tla/allocation/traces/Composite4.txt @@ -0,0 +1,528 @@ +TLC2 Version 2.19 of 08 August 2024 (rev: 5a47802) +Running breadth-first search Model-Checking with fp 56 and seed 6629413510475157693 with 4 workers on 10 cores with 3641MB heap and 64MB offheap memory [pid: 93863] (Mac OS X 15.7.7 aarch64, Homebrew 26.0.2 x86_64, MSBDiskFPSet, DiskStateQueue). +Parsing file /Users/mebauer/realm/tla/allocation/MCDeferredAlloc.tla +Parsing file /Users/mebauer/realm/tla/allocation/DeferredAlloc.tla +Parsing file /private/tmp/claude-502/-Users-mebauer-realm-src-realm/e16dc9f9-47a4-47cb-a4d3-fd41b34c5bce/scratchpad/jtmp/Integers.tla +Parsing file /private/tmp/claude-502/-Users-mebauer-realm-src-realm/e16dc9f9-47a4-47cb-a4d3-fd41b34c5bce/scratchpad/jtmp/Sequences.tla +Parsing file /private/tmp/claude-502/-Users-mebauer-realm-src-realm/e16dc9f9-47a4-47cb-a4d3-fd41b34c5bce/scratchpad/jtmp/FiniteSets.tla +Parsing file /private/tmp/claude-502/-Users-mebauer-realm-src-realm/e16dc9f9-47a4-47cb-a4d3-fd41b34c5bce/scratchpad/jtmp/TLC.tla +Parsing file /private/tmp/claude-502/-Users-mebauer-realm-src-realm/e16dc9f9-47a4-47cb-a4d3-fd41b34c5bce/scratchpad/jtmp/Naturals.tla +Semantic processing of module Naturals +Semantic processing of module Integers +Semantic processing of module Sequences +Semantic processing of module FiniteSets +Semantic processing of module TLC +Semantic processing of module DeferredAlloc +Semantic processing of module MCDeferredAlloc +Starting... (2026-08-25 14:23:05) +Computing initial states... +Finished computing initial states: 1 distinct state generated at 2026-08-25 14:23:05. +Progress(11) at 2026-08-25 14:23:08: 61,288 states generated (61,288 s/min), 34,916 distinct states found (34,916 ds/min), 16,326 states left on queue. +Error: Invariant INV_CurrentMatchesGround is violated. +Error: The behavior up to this point is: +State 1: +/\ destroyRequested = <> +/\ balC = <<"NONE", "NONE", "NONE", "NONE", "NONE">> +/\ balD = <<"NONE", "NONE", "NONE", "NONE", "NONE">> +/\ instOffset = <<-1, -1, -1, -1, -1>> +/\ preC = << [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ preD = << [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ seqCtr = 0 +/\ cur = << >> +/\ allocatedEver = {} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = <<>> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE", "NONE", "NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = << >> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<>> +/\ wasDeferred = {} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"UNFIRED", "UNFIRED", "UNFIRED", "UNFIRED", "UNFIRED">> +/\ instState = <<"UNREQUESTED", "UNREQUESTED", "UNREQUESTED", "UNREQUESTED", "UNREQUESTED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0, 0, 0, 0>> +/\ dupAlloc = FALSE +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 2: +/\ destroyRequested = <> +/\ balC = <<"NONE", "NONE", "NONE", "NONE", "NONE">> +/\ balD = <<"NONE", "NONE", "NONE", "NONE", "NONE">> +/\ instOffset = <<0, -1, -1, -1, -1>> +/\ preC = << [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ preD = << [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ seqCtr = 0 +/\ cur = <<[first |-> 0, size |-> 2]>> +/\ allocatedEver = {1} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = <<>> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE", "NONE", "NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = << >> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<>> +/\ wasDeferred = {} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"CLEAN", "UNFIRED", "UNFIRED", "UNFIRED", "UNFIRED">> +/\ instState = <<"ALLOCATED", "UNREQUESTED", "UNREQUESTED", "UNREQUESTED", "UNREQUESTED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0, 0, 0, 0>> +/\ dupAlloc = FALSE +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 3: +/\ destroyRequested = <> +/\ balC = <<"NONE", "NONE", "NONE", "NONE", "NONE">> +/\ balD = <<"NONE", "NONE", "NONE", "NONE", "NONE">> +/\ instOffset = <<0, -1, 2, -1, -1>> +/\ preC = << [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ preD = << [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ seqCtr = 0 +/\ cur = (1 :> [first |-> 0, size |-> 2] @@ 3 :> [first |-> 2, size |-> 1]) +/\ allocatedEver = {1, 3} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = <<>> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE", "NONE", "NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = << >> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<>> +/\ wasDeferred = {} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"CLEAN", "UNFIRED", "CLEAN", "UNFIRED", "UNFIRED">> +/\ instState = <<"ALLOCATED", "UNREQUESTED", "ALLOCATED", "UNREQUESTED", "UNREQUESTED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0, 0, 0, 0>> +/\ dupAlloc = FALSE +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 4: +/\ destroyRequested = <> +/\ balC = <<"NONE", "NONE", "NONE", "NONE", "NONE">> +/\ balD = <<"NONE", "NONE", "NONE", "NONE", "NONE">> +/\ instOffset = <<0, -1, 2, 3, -1>> +/\ preC = << [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ preD = << [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ seqCtr = 0 +/\ cur = ( 1 :> [first |-> 0, size |-> 2] @@ + 3 :> [first |-> 2, size |-> 1] @@ + 4 :> [first |-> 3, size |-> 1] ) +/\ allocatedEver = {1, 3, 4} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = <<>> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE", "NONE", "NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = << >> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<>> +/\ wasDeferred = {} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"CLEAN", "UNFIRED", "CLEAN", "CLEAN", "UNFIRED">> +/\ instState = <<"ALLOCATED", "UNREQUESTED", "ALLOCATED", "ALLOCATED", "UNREQUESTED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0, 0, 0, 0>> +/\ dupAlloc = FALSE +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 5: +/\ destroyRequested = <> +/\ balC = <<"NONE", "NONE", "NONE", "NONE", "NONE">> +/\ balD = <<"PENDING", "NONE", "NONE", "NONE", "NONE">> +/\ instOffset = <<0, -1, 2, 3, -1>> +/\ preC = << [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ preD = << [deps |-> {1}, ballistic |-> TRUE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ seqCtr = 1 +/\ cur = ( 1 :> [first |-> 0, size |-> 2] @@ + 3 :> [first |-> 2, size |-> 1] @@ + 4 :> [first |-> 3, size |-> 1] ) +/\ allocatedEver = {1, 3, 4} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = <<[inst |-> 1, isReady |-> FALSE, defNote |-> FALSE, seq |-> 1]>> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE", "NONE", "NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = << >> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<>> +/\ wasDeferred = {} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"CLEAN", "UNFIRED", "CLEAN", "CLEAN", "UNFIRED">> +/\ instState = <<"ALLOCATED", "UNREQUESTED", "ALLOCATED", "ALLOCATED", "UNREQUESTED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0, 0, 0, 0>> +/\ dupAlloc = FALSE +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 6: +/\ destroyRequested = <> +/\ balC = <<"NONE", "NONE", "NONE", "NONE", "NONE">> +/\ balD = <<"PENDING", "NONE", "NONE", "NONE", "NONE">> +/\ instOffset = <<0, -1, 2, 3, -1>> +/\ preC = << [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ preD = << [deps |-> {1}, ballistic |-> TRUE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ seqCtr = 1 +/\ cur = ( 1 :> [first |-> 0, size |-> 2] @@ + 3 :> [first |-> 2, size |-> 1] @@ + 4 :> [first |-> 3, size |-> 1] ) +/\ allocatedEver = {1, 3, 4} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = <<[inst |-> 1, isReady |-> FALSE, defNote |-> FALSE, seq |-> 1]>> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE", "NONE", "NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = ( 1 :> [first |-> 0, size |-> 2] @@ + 3 :> [first |-> 2, size |-> 1] @@ + 4 :> [first |-> 3, size |-> 1] ) +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<[inst |-> 2, size |-> 2, lastSeq |-> 1]>> +/\ wasDeferred = {2} +/\ structuralAssertFailed = FALSE +/\ fut = ( 2 :> [first |-> 0, size |-> 2] @@ + 3 :> [first |-> 2, size |-> 1] @@ + 4 :> [first |-> 3, size |-> 1] ) +/\ eCreated = <<"CLEAN", "UNFIRED", "CLEAN", "CLEAN", "UNFIRED">> +/\ instState = <<"ALLOCATED", "ALLOC_DEFERRED", "ALLOCATED", "ALLOCATED", "UNREQUESTED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0, 0, 0, 0>> +/\ dupAlloc = FALSE +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 7: +/\ destroyRequested = <> +/\ balC = <<"NONE", "NONE", "NONE", "NONE", "NONE">> +/\ balD = <<"PENDING", "NONE", "NONE", "NONE", "NONE">> +/\ instOffset = <<0, -1, 2, 3, -1>> +/\ preC = << [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ preD = << [deps |-> {1}, ballistic |-> TRUE], + [deps |-> {2}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ seqCtr = 2 +/\ cur = ( 1 :> [first |-> 0, size |-> 2] @@ + 3 :> [first |-> 2, size |-> 1] @@ + 4 :> [first |-> 3, size |-> 1] ) +/\ allocatedEver = {1, 3, 4} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = << [inst |-> 1, isReady |-> FALSE, defNote |-> FALSE, seq |-> 1], + [inst |-> 2, isReady |-> FALSE, defNote |-> FALSE, seq |-> 2] >> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE", "NONE", "NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = ( 1 :> [first |-> 0, size |-> 2] @@ + 3 :> [first |-> 2, size |-> 1] @@ + 4 :> [first |-> 3, size |-> 1] ) +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<[inst |-> 2, size |-> 2, lastSeq |-> 1]>> +/\ wasDeferred = {2} +/\ structuralAssertFailed = FALSE +/\ fut = (3 :> [first |-> 2, size |-> 1] @@ 4 :> [first |-> 3, size |-> 1]) +/\ eCreated = <<"CLEAN", "UNFIRED", "CLEAN", "CLEAN", "UNFIRED">> +/\ instState = <<"ALLOCATED", "ALLOC_DEFERRED", "ALLOCATED", "ALLOCATED", "UNREQUESTED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0, 0, 0, 0>> +/\ dupAlloc = FALSE +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 8: +/\ destroyRequested = <> +/\ balC = <<"NONE", "NONE", "NONE", "NONE", "NONE">> +/\ balD = <<"FIRED", "NONE", "NONE", "NONE", "NONE">> +/\ instOffset = <<0, -1, 2, 3, -1>> +/\ preC = << [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ preD = << [deps |-> {1}, ballistic |-> TRUE], + [deps |-> {2}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ seqCtr = 2 +/\ cur = ( 1 :> [first |-> 0, size |-> 2] @@ + 3 :> [first |-> 2, size |-> 1] @@ + 4 :> [first |-> 3, size |-> 1] ) +/\ allocatedEver = {1, 3, 4} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = << [inst |-> 1, isReady |-> FALSE, defNote |-> FALSE, seq |-> 1], + [inst |-> 2, isReady |-> FALSE, defNote |-> FALSE, seq |-> 2] >> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE", "NONE", "NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = ( 1 :> [first |-> 0, size |-> 2] @@ + 3 :> [first |-> 2, size |-> 1] @@ + 4 :> [first |-> 3, size |-> 1] ) +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<[inst |-> 2, size |-> 2, lastSeq |-> 1]>> +/\ wasDeferred = {2} +/\ structuralAssertFailed = FALSE +/\ fut = (3 :> [first |-> 2, size |-> 1] @@ 4 :> [first |-> 3, size |-> 1]) +/\ eCreated = <<"CLEAN", "UNFIRED", "CLEAN", "CLEAN", "UNFIRED">> +/\ instState = <<"ALLOCATED", "ALLOC_DEFERRED", "ALLOCATED", "ALLOCATED", "UNREQUESTED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0, 0, 0, 0>> +/\ dupAlloc = FALSE +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 9: +/\ destroyRequested = <> +/\ balC = <<"NONE", "NONE", "NONE", "NONE", "NONE">> +/\ balD = <<"FIRED", "NONE", "NONE", "NONE", "NONE">> +/\ instOffset = <<0, -1, 2, 3, -1>> +/\ preC = << [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ preD = << [deps |-> {1}, ballistic |-> TRUE], + [deps |-> {2}, ballistic |-> FALSE], + [deps |-> {3}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ seqCtr = 3 +/\ cur = ( 1 :> [first |-> 0, size |-> 2] @@ + 3 :> [first |-> 2, size |-> 1] @@ + 4 :> [first |-> 3, size |-> 1] ) +/\ allocatedEver = {1, 3, 4} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = << [inst |-> 1, isReady |-> FALSE, defNote |-> FALSE, seq |-> 1], + [inst |-> 2, isReady |-> FALSE, defNote |-> FALSE, seq |-> 2], + [inst |-> 3, isReady |-> TRUE, defNote |-> TRUE, seq |-> 3] >> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE", "NONE", "NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = (1 :> [first |-> 0, size |-> 2] @@ 4 :> [first |-> 3, size |-> 1]) +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<[inst |-> 2, size |-> 2, lastSeq |-> 1]>> +/\ wasDeferred = {2} +/\ structuralAssertFailed = FALSE +/\ fut = (4 :> [first |-> 3, size |-> 1]) +/\ eCreated = <<"CLEAN", "UNFIRED", "CLEAN", "CLEAN", "UNFIRED">> +/\ instState = <<"ALLOCATED", "ALLOC_DEFERRED", "ALLOCATED", "ALLOCATED", "UNREQUESTED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0, 0, 0, 0>> +/\ dupAlloc = FALSE +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 10: +/\ destroyRequested = <> +/\ balC = <<"NONE", "NONE", "NONE", "NONE", "NONE">> +/\ balD = <<"FIRED", "NONE", "NONE", "NONE", "NONE">> +/\ instOffset = <<0, 0, 2, 3, -1>> +/\ preC = << [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ preD = << [deps |-> {1}, ballistic |-> TRUE], + [deps |-> {2}, ballistic |-> FALSE], + [deps |-> {3}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ seqCtr = 3 +/\ cur = ( 2 :> [first |-> 0, size |-> 2] @@ + 3 :> [first |-> 2, size |-> 1] @@ + 4 :> [first |-> 3, size |-> 1] ) +/\ allocatedEver = {1, 2, 3, 4} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = << [inst |-> 2, isReady |-> FALSE, defNote |-> FALSE, seq |-> 2], + [inst |-> 3, isReady |-> TRUE, defNote |-> TRUE, seq |-> 3] >> +/\ curFreed = {1} +/\ failedVia = <<"NONE", "NONE", "NONE", "NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = (4 :> [first |-> 3, size |-> 1]) +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<>> +/\ wasDeferred = {2} +/\ structuralAssertFailed = FALSE +/\ fut = (4 :> [first |-> 3, size |-> 1]) +/\ eCreated = <<"CLEAN", "CLEAN", "CLEAN", "CLEAN", "UNFIRED">> +/\ instState = <<"DESTROYED", "ALLOCATED", "ALLOCATED", "ALLOCATED", "UNREQUESTED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<1, 0, 0, 0, 0>> +/\ dupAlloc = FALSE +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 11: +/\ destroyRequested = <> +/\ balC = <<"NONE", "NONE", "NONE", "NONE", "NONE">> +/\ balD = <<"FIRED", "NONE", "NONE", "NONE", "NONE">> +/\ instOffset = <<0, 0, 2, 3, -1>> +/\ preC = << [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ preD = << [deps |-> {1}, ballistic |-> TRUE], + [deps |-> {2}, ballistic |-> FALSE], + [deps |-> {3}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ seqCtr = 3 +/\ cur = ( 2 :> [first |-> 0, size |-> 2] @@ + 3 :> [first |-> 2, size |-> 1] @@ + 4 :> [first |-> 3, size |-> 1] ) +/\ allocatedEver = {1, 2, 3, 4} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = << [inst |-> 2, isReady |-> FALSE, defNote |-> FALSE, seq |-> 2], + [inst |-> 3, isReady |-> TRUE, defNote |-> TRUE, seq |-> 3] >> +/\ curFreed = {1} +/\ failedVia = <<"NONE", "NONE", "NONE", "NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = ( 2 :> [first |-> 0, size |-> 2] @@ + 3 :> [first |-> 2, size |-> 1] @@ + 4 :> [first |-> 3, size |-> 1] ) +/\ readyAtRebuild = TRUE +/\ pendingAllocs = <<[inst |-> 5, size |-> 1, lastSeq |-> 3]>> +/\ wasDeferred = {2, 5} +/\ structuralAssertFailed = FALSE +/\ fut = (4 :> [first |-> 3, size |-> 1] @@ 5 :> [first |-> 0, size |-> 1]) +/\ eCreated = <<"CLEAN", "CLEAN", "CLEAN", "CLEAN", "UNFIRED">> +/\ instState = <<"DESTROYED", "ALLOCATED", "ALLOCATED", "ALLOCATED", "ALLOC_DEFERRED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<1, 0, 0, 0, 0>> +/\ dupAlloc = FALSE +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 12: +/\ destroyRequested = <> +/\ balC = <<"NONE", "NONE", "NONE", "NONE", "NONE">> +/\ balD = <<"FIRED", "NONE", "NONE", "NONE", "NONE">> +/\ instOffset = <<0, 0, 2, 3, 3>> +/\ preC = << [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ preD = << [deps |-> {1}, ballistic |-> TRUE], + [deps |-> {2}, ballistic |-> FALSE], + [deps |-> {3}, ballistic |-> FALSE], + [deps |-> {4}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ seqCtr = 3 +/\ cur = ( 2 :> [first |-> 0, size |-> 2] @@ + 3 :> [first |-> 2, size |-> 1] @@ + 5 :> [first |-> 3, size |-> 1] ) +/\ allocatedEver = {1, 2, 3, 4, 5} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = <<[inst |-> 2, isReady |-> FALSE, defNote |-> FALSE, seq |-> 2]>> +/\ curFreed = {1, 3, 4} +/\ failedVia = <<"NONE", "NONE", "NONE", "NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = (2 :> [first |-> 0, size |-> 2] @@ 3 :> [first |-> 2, size |-> 1]) +/\ readyAtRebuild = TRUE +/\ pendingAllocs = <<>> +/\ wasDeferred = {2, 5} +/\ structuralAssertFailed = FALSE +/\ fut = (5 :> [first |-> 0, size |-> 1]) +/\ eCreated = <<"CLEAN", "CLEAN", "CLEAN", "CLEAN", "CLEAN">> +/\ instState = <<"DESTROYED", "ALLOCATED", "DESTROYED", "DESTROYED", "ALLOCATED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<1, 0, 1, 1, 0>> +/\ dupAlloc = FALSE +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +93795 states generated, 51904 distinct states found, 20948 states left on queue. +The depth of the complete state graph search is 12. +The average outdegree of the complete state graph is 2 (minimum is 0, the maximum 5 and the 95th percentile is 4). +Finished in 05s at (2026-08-25 14:23:10) diff --git a/tla/allocation/traces/EventLoop.trace.txt b/tla/allocation/traces/EventLoop.trace.txt new file mode 100644 index 0000000000..77c4d6bec4 --- /dev/null +++ b/tla/allocation/traces/EventLoop.trace.txt @@ -0,0 +1,236 @@ +TLC2 Version 2.19 of 08 August 2024 (rev: 5a47802) +Running breadth-first search Model-Checking with fp 49 and seed -6665322731745831606 with 4 workers on 10 cores with 3641MB heap and 64MB offheap memory [pid: 92945] (Mac OS X 15.7.7 aarch64, Homebrew 26.0.2 x86_64, MSBDiskFPSet, DiskStateQueue). +Parsing file /Users/mebauer/realm/tla/allocation/MCDeferredAlloc.tla +Parsing file /Users/mebauer/realm/tla/allocation/DeferredAlloc.tla +Parsing file /Users/mebauer/realm/tla/allocation/jtmp/Integers.tla +Parsing file /Users/mebauer/realm/tla/allocation/jtmp/Sequences.tla +Parsing file /Users/mebauer/realm/tla/allocation/jtmp/FiniteSets.tla +Parsing file /Users/mebauer/realm/tla/allocation/jtmp/TLC.tla +Parsing file /Users/mebauer/realm/tla/allocation/jtmp/Naturals.tla +Semantic processing of module Naturals +Semantic processing of module Integers +Semantic processing of module Sequences +Semantic processing of module FiniteSets +Semantic processing of module TLC +Semantic processing of module DeferredAlloc +Semantic processing of module MCDeferredAlloc +Starting... (2026-08-25 13:49:20) +Computing initial states... +Finished computing initial states: 1 distinct state generated at 2026-08-25 13:49:20. +Error: Deadlock reached. +Error: The behavior up to this point is: +State 1: +/\ destroyRequested = <> +/\ balC = <<"NONE", "NONE">> +/\ balD = <<"NONE", "NONE">> +/\ instOffset = <<-1, -1>> +/\ preC = <<[deps |-> {}, ballistic |-> FALSE], [deps |-> {}, ballistic |-> FALSE]>> +/\ preD = <<[deps |-> {}, ballistic |-> FALSE], [deps |-> {}, ballistic |-> FALSE]>> +/\ seqCtr = 0 +/\ cur = << >> +/\ allocatedEver = {} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = <<>> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = << >> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<>> +/\ wasDeferred = {} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"UNFIRED", "UNFIRED">> +/\ instState = <<"UNREQUESTED", "UNREQUESTED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0>> +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 2: +/\ destroyRequested = <> +/\ balC = <<"NONE", "NONE">> +/\ balD = <<"NONE", "NONE">> +/\ instOffset = <<0, -1>> +/\ preC = <<[deps |-> {}, ballistic |-> FALSE], [deps |-> {}, ballistic |-> FALSE]>> +/\ preD = <<[deps |-> {}, ballistic |-> FALSE], [deps |-> {}, ballistic |-> FALSE]>> +/\ seqCtr = 0 +/\ cur = <<[first |-> 0, size |-> 3]>> +/\ allocatedEver = {1} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = <<>> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = << >> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<>> +/\ wasDeferred = {} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"CLEAN", "UNFIRED">> +/\ instState = <<"ALLOCATED", "UNREQUESTED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0>> +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 3: +/\ destroyRequested = <> +/\ balC = <<"NONE", "PENDING">> +/\ balD = <<"NONE", "NONE">> +/\ instOffset = <<0, -1>> +/\ preC = <<[deps |-> {}, ballistic |-> FALSE], [deps |-> {}, ballistic |-> TRUE]>> +/\ preD = <<[deps |-> {}, ballistic |-> FALSE], [deps |-> {}, ballistic |-> FALSE]>> +/\ seqCtr = 0 +/\ cur = <<[first |-> 0, size |-> 3]>> +/\ allocatedEver = {1} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = <<>> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = << >> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<>> +/\ wasDeferred = {} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"CLEAN", "UNFIRED">> +/\ instState = <<"ALLOCATED", "CREATE_PENDING">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0>> +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 4: +/\ destroyRequested = <> +/\ balC = <<"NONE", "PENDING">> +/\ balD = <<"NONE", "NONE">> +/\ instOffset = <<0, -1>> +/\ preC = <<[deps |-> {}, ballistic |-> FALSE], [deps |-> {}, ballistic |-> TRUE]>> +/\ preD = <<[deps |-> {1, 2}, ballistic |-> FALSE], [deps |-> {}, ballistic |-> FALSE]>> +/\ seqCtr = 1 +/\ cur = <<[first |-> 0, size |-> 3]>> +/\ allocatedEver = {1} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = <<[inst |-> 1, isReady |-> FALSE, defNote |-> FALSE, seq |-> 1]>> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = << >> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<>> +/\ wasDeferred = {} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"CLEAN", "UNFIRED">> +/\ instState = <<"ALLOCATED", "CREATE_PENDING">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0>> +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 5: +/\ destroyRequested = <> +/\ balC = <<"NONE", "PENDING">> +/\ balD = <<"NONE", "NONE">> +/\ instOffset = <<0, -1>> +/\ preC = <<[deps |-> {}, ballistic |-> FALSE], [deps |-> {}, ballistic |-> TRUE]>> +/\ preD = <<[deps |-> {1, 2}, ballistic |-> FALSE], [deps |-> {2}, ballistic |-> FALSE]>> +/\ seqCtr = 1 +/\ cur = <<[first |-> 0, size |-> 3]>> +/\ allocatedEver = {1} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = <<[inst |-> 1, isReady |-> FALSE, defNote |-> FALSE, seq |-> 1]>> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = << >> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<>> +/\ wasDeferred = {} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"CLEAN", "UNFIRED">> +/\ instState = <<"ALLOCATED", "CREATE_PENDING_DESTROY">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0>> +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 6: +/\ destroyRequested = <> +/\ balC = <<"NONE", "FIRED">> +/\ balD = <<"NONE", "NONE">> +/\ instOffset = <<0, -1>> +/\ preC = <<[deps |-> {}, ballistic |-> FALSE], [deps |-> {}, ballistic |-> TRUE]>> +/\ preD = <<[deps |-> {1, 2}, ballistic |-> FALSE], [deps |-> {2}, ballistic |-> FALSE]>> +/\ seqCtr = 1 +/\ cur = <<[first |-> 0, size |-> 3]>> +/\ allocatedEver = {1} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = <<[inst |-> 1, isReady |-> FALSE, defNote |-> FALSE, seq |-> 1]>> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = << >> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<>> +/\ wasDeferred = {} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"CLEAN", "UNFIRED">> +/\ instState = <<"ALLOCATED", "CREATE_PENDING_DESTROY">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0>> +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 7: +/\ destroyRequested = <> +/\ balC = <<"NONE", "FIRED">> +/\ balD = <<"NONE", "NONE">> +/\ instOffset = <<0, -1>> +/\ preC = <<[deps |-> {}, ballistic |-> FALSE], [deps |-> {}, ballistic |-> TRUE]>> +/\ preD = <<[deps |-> {1, 2}, ballistic |-> FALSE], [deps |-> {2}, ballistic |-> FALSE]>> +/\ seqCtr = 2 +/\ cur = <<[first |-> 0, size |-> 3]>> +/\ allocatedEver = {1} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = << [inst |-> 1, isReady |-> FALSE, defNote |-> FALSE, seq |-> 1], + [inst |-> 2, isReady |-> FALSE, defNote |-> FALSE, seq |-> 2] >> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = <<[first |-> 0, size |-> 3]>> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<[inst |-> 2, size |-> 3, lastSeq |-> 1]>> +/\ wasDeferred = {2} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"CLEAN", "UNFIRED">> +/\ instState = <<"ALLOCATED", "ALLOC_DEFERRED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0>> +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +57 states generated, 39 distinct states found, 3 states left on queue. +The depth of the complete state graph search is 8. +The average outdegree of the complete state graph is 1 (minimum is 0, the maximum 3 and the 95th percentile is 2). +Finished in 00s at (2026-08-25 13:49:20) diff --git a/tla/allocation/traces/GCRipple-orderguard-misfire.txt b/tla/allocation/traces/GCRipple-orderguard-misfire.txt new file mode 100644 index 0000000000..b1a5f955af --- /dev/null +++ b/tla/allocation/traces/GCRipple-orderguard-misfire.txt @@ -0,0 +1,217 @@ +TLC2 Version 2.19 of 08 August 2024 (rev: 5a47802) +Running breadth-first search Model-Checking with fp 69 and seed -5056495996037738133 with 4 workers on 10 cores with 3641MB heap and 64MB offheap memory [pid: 7804] (Mac OS X 15.7.7 aarch64, Homebrew 26.0.2 x86_64, MSBDiskFPSet, DiskStateQueue). +Parsing file /Users/mebauer/realm/tla/allocation/MCDeferredAlloc.tla +Parsing file /Users/mebauer/realm/tla/allocation/DeferredAlloc.tla +Parsing file /private/tmp/claude-502/-Users-mebauer-realm-src-realm/e16dc9f9-47a4-47cb-a4d3-fd41b34c5bce/scratchpad/tlc-jtmp/Integers.tla +Parsing file /private/tmp/claude-502/-Users-mebauer-realm-src-realm/e16dc9f9-47a4-47cb-a4d3-fd41b34c5bce/scratchpad/tlc-jtmp/Sequences.tla +Parsing file /private/tmp/claude-502/-Users-mebauer-realm-src-realm/e16dc9f9-47a4-47cb-a4d3-fd41b34c5bce/scratchpad/tlc-jtmp/FiniteSets.tla +Parsing file /private/tmp/claude-502/-Users-mebauer-realm-src-realm/e16dc9f9-47a4-47cb-a4d3-fd41b34c5bce/scratchpad/tlc-jtmp/TLC.tla +Parsing file /private/tmp/claude-502/-Users-mebauer-realm-src-realm/e16dc9f9-47a4-47cb-a4d3-fd41b34c5bce/scratchpad/tlc-jtmp/Naturals.tla +Semantic processing of module Naturals +Semantic processing of module Integers +Semantic processing of module Sequences +Semantic processing of module FiniteSets +Semantic processing of module TLC +Semantic processing of module DeferredAlloc +Semantic processing of module MCDeferredAlloc +Starting... (2026-08-25 23:57:59) +Computing initial states... +Finished computing initial states: 1 distinct state generated at 2026-08-25 23:57:59. +Error: Invariant INV_GCRippleSuccessFunded is violated. +Error: The behavior up to this point is: +State 1: +/\ destroyRequested = <> +/\ balC = <<"NONE", "NONE">> +/\ balD = <<"NONE", "NONE">> +/\ instOffset = <<-1, -1>> +/\ preC = <<[deps |-> {}, ballistic |-> FALSE], [deps |-> {}, ballistic |-> FALSE]>> +/\ preD = <<[deps |-> {}, ballistic |-> FALSE], [deps |-> {}, ballistic |-> FALSE]>> +/\ seqCtr = 0 +/\ cur = << >> +/\ allocatedEver = {} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = <<>> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = << >> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<>> +/\ wasDeferred = {} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"UNFIRED", "UNFIRED">> +/\ instState = <<"UNREQUESTED", "UNREQUESTED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0>> +/\ reqCap = <<0, 0>> +/\ dupAlloc = FALSE +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 2: +/\ destroyRequested = <> +/\ balC = <<"NONE", "PENDING">> +/\ balD = <<"NONE", "NONE">> +/\ instOffset = <<-1, -1>> +/\ preC = <<[deps |-> {}, ballistic |-> FALSE], [deps |-> {}, ballistic |-> TRUE]>> +/\ preD = <<[deps |-> {}, ballistic |-> FALSE], [deps |-> {}, ballistic |-> FALSE]>> +/\ seqCtr = 0 +/\ cur = << >> +/\ allocatedEver = {} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = <<>> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = << >> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<>> +/\ wasDeferred = {} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"UNFIRED", "UNFIRED">> +/\ instState = <<"UNREQUESTED", "CREATE_PENDING">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0>> +/\ reqCap = <<0, 0>> +/\ dupAlloc = FALSE +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 3: +/\ destroyRequested = <> +/\ balC = <<"NONE", "FIRED">> +/\ balD = <<"NONE", "NONE">> +/\ instOffset = <<-1, -1>> +/\ preC = <<[deps |-> {}, ballistic |-> FALSE], [deps |-> {}, ballistic |-> TRUE]>> +/\ preD = <<[deps |-> {}, ballistic |-> FALSE], [deps |-> {}, ballistic |-> FALSE]>> +/\ seqCtr = 0 +/\ cur = << >> +/\ allocatedEver = {} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = <<>> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = << >> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<>> +/\ wasDeferred = {} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"UNFIRED", "UNFIRED">> +/\ instState = <<"UNREQUESTED", "CREATE_PENDING">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0>> +/\ reqCap = <<0, 0>> +/\ dupAlloc = FALSE +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 4: +/\ destroyRequested = <> +/\ balC = <<"NONE", "FIRED">> +/\ balD = <<"NONE", "NONE">> +/\ instOffset = <<-1, 0>> +/\ preC = <<[deps |-> {}, ballistic |-> FALSE], [deps |-> {}, ballistic |-> TRUE]>> +/\ preD = <<[deps |-> {}, ballistic |-> FALSE], [deps |-> {}, ballistic |-> FALSE]>> +/\ seqCtr = 0 +/\ cur = (2 :> [first |-> 0, size |-> 3]) +/\ allocatedEver = {2} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = <<>> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = << >> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<>> +/\ wasDeferred = {} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"UNFIRED", "CLEAN">> +/\ instState = <<"UNREQUESTED", "ALLOCATED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0>> +/\ reqCap = <<0, 0>> +/\ dupAlloc = FALSE +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 5: +/\ destroyRequested = <> +/\ balC = <<"NONE", "FIRED">> +/\ balD = <<"NONE", "NONE">> +/\ instOffset = <<-1, 0>> +/\ preC = <<[deps |-> {}, ballistic |-> FALSE], [deps |-> {}, ballistic |-> TRUE]>> +/\ preD = <<[deps |-> {}, ballistic |-> FALSE], [deps |-> {2}, ballistic |-> FALSE]>> +/\ seqCtr = 0 +/\ cur = << >> +/\ allocatedEver = {2} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = <<>> +/\ curFreed = {2} +/\ failedVia = <<"NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = << >> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<>> +/\ wasDeferred = {} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"UNFIRED", "CLEAN">> +/\ instState = <<"UNREQUESTED", "DESTROYED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 1>> +/\ reqCap = <<0, 0>> +/\ dupAlloc = FALSE +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 6: +/\ destroyRequested = <> +/\ balC = <<"NONE", "FIRED">> +/\ balD = <<"NONE", "NONE">> +/\ instOffset = <<0, 0>> +/\ preC = <<[deps |-> {}, ballistic |-> FALSE], [deps |-> {}, ballistic |-> TRUE]>> +/\ preD = <<[deps |-> {}, ballistic |-> FALSE], [deps |-> {2}, ballistic |-> FALSE]>> +/\ seqCtr = 0 +/\ cur = <<[first |-> 0, size |-> 3]>> +/\ allocatedEver = {1, 2} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = <<>> +/\ curFreed = {2} +/\ failedVia = <<"NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = << >> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<>> +/\ wasDeferred = {} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"CLEAN", "CLEAN">> +/\ instState = <<"ALLOCATED", "DESTROYED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 1>> +/\ reqCap = <<0, 0>> +/\ dupAlloc = FALSE +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +89 states generated, 61 distinct states found, 10 states left on queue. +The depth of the complete state graph search is 10. +The average outdegree of the complete state graph is 1 (minimum is 0, the maximum 3 and the 95th percentile is 2). +Finished in 00s at (2026-08-25 23:57:59) diff --git a/tla/allocation/traces/Inversion-bug5-deadlock.txt b/tla/allocation/traces/Inversion-bug5-deadlock.txt new file mode 100644 index 0000000000..db3295fb89 --- /dev/null +++ b/tla/allocation/traces/Inversion-bug5-deadlock.txt @@ -0,0 +1,495 @@ +TLC2 Version 2.19 of 08 August 2024 (rev: 5a47802) +Running breadth-first search Model-Checking with fp 55 and seed 946484794130481354 with 4 workers on 10 cores with 3641MB heap and 64MB offheap memory [pid: 7809] (Mac OS X 15.7.7 aarch64, Homebrew 26.0.2 x86_64, MSBDiskFPSet, DiskStateQueue). +Parsing file /Users/mebauer/realm/tla/allocation/MCDeferredAlloc.tla +Parsing file /Users/mebauer/realm/tla/allocation/DeferredAlloc.tla +Parsing file /private/tmp/claude-502/-Users-mebauer-realm-src-realm/e16dc9f9-47a4-47cb-a4d3-fd41b34c5bce/scratchpad/tlc-jtmp/Integers.tla +Parsing file /private/tmp/claude-502/-Users-mebauer-realm-src-realm/e16dc9f9-47a4-47cb-a4d3-fd41b34c5bce/scratchpad/tlc-jtmp/Sequences.tla +Parsing file /private/tmp/claude-502/-Users-mebauer-realm-src-realm/e16dc9f9-47a4-47cb-a4d3-fd41b34c5bce/scratchpad/tlc-jtmp/FiniteSets.tla +Parsing file /private/tmp/claude-502/-Users-mebauer-realm-src-realm/e16dc9f9-47a4-47cb-a4d3-fd41b34c5bce/scratchpad/tlc-jtmp/TLC.tla +Parsing file /private/tmp/claude-502/-Users-mebauer-realm-src-realm/e16dc9f9-47a4-47cb-a4d3-fd41b34c5bce/scratchpad/tlc-jtmp/Naturals.tla +Semantic processing of module Naturals +Semantic processing of module Integers +Semantic processing of module Sequences +Semantic processing of module FiniteSets +Semantic processing of module TLC +Semantic processing of module DeferredAlloc +Semantic processing of module MCDeferredAlloc +Starting... (2026-08-25 23:58:00) +Computing initial states... +Finished computing initial states: 1 distinct state generated at 2026-08-25 23:58:00. +Error: Deadlock reached. +Error: The behavior up to this point is: +State 1: +/\ destroyRequested = <> +/\ balC = <<"NONE", "NONE", "NONE">> +/\ balD = <<"NONE", "NONE", "NONE">> +/\ instOffset = <<-1, -1, -1>> +/\ preC = << [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ preD = << [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ seqCtr = 0 +/\ cur = << >> +/\ allocatedEver = {} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = <<>> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = << >> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<>> +/\ wasDeferred = {} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"UNFIRED", "UNFIRED", "UNFIRED">> +/\ instState = <<"UNREQUESTED", "UNREQUESTED", "UNREQUESTED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0, 0>> +/\ reqCap = <<0, 0, 0>> +/\ dupAlloc = FALSE +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 2: +/\ destroyRequested = <> +/\ balC = <<"NONE", "NONE", "NONE">> +/\ balD = <<"NONE", "NONE", "NONE">> +/\ instOffset = <<0, -1, -1>> +/\ preC = << [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ preD = << [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ seqCtr = 0 +/\ cur = <<[first |-> 0, size |-> 3]>> +/\ allocatedEver = {1} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = <<>> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = << >> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<>> +/\ wasDeferred = {} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"CLEAN", "UNFIRED", "UNFIRED">> +/\ instState = <<"ALLOCATED", "UNREQUESTED", "UNREQUESTED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0, 0>> +/\ reqCap = <<0, 0, 0>> +/\ dupAlloc = FALSE +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 3: +/\ destroyRequested = <> +/\ balC = <<"NONE", "PENDING", "NONE">> +/\ balD = <<"NONE", "NONE", "NONE">> +/\ instOffset = <<0, -1, -1>> +/\ preC = << [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> TRUE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ preD = << [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ seqCtr = 0 +/\ cur = <<[first |-> 0, size |-> 3]>> +/\ allocatedEver = {1} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = <<>> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = << >> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<>> +/\ wasDeferred = {} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"CLEAN", "UNFIRED", "UNFIRED">> +/\ instState = <<"ALLOCATED", "CREATE_PENDING", "UNREQUESTED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0, 0>> +/\ reqCap = <<0, 0, 0>> +/\ dupAlloc = FALSE +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 4: +/\ destroyRequested = <> +/\ balC = <<"NONE", "PENDING", "NONE">> +/\ balD = <<"NONE", "NONE", "NONE">> +/\ instOffset = <<0, -1, -1>> +/\ preC = << [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> TRUE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ preD = << [deps |-> {1, 2}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ seqCtr = 1 +/\ cur = <<[first |-> 0, size |-> 3]>> +/\ allocatedEver = {1} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = <<[inst |-> 1, isReady |-> FALSE, defNote |-> FALSE, seq |-> 1]>> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = << >> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<>> +/\ wasDeferred = {} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"CLEAN", "UNFIRED", "UNFIRED">> +/\ instState = <<"ALLOCATED", "CREATE_PENDING", "UNREQUESTED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0, 0>> +/\ reqCap = <<0, 0, 0>> +/\ dupAlloc = FALSE +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 5: +/\ destroyRequested = <> +/\ balC = <<"NONE", "PENDING", "NONE">> +/\ balD = <<"NONE", "NONE", "NONE">> +/\ instOffset = <<0, -1, -1>> +/\ preC = << [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> TRUE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ preD = << [deps |-> {1, 2}, ballistic |-> FALSE], + [deps |-> {2}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ seqCtr = 1 +/\ cur = <<[first |-> 0, size |-> 3]>> +/\ allocatedEver = {1} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = <<[inst |-> 1, isReady |-> FALSE, defNote |-> FALSE, seq |-> 1]>> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = << >> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<>> +/\ wasDeferred = {} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"CLEAN", "UNFIRED", "UNFIRED">> +/\ instState = <<"ALLOCATED", "CREATE_PENDING_DESTROY", "UNREQUESTED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0, 0>> +/\ reqCap = <<0, 0, 0>> +/\ dupAlloc = FALSE +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 6: +/\ destroyRequested = <> +/\ balC = <<"NONE", "FIRED", "NONE">> +/\ balD = <<"NONE", "NONE", "NONE">> +/\ instOffset = <<0, -1, -1>> +/\ preC = << [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> TRUE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ preD = << [deps |-> {1, 2}, ballistic |-> FALSE], + [deps |-> {2}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ seqCtr = 1 +/\ cur = <<[first |-> 0, size |-> 3]>> +/\ allocatedEver = {1} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = <<[inst |-> 1, isReady |-> FALSE, defNote |-> FALSE, seq |-> 1]>> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = << >> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<>> +/\ wasDeferred = {} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"CLEAN", "UNFIRED", "UNFIRED">> +/\ instState = <<"ALLOCATED", "CREATE_PENDING_DESTROY", "UNREQUESTED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0, 0>> +/\ reqCap = <<0, 0, 0>> +/\ dupAlloc = FALSE +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 7: +/\ destroyRequested = <> +/\ balC = <<"NONE", "FIRED", "NONE">> +/\ balD = <<"NONE", "NONE", "NONE">> +/\ instOffset = <<0, -2, -1>> +/\ preC = << [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> TRUE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ preD = << [deps |-> {1, 2}, ballistic |-> FALSE], + [deps |-> {2}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ seqCtr = 2 +/\ cur = <<[first |-> 0, size |-> 3]>> +/\ allocatedEver = {1} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = << [inst |-> 1, isReady |-> FALSE, defNote |-> FALSE, seq |-> 1], + [inst |-> 2, isReady |-> FALSE, defNote |-> FALSE, seq |-> 2] >> +/\ curFreed = {} +/\ failedVia = <<"NONE", "INSTANT", "NONE">> +/\ missingFree = FALSE +/\ rel = << >> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<>> +/\ wasDeferred = {} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"CLEAN", "POISONED", "UNFIRED">> +/\ instState = <<"ALLOCATED", "FAILED", "UNREQUESTED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0, 0>> +/\ reqCap = <<0, 0, 0>> +/\ dupAlloc = FALSE +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 8: +/\ destroyRequested = <> +/\ balC = <<"NONE", "FIRED", "PENDING">> +/\ balD = <<"NONE", "NONE", "NONE">> +/\ instOffset = <<0, -2, -1>> +/\ preC = << [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> TRUE], + [deps |-> {}, ballistic |-> TRUE] >> +/\ preD = << [deps |-> {1, 2}, ballistic |-> FALSE], + [deps |-> {2}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ seqCtr = 2 +/\ cur = <<[first |-> 0, size |-> 3]>> +/\ allocatedEver = {1} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = << [inst |-> 1, isReady |-> FALSE, defNote |-> FALSE, seq |-> 1], + [inst |-> 2, isReady |-> FALSE, defNote |-> FALSE, seq |-> 2] >> +/\ curFreed = {} +/\ failedVia = <<"NONE", "INSTANT", "NONE">> +/\ missingFree = FALSE +/\ rel = << >> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<>> +/\ wasDeferred = {} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"CLEAN", "POISONED", "UNFIRED">> +/\ instState = <<"ALLOCATED", "FAILED", "CREATE_PENDING">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0, 0>> +/\ reqCap = <<0, 0, 2>> +/\ dupAlloc = FALSE +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 9: +/\ destroyRequested = <> +/\ balC = <<"NONE", "FIRED", "PENDING">> +/\ balD = <<"NONE", "NONE", "NONE">> +/\ instOffset = <<0, -2, -1>> +/\ preC = << [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> TRUE], + [deps |-> {}, ballistic |-> TRUE] >> +/\ preD = << [deps |-> {1, 2}, ballistic |-> FALSE], + [deps |-> {2}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ seqCtr = 2 +/\ cur = <<[first |-> 0, size |-> 3]>> +/\ allocatedEver = {1} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = <<[inst |-> 1, isReady |-> FALSE, defNote |-> FALSE, seq |-> 1]>> +/\ curFreed = {} +/\ failedVia = <<"NONE", "INSTANT", "NONE">> +/\ missingFree = FALSE +/\ rel = << >> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<>> +/\ wasDeferred = {} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"CLEAN", "POISONED", "UNFIRED">> +/\ instState = <<"ALLOCATED", "FAILED", "CREATE_PENDING">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0, 0>> +/\ reqCap = <<0, 0, 2>> +/\ dupAlloc = FALSE +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 10: +/\ destroyRequested = <> +/\ balC = <<"NONE", "FIRED", "FIRED">> +/\ balD = <<"NONE", "NONE", "NONE">> +/\ instOffset = <<0, -2, -1>> +/\ preC = << [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> TRUE], + [deps |-> {}, ballistic |-> TRUE] >> +/\ preD = << [deps |-> {1, 2}, ballistic |-> FALSE], + [deps |-> {2}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ seqCtr = 2 +/\ cur = <<[first |-> 0, size |-> 3]>> +/\ allocatedEver = {1} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = <<[inst |-> 1, isReady |-> FALSE, defNote |-> FALSE, seq |-> 1]>> +/\ curFreed = {} +/\ failedVia = <<"NONE", "INSTANT", "NONE">> +/\ missingFree = FALSE +/\ rel = << >> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<>> +/\ wasDeferred = {} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"CLEAN", "POISONED", "UNFIRED">> +/\ instState = <<"ALLOCATED", "FAILED", "CREATE_PENDING">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0, 0>> +/\ reqCap = <<0, 0, 2>> +/\ dupAlloc = FALSE +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 11: +/\ destroyRequested = <> +/\ balC = <<"NONE", "FIRED", "FIRED">> +/\ balD = <<"NONE", "NONE", "NONE">> +/\ instOffset = <<0, -2, -1>> +/\ preC = << [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> TRUE], + [deps |-> {}, ballistic |-> TRUE] >> +/\ preD = << [deps |-> {1, 2}, ballistic |-> FALSE], + [deps |-> {2}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ seqCtr = 2 +/\ cur = <<[first |-> 0, size |-> 3]>> +/\ allocatedEver = {1} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = <<[inst |-> 1, isReady |-> FALSE, defNote |-> FALSE, seq |-> 1]>> +/\ curFreed = {} +/\ failedVia = <<"NONE", "INSTANT", "NONE">> +/\ missingFree = FALSE +/\ rel = <<[first |-> 0, size |-> 3]>> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<[inst |-> 3, size |-> 2, lastSeq |-> 2]>> +/\ wasDeferred = {3} +/\ structuralAssertFailed = FALSE +/\ fut = (3 :> [first |-> 0, size |-> 2]) +/\ eCreated = <<"CLEAN", "POISONED", "UNFIRED">> +/\ instState = <<"ALLOCATED", "FAILED", "ALLOC_DEFERRED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0, 0>> +/\ reqCap = <<0, 0, 0>> +/\ dupAlloc = FALSE +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 12: +/\ destroyRequested = <> +/\ balC = <<"NONE", "FIRED", "FIRED">> +/\ balD = <<"NONE", "NONE", "NONE">> +/\ instOffset = <<0, -2, -1>> +/\ preC = << [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> TRUE], + [deps |-> {}, ballistic |-> TRUE] >> +/\ preD = << [deps |-> {1, 2}, ballistic |-> FALSE], + [deps |-> {2}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ seqCtr = 2 +/\ cur = <<[first |-> 0, size |-> 3]>> +/\ allocatedEver = {1} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = <<>> +/\ curFreed = {} +/\ failedVia = <<"NONE", "INSTANT", "NONE">> +/\ missingFree = FALSE +/\ rel = <<[first |-> 0, size |-> 3]>> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<[inst |-> 3, size |-> 2, lastSeq |-> 2]>> +/\ wasDeferred = {3} +/\ structuralAssertFailed = FALSE +/\ fut = <<[first |-> 0, size |-> 3]>> +/\ eCreated = <<"CLEAN", "POISONED", "UNFIRED">> +/\ instState = <<"ALLOCATED", "FAILED", "ALLOC_DEFERRED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0, 0>> +/\ reqCap = <<0, 0, 0>> +/\ dupAlloc = FALSE +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 13: +/\ destroyRequested = <> +/\ balC = <<"NONE", "FIRED", "FIRED">> +/\ balD = <<"NONE", "NONE", "NONE">> +/\ instOffset = <<0, -2, -1>> +/\ preC = << [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> TRUE], + [deps |-> {}, ballistic |-> TRUE] >> +/\ preD = << [deps |-> {1, 2}, ballistic |-> FALSE], + [deps |-> {2}, ballistic |-> FALSE], + [deps |-> {3}, ballistic |-> FALSE] >> +/\ seqCtr = 3 +/\ cur = <<[first |-> 0, size |-> 3]>> +/\ allocatedEver = {1} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = <<[inst |-> 3, isReady |-> FALSE, defNote |-> FALSE, seq |-> 3]>> +/\ curFreed = {} +/\ failedVia = <<"NONE", "INSTANT", "NONE">> +/\ missingFree = FALSE +/\ rel = <<[first |-> 0, size |-> 3]>> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<[inst |-> 3, size |-> 2, lastSeq |-> 2]>> +/\ wasDeferred = {3} +/\ structuralAssertFailed = FALSE +/\ fut = <<[first |-> 0, size |-> 3]>> +/\ eCreated = <<"CLEAN", "POISONED", "UNFIRED">> +/\ instState = <<"ALLOCATED", "FAILED", "ALLOC_DEFERRED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0, 0>> +/\ reqCap = <<0, 0, 0>> +/\ dupAlloc = FALSE +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +469 states generated, 254 distinct states found, 7 states left on queue. +The depth of the complete state graph search is 14. +The average outdegree of the complete state graph is 1 (minimum is 0, the maximum 4 and the 95th percentile is 3). +Finished in 00s at (2026-08-25 23:58:01) diff --git a/tla/allocation/traces/Liveness-bug1.txt b/tla/allocation/traces/Liveness-bug1.txt new file mode 100644 index 0000000000..93c57d4409 --- /dev/null +++ b/tla/allocation/traces/Liveness-bug1.txt @@ -0,0 +1,281 @@ +TLC2 Version 2.19 of 08 August 2024 (rev: 5a47802) +Running breadth-first search Model-Checking with fp 54 and seed -3212275733632134920 with 8 workers on 10 cores with 5461MB heap and 64MB offheap memory [pid: 94301] (Mac OS X 15.7.7 aarch64, Homebrew 26.0.2 x86_64, MSBDiskFPSet, DiskStateQueue). +Parsing file /Users/mebauer/realm/tla/allocation/MCDeferredAlloc.tla +Parsing file /Users/mebauer/realm/tla/allocation/DeferredAlloc.tla +Parsing file /private/tmp/claude-502/-Users-mebauer-realm-src-realm/e16dc9f9-47a4-47cb-a4d3-fd41b34c5bce/scratchpad/jtmp/Integers.tla +Parsing file /private/tmp/claude-502/-Users-mebauer-realm-src-realm/e16dc9f9-47a4-47cb-a4d3-fd41b34c5bce/scratchpad/jtmp/Sequences.tla +Parsing file /private/tmp/claude-502/-Users-mebauer-realm-src-realm/e16dc9f9-47a4-47cb-a4d3-fd41b34c5bce/scratchpad/jtmp/FiniteSets.tla +Parsing file /private/tmp/claude-502/-Users-mebauer-realm-src-realm/e16dc9f9-47a4-47cb-a4d3-fd41b34c5bce/scratchpad/jtmp/TLC.tla +Parsing file /private/tmp/claude-502/-Users-mebauer-realm-src-realm/e16dc9f9-47a4-47cb-a4d3-fd41b34c5bce/scratchpad/jtmp/Naturals.tla +Semantic processing of module Naturals +Semantic processing of module Integers +Semantic processing of module Sequences +Semantic processing of module FiniteSets +Semantic processing of module TLC +Semantic processing of module DeferredAlloc +Semantic processing of module MCDeferredAlloc +Starting... (2026-08-25 14:36:36) +Warning: Declaring state or action constraints during liveness checking is dangerous: Please read section 14.3.5 on page 247 of Specifying Systems (https://lamport.azurewebsites.net/tla/book.html) and optionally the discussion at https://discuss.tlapl.us/msg00994.html for more details. +(Use the -nowarning option to disable this warning.) +Implied-temporal checking--satisfiability problem has 2 branches. +Computing initial states... +Finished computing initial states: 1 distinct state generated at 2026-08-25 14:36:36. +Checking 2 branches of temporal properties for the current state space with 7062 total distinct states at (2026-08-25 14:36:39) +Error: Temporal properties were violated. + +Error: The following behavior constitutes a counter-example: + +State 1: +/\ destroyRequested = <> +/\ balC = <<"NONE", "NONE">> +/\ balD = <<"NONE", "NONE">> +/\ instOffset = <<-1, -1>> +/\ preC = <<[deps |-> {}, ballistic |-> FALSE], [deps |-> {}, ballistic |-> FALSE]>> +/\ preD = <<[deps |-> {}, ballistic |-> FALSE], [deps |-> {}, ballistic |-> FALSE]>> +/\ seqCtr = 0 +/\ cur = << >> +/\ allocatedEver = {} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = <<>> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = << >> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<>> +/\ wasDeferred = {} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"UNFIRED", "UNFIRED">> +/\ instState = <<"UNREQUESTED", "UNREQUESTED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0>> +/\ dupAlloc = FALSE +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 2: +/\ destroyRequested = <> +/\ balC = <<"NONE", "NONE">> +/\ balD = <<"NONE", "NONE">> +/\ instOffset = <<-1, 0>> +/\ preC = <<[deps |-> {}, ballistic |-> FALSE], [deps |-> {}, ballistic |-> FALSE]>> +/\ preD = <<[deps |-> {}, ballistic |-> FALSE], [deps |-> {}, ballistic |-> FALSE]>> +/\ seqCtr = 0 +/\ cur = (2 :> [first |-> 0, size |-> 3]) +/\ allocatedEver = {2} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = <<>> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = << >> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<>> +/\ wasDeferred = {} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"UNFIRED", "CLEAN">> +/\ instState = <<"UNREQUESTED", "ALLOCATED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0>> +/\ dupAlloc = FALSE +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 3: +/\ destroyRequested = <> +/\ balC = <<"PENDING", "NONE">> +/\ balD = <<"NONE", "NONE">> +/\ instOffset = <<-1, 0>> +/\ preC = <<[deps |-> {}, ballistic |-> TRUE], [deps |-> {}, ballistic |-> FALSE]>> +/\ preD = <<[deps |-> {}, ballistic |-> FALSE], [deps |-> {}, ballistic |-> FALSE]>> +/\ seqCtr = 0 +/\ cur = (2 :> [first |-> 0, size |-> 3]) +/\ allocatedEver = {2} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = <<>> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = << >> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<>> +/\ wasDeferred = {} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"UNFIRED", "CLEAN">> +/\ instState = <<"CREATE_PENDING", "ALLOCATED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0>> +/\ dupAlloc = FALSE +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 4: +/\ destroyRequested = <> +/\ balC = <<"PENDING", "NONE">> +/\ balD = <<"NONE", "NONE">> +/\ instOffset = <<-1, 0>> +/\ preC = <<[deps |-> {}, ballistic |-> TRUE], [deps |-> {}, ballistic |-> FALSE]>> +/\ preD = <<[deps |-> {}, ballistic |-> FALSE], [deps |-> {1, 2}, ballistic |-> FALSE]>> +/\ seqCtr = 1 +/\ cur = (2 :> [first |-> 0, size |-> 3]) +/\ allocatedEver = {2} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = <<[inst |-> 2, isReady |-> FALSE, defNote |-> FALSE, seq |-> 1]>> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = << >> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<>> +/\ wasDeferred = {} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"UNFIRED", "CLEAN">> +/\ instState = <<"CREATE_PENDING", "ALLOCATED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0>> +/\ dupAlloc = FALSE +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 5: +/\ destroyRequested = <> +/\ balC = <<"FIRED", "NONE">> +/\ balD = <<"NONE", "NONE">> +/\ instOffset = <<-1, 0>> +/\ preC = <<[deps |-> {}, ballistic |-> TRUE], [deps |-> {}, ballistic |-> FALSE]>> +/\ preD = <<[deps |-> {}, ballistic |-> FALSE], [deps |-> {1, 2}, ballistic |-> FALSE]>> +/\ seqCtr = 1 +/\ cur = (2 :> [first |-> 0, size |-> 3]) +/\ allocatedEver = {2} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = <<[inst |-> 2, isReady |-> FALSE, defNote |-> FALSE, seq |-> 1]>> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = << >> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<>> +/\ wasDeferred = {} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"UNFIRED", "CLEAN">> +/\ instState = <<"CREATE_PENDING", "ALLOCATED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0>> +/\ dupAlloc = FALSE +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 6: +/\ destroyRequested = <> +/\ balC = <<"FIRED", "NONE">> +/\ balD = <<"NONE", "NONE">> +/\ instOffset = <<-1, 0>> +/\ preC = <<[deps |-> {}, ballistic |-> TRUE], [deps |-> {}, ballistic |-> FALSE]>> +/\ preD = <<[deps |-> {}, ballistic |-> FALSE], [deps |-> {1, 2}, ballistic |-> FALSE]>> +/\ seqCtr = 1 +/\ cur = (2 :> [first |-> 0, size |-> 3]) +/\ allocatedEver = {2} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = <<[inst |-> 2, isReady |-> FALSE, defNote |-> FALSE, seq |-> 1]>> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = (2 :> [first |-> 0, size |-> 3]) +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<[inst |-> 1, size |-> 3, lastSeq |-> 1]>> +/\ wasDeferred = {1} +/\ structuralAssertFailed = FALSE +/\ fut = <<[first |-> 0, size |-> 3]>> +/\ eCreated = <<"UNFIRED", "CLEAN">> +/\ instState = <<"ALLOC_DEFERRED", "ALLOCATED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0>> +/\ dupAlloc = FALSE +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 7: +/\ destroyRequested = <> +/\ balC = <<"FIRED", "NONE">> +/\ balD = <<"PENDING", "NONE">> +/\ instOffset = <<-1, 0>> +/\ preC = <<[deps |-> {}, ballistic |-> TRUE], [deps |-> {}, ballistic |-> FALSE]>> +/\ preD = <<[deps |-> {1}, ballistic |-> TRUE], [deps |-> {1, 2}, ballistic |-> FALSE]>> +/\ seqCtr = 2 +/\ cur = (2 :> [first |-> 0, size |-> 3]) +/\ allocatedEver = {2} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = << [inst |-> 2, isReady |-> FALSE, defNote |-> FALSE, seq |-> 1], + [inst |-> 1, isReady |-> FALSE, defNote |-> FALSE, seq |-> 2] >> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = (2 :> [first |-> 0, size |-> 3]) +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<[inst |-> 1, size |-> 3, lastSeq |-> 1]>> +/\ wasDeferred = {1} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"UNFIRED", "CLEAN">> +/\ instState = <<"ALLOC_DEFERRED", "ALLOCATED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0>> +/\ dupAlloc = FALSE +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 8: +/\ destroyRequested = <> +/\ balC = <<"FIRED", "NONE">> +/\ balD = <<"FIRED", "NONE">> +/\ instOffset = <<-1, 0>> +/\ preC = <<[deps |-> {}, ballistic |-> TRUE], [deps |-> {}, ballistic |-> FALSE]>> +/\ preD = <<[deps |-> {1}, ballistic |-> TRUE], [deps |-> {1, 2}, ballistic |-> FALSE]>> +/\ seqCtr = 2 +/\ cur = (2 :> [first |-> 0, size |-> 3]) +/\ allocatedEver = {2} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = << [inst |-> 2, isReady |-> FALSE, defNote |-> FALSE, seq |-> 1], + [inst |-> 1, isReady |-> FALSE, defNote |-> FALSE, seq |-> 2] >> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = (2 :> [first |-> 0, size |-> 3]) +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<[inst |-> 1, size |-> 3, lastSeq |-> 1]>> +/\ wasDeferred = {1} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"UNFIRED", "CLEAN">> +/\ instState = <<"ALLOC_DEFERRED", "ALLOCATED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0>> +/\ dupAlloc = FALSE +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 9: Stuttering +Finished checking temporal properties in 00s at 2026-08-25 14:36:39 +7273 states generated, 3531 distinct states found, 950 states left on queue. +Finished in 03s at (2026-08-25 14:36:40) diff --git a/tla/allocation/traces/LivenessNoCross-pass.txt b/tla/allocation/traces/LivenessNoCross-pass.txt new file mode 100644 index 0000000000..b6ddb39c56 --- /dev/null +++ b/tla/allocation/traces/LivenessNoCross-pass.txt @@ -0,0 +1,36 @@ +TLC2 Version 2.19 of 08 August 2024 (rev: 5a47802) +Running breadth-first search Model-Checking with fp 115 and seed -3870028390920141770 with 4 workers on 10 cores with 3641MB heap and 64MB offheap memory [pid: 94317] (Mac OS X 15.7.7 aarch64, Homebrew 26.0.2 x86_64, MSBDiskFPSet, DiskStateQueue). +Parsing file /Users/mebauer/realm/tla/allocation/MCDeferredAlloc.tla +Parsing file /Users/mebauer/realm/tla/allocation/DeferredAlloc.tla +Parsing file /private/tmp/claude-502/-Users-mebauer-realm-src-realm/e16dc9f9-47a4-47cb-a4d3-fd41b34c5bce/scratchpad/jtmp/Integers.tla +Parsing file /private/tmp/claude-502/-Users-mebauer-realm-src-realm/e16dc9f9-47a4-47cb-a4d3-fd41b34c5bce/scratchpad/jtmp/Sequences.tla +Parsing file /private/tmp/claude-502/-Users-mebauer-realm-src-realm/e16dc9f9-47a4-47cb-a4d3-fd41b34c5bce/scratchpad/jtmp/FiniteSets.tla +Parsing file /private/tmp/claude-502/-Users-mebauer-realm-src-realm/e16dc9f9-47a4-47cb-a4d3-fd41b34c5bce/scratchpad/jtmp/TLC.tla +Parsing file /private/tmp/claude-502/-Users-mebauer-realm-src-realm/e16dc9f9-47a4-47cb-a4d3-fd41b34c5bce/scratchpad/jtmp/Naturals.tla +Semantic processing of module Naturals +Semantic processing of module Integers +Semantic processing of module Sequences +Semantic processing of module FiniteSets +Semantic processing of module TLC +Semantic processing of module DeferredAlloc +Semantic processing of module MCDeferredAlloc +Starting... (2026-08-25 14:36:49) +Warning: Declaring state or action constraints during liveness checking is dangerous: Please read section 14.3.5 on page 247 of Specifying Systems (https://lamport.azurewebsites.net/tla/book.html) and optionally the discussion at https://discuss.tlapl.us/msg00994.html for more details. +(Use the -nowarning option to disable this warning.) +Implied-temporal checking--satisfiability problem has 2 branches. +Computing initial states... +Finished computing initial states: 1 distinct state generated at 2026-08-25 14:36:49. +Checking 2 branches of temporal properties for the current state space with 3130 total distinct states at (2026-08-25 14:36:52) +Finished checking temporal properties in 00s at 2026-08-25 14:36:52 +Progress(10) at 2026-08-25 14:36:52: 3,013 states generated (3,013 s/min), 1,566 distinct states found (1,566 ds/min), 305 states left on queue. +Progress(13) at 2026-08-25 14:36:53: 3,929 states generated, 1,987 distinct states found, 0 states left on queue. +Checking 2 branches of temporal properties for the complete state space with 3974 total distinct states at (2026-08-25 14:36:53) +Finished checking temporal properties in 00s at 2026-08-25 14:36:53 +Model checking completed. No error has been found. + Estimates of the probability that TLC did not check all reachable states + because two distinct states had the same fingerprint: + calculated (optimistic): val = 2.1E-13 +3929 states generated, 1987 distinct states found, 0 states left on queue. +The depth of the complete state graph search is 13. +The average outdegree of the complete state graph is 1 (minimum is 0, the maximum 7 and the 95th percentile is 3). +Finished in 04s at (2026-08-25 14:36:53) diff --git a/tla/allocation/traces/Safety.trace.txt b/tla/allocation/traces/Safety.trace.txt new file mode 100644 index 0000000000..be72f5d5e9 --- /dev/null +++ b/tla/allocation/traces/Safety.trace.txt @@ -0,0 +1,32 @@ +TLC2 Version 2.19 of 08 August 2024 (rev: 5a47802) +Running breadth-first search Model-Checking with fp 27 and seed 5527827037521654291 with 8 workers on 10 cores with 5461MB heap and 64MB offheap memory [pid: 92957] (Mac OS X 15.7.7 aarch64, Homebrew 26.0.2 x86_64, MSBDiskFPSet, DiskStateQueue). +Parsing file /Users/mebauer/realm/tla/allocation/MCDeferredAlloc.tla +Parsing file /Users/mebauer/realm/tla/allocation/DeferredAlloc.tla +Parsing file /Users/mebauer/realm/tla/allocation/jtmp/Integers.tla +Parsing file /Users/mebauer/realm/tla/allocation/jtmp/Sequences.tla +Parsing file /Users/mebauer/realm/tla/allocation/jtmp/FiniteSets.tla +Parsing file /Users/mebauer/realm/tla/allocation/jtmp/TLC.tla +Parsing file /Users/mebauer/realm/tla/allocation/jtmp/Naturals.tla +Semantic processing of module Naturals +Semantic processing of module Integers +Semantic processing of module Sequences +Semantic processing of module FiniteSets +Semantic processing of module TLC +Semantic processing of module DeferredAlloc +Semantic processing of module MCDeferredAlloc +Starting... (2026-08-25 13:49:37) +Computing initial states... +Finished computing initial states: 1 distinct state generated at 2026-08-25 13:49:37. +Progress(7) at 2026-08-25 13:49:40: 1,121,969 states generated (1,121,969 s/min), 601,344 distinct states found (601,344 ds/min), 576,873 states left on queue. +Progress(8) at 2026-08-25 13:50:40: 34,507,298 states generated (33,385,329 s/min), 14,274,924 distinct states found (13,673,580 ds/min), 13,411,312 states left on queue. +Progress(9) at 2026-08-25 13:51:40: 68,473,784 states generated (33,966,486 s/min), 27,806,972 distinct states found (13,532,048 ds/min), 26,030,379 states left on queue. +Progress(9) at 2026-08-25 13:52:40: 102,380,964 states generated (33,907,180 s/min), 39,904,992 distinct states found (12,098,020 ds/min), 37,034,642 states left on queue. +Progress(9) at 2026-08-25 13:53:40: 141,119,071 states generated (38,738,107 s/min), 53,505,286 distinct states found (13,600,294 ds/min), 49,388,241 states left on queue. +Progress(9) at 2026-08-25 13:54:40: 173,930,987 states generated (32,811,916 s/min), 65,758,823 distinct states found (12,253,537 ds/min), 60,633,437 states left on queue. +Progress(9) at 2026-08-25 13:55:40: 195,321,642 states generated (21,390,655 s/min), 73,571,211 distinct states found (7,812,388 ds/min), 67,881,239 states left on queue. +Progress(9) at 2026-08-25 13:56:40: 220,452,735 states generated (25,131,093 s/min), 81,849,419 distinct states found (8,278,208 ds/min), 75,319,280 states left on queue. +Progress(9) at 2026-08-25 13:57:40: 244,756,283 states generated (24,303,548 s/min), 89,350,486 distinct states found (7,501,067 ds/min), 81,909,940 states left on queue. +Progress(9) at 2026-08-25 13:58:40: 263,541,213 states generated (18,784,930 s/min), 95,744,015 distinct states found (6,393,529 ds/min), 87,673,627 states left on queue. +Progress(9) at 2026-08-25 13:59:40: 283,682,496 states generated (20,141,283 s/min), 102,716,060 distinct states found (6,972,045 ds/min), 93,890,479 states left on queue. +Progress(9) at 2026-08-25 14:00:41: 308,294,093 states generated (24,611,597 s/min), 110,686,883 distinct states found (7,970,823 ds/min), 100,916,654 states left on queue. +Progress(9) at 2026-08-25 14:01:41: 331,706,740 states generated (23,412,647 s/min), 118,410,450 distinct states found (7,723,567 ds/min), 107,783,417 states left on queue. diff --git a/tla/allocation/traces/SafetyMini.trace.txt b/tla/allocation/traces/SafetyMini.trace.txt new file mode 100644 index 0000000000..ac3646c696 --- /dev/null +++ b/tla/allocation/traces/SafetyMini.trace.txt @@ -0,0 +1,338 @@ +TLC2 Version 2.19 of 08 August 2024 (rev: 5a47802) +Running breadth-first search Model-Checking with fp 54 and seed -7820568895716415763 with 8 workers on 10 cores with 5461MB heap and 64MB offheap memory [pid: 93283] (Mac OS X 15.7.7 aarch64, Homebrew 26.0.2 x86_64, MSBDiskFPSet, DiskStateQueue). +Parsing file /Users/mebauer/realm/tla/allocation/MCDeferredAlloc.tla +Parsing file /Users/mebauer/realm/tla/allocation/DeferredAlloc.tla +Parsing file /Users/mebauer/realm/tla/allocation/jtmp/Integers.tla +Parsing file /Users/mebauer/realm/tla/allocation/jtmp/Sequences.tla +Parsing file /Users/mebauer/realm/tla/allocation/jtmp/FiniteSets.tla +Parsing file /Users/mebauer/realm/tla/allocation/jtmp/TLC.tla +Parsing file /Users/mebauer/realm/tla/allocation/jtmp/Naturals.tla +Semantic processing of module Naturals +Semantic processing of module Integers +Semantic processing of module Sequences +Semantic processing of module FiniteSets +Semantic processing of module TLC +Semantic processing of module DeferredAlloc +Semantic processing of module MCDeferredAlloc +Starting... (2026-08-25 14:02:47) +Computing initial states... +Finished computing initial states: 1 distinct state generated at 2026-08-25 14:02:48. +Progress(8) at 2026-08-25 14:02:51: 1,208,698 states generated (1,208,698 s/min), 476,526 distinct states found (476,526 ds/min), 351,381 states left on queue. +Error: Invariant INV_NoReadyWhenNoPendingAllocs is violated. +Error: The behavior up to this point is: +State 1: +/\ destroyRequested = <> +/\ balC = <<"NONE", "NONE", "NONE">> +/\ balD = <<"NONE", "NONE", "NONE">> +/\ instOffset = <<-1, -1, -1>> +/\ preC = << [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ preD = << [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ seqCtr = 0 +/\ cur = << >> +/\ allocatedEver = {} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = <<>> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = << >> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<>> +/\ wasDeferred = {} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"UNFIRED", "UNFIRED", "UNFIRED">> +/\ instState = <<"UNREQUESTED", "UNREQUESTED", "UNREQUESTED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0, 0>> +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 2: +/\ destroyRequested = <> +/\ balC = <<"NONE", "NONE", "NONE">> +/\ balD = <<"NONE", "NONE", "NONE">> +/\ instOffset = <<0, -1, -1>> +/\ preC = << [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ preD = << [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ seqCtr = 0 +/\ cur = <<[first |-> 0, size |-> 2]>> +/\ allocatedEver = {1} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = <<>> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = << >> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<>> +/\ wasDeferred = {} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"CLEAN", "UNFIRED", "UNFIRED">> +/\ instState = <<"ALLOCATED", "UNREQUESTED", "UNREQUESTED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0, 0>> +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 3: +/\ destroyRequested = <> +/\ balC = <<"NONE", "NONE", "NONE">> +/\ balD = <<"NONE", "NONE", "NONE">> +/\ instOffset = <<0, -1, 2>> +/\ preC = << [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ preD = << [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ seqCtr = 0 +/\ cur = (1 :> [first |-> 0, size |-> 2] @@ 3 :> [first |-> 2, size |-> 1]) +/\ allocatedEver = {1, 3} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = <<>> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = << >> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<>> +/\ wasDeferred = {} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"CLEAN", "UNFIRED", "CLEAN">> +/\ instState = <<"ALLOCATED", "UNREQUESTED", "ALLOCATED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0, 0>> +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 4: +/\ destroyRequested = <> +/\ balC = <<"NONE", "NONE", "NONE">> +/\ balD = <<"PENDING", "NONE", "NONE">> +/\ instOffset = <<0, -1, 2>> +/\ preC = << [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ preD = << [deps |-> {1}, ballistic |-> TRUE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ seqCtr = 1 +/\ cur = (1 :> [first |-> 0, size |-> 2] @@ 3 :> [first |-> 2, size |-> 1]) +/\ allocatedEver = {1, 3} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = <<[inst |-> 1, isReady |-> FALSE, defNote |-> FALSE, seq |-> 1]>> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = << >> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<>> +/\ wasDeferred = {} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"CLEAN", "UNFIRED", "CLEAN">> +/\ instState = <<"ALLOCATED", "UNREQUESTED", "ALLOCATED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0, 0>> +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 5: +/\ destroyRequested = <> +/\ balC = <<"NONE", "NONE", "NONE">> +/\ balD = <<"PENDING", "NONE", "NONE">> +/\ instOffset = <<0, -1, 2>> +/\ preC = << [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ preD = << [deps |-> {1}, ballistic |-> TRUE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ seqCtr = 1 +/\ cur = (1 :> [first |-> 0, size |-> 2] @@ 3 :> [first |-> 2, size |-> 1]) +/\ allocatedEver = {1, 3} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = <<[inst |-> 1, isReady |-> FALSE, defNote |-> FALSE, seq |-> 1]>> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = (1 :> [first |-> 0, size |-> 2] @@ 3 :> [first |-> 2, size |-> 1]) +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<[inst |-> 2, size |-> 2, lastSeq |-> 1]>> +/\ wasDeferred = {2} +/\ structuralAssertFailed = FALSE +/\ fut = (2 :> [first |-> 0, size |-> 2] @@ 3 :> [first |-> 2, size |-> 1]) +/\ eCreated = <<"CLEAN", "UNFIRED", "CLEAN">> +/\ instState = <<"ALLOCATED", "ALLOC_DEFERRED", "ALLOCATED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0, 0>> +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 6: +/\ destroyRequested = <> +/\ balC = <<"NONE", "NONE", "NONE">> +/\ balD = <<"PENDING", "NONE", "NONE">> +/\ instOffset = <<0, -1, 2>> +/\ preC = << [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ preD = << [deps |-> {1}, ballistic |-> TRUE], + [deps |-> {2}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ seqCtr = 2 +/\ cur = (1 :> [first |-> 0, size |-> 2] @@ 3 :> [first |-> 2, size |-> 1]) +/\ allocatedEver = {1, 3} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = << [inst |-> 1, isReady |-> FALSE, defNote |-> FALSE, seq |-> 1], + [inst |-> 2, isReady |-> FALSE, defNote |-> FALSE, seq |-> 2] >> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = (1 :> [first |-> 0, size |-> 2] @@ 3 :> [first |-> 2, size |-> 1]) +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<[inst |-> 2, size |-> 2, lastSeq |-> 1]>> +/\ wasDeferred = {2} +/\ structuralAssertFailed = FALSE +/\ fut = (3 :> [first |-> 2, size |-> 1]) +/\ eCreated = <<"CLEAN", "UNFIRED", "CLEAN">> +/\ instState = <<"ALLOCATED", "ALLOC_DEFERRED", "ALLOCATED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0, 0>> +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 7: +/\ destroyRequested = <> +/\ balC = <<"NONE", "NONE", "NONE">> +/\ balD = <<"PENDING", "NONE", "NONE">> +/\ instOffset = <<0, -1, 2>> +/\ preC = << [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ preD = << [deps |-> {1}, ballistic |-> TRUE], + [deps |-> {2}, ballistic |-> FALSE], + [deps |-> {3}, ballistic |-> FALSE] >> +/\ seqCtr = 3 +/\ cur = (1 :> [first |-> 0, size |-> 2] @@ 3 :> [first |-> 2, size |-> 1]) +/\ allocatedEver = {1, 3} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = << [inst |-> 1, isReady |-> FALSE, defNote |-> FALSE, seq |-> 1], + [inst |-> 2, isReady |-> FALSE, defNote |-> FALSE, seq |-> 2], + [inst |-> 3, isReady |-> TRUE, defNote |-> TRUE, seq |-> 3] >> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = <<[first |-> 0, size |-> 2]>> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<[inst |-> 2, size |-> 2, lastSeq |-> 1]>> +/\ wasDeferred = {2} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"CLEAN", "UNFIRED", "CLEAN">> +/\ instState = <<"ALLOCATED", "ALLOC_DEFERRED", "ALLOCATED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0, 0>> +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 8: +/\ destroyRequested = <> +/\ balC = <<"NONE", "NONE", "NONE">> +/\ balD = <<"FIRED", "NONE", "NONE">> +/\ instOffset = <<0, -1, 2>> +/\ preC = << [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ preD = << [deps |-> {1}, ballistic |-> TRUE], + [deps |-> {2}, ballistic |-> FALSE], + [deps |-> {3}, ballistic |-> FALSE] >> +/\ seqCtr = 3 +/\ cur = (1 :> [first |-> 0, size |-> 2] @@ 3 :> [first |-> 2, size |-> 1]) +/\ allocatedEver = {1, 3} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = << [inst |-> 1, isReady |-> FALSE, defNote |-> FALSE, seq |-> 1], + [inst |-> 2, isReady |-> FALSE, defNote |-> FALSE, seq |-> 2], + [inst |-> 3, isReady |-> TRUE, defNote |-> TRUE, seq |-> 3] >> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = <<[first |-> 0, size |-> 2]>> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<[inst |-> 2, size |-> 2, lastSeq |-> 1]>> +/\ wasDeferred = {2} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"CLEAN", "UNFIRED", "CLEAN">> +/\ instState = <<"ALLOCATED", "ALLOC_DEFERRED", "ALLOCATED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0, 0>> +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 9: +/\ destroyRequested = <> +/\ balC = <<"NONE", "NONE", "NONE">> +/\ balD = <<"FIRED", "NONE", "NONE">> +/\ instOffset = <<0, 0, 2>> +/\ preC = << [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE], + [deps |-> {}, ballistic |-> FALSE] >> +/\ preD = << [deps |-> {1}, ballistic |-> TRUE], + [deps |-> {2}, ballistic |-> FALSE], + [deps |-> {3}, ballistic |-> FALSE] >> +/\ seqCtr = 3 +/\ cur = (2 :> [first |-> 0, size |-> 2] @@ 3 :> [first |-> 2, size |-> 1]) +/\ allocatedEver = {1, 2, 3} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = << [inst |-> 2, isReady |-> FALSE, defNote |-> FALSE, seq |-> 2], + [inst |-> 3, isReady |-> TRUE, defNote |-> TRUE, seq |-> 3] >> +/\ curFreed = {1} +/\ failedVia = <<"NONE", "NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = << >> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<>> +/\ wasDeferred = {2} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"CLEAN", "CLEAN", "CLEAN">> +/\ instState = <<"DESTROYED", "ALLOCATED", "ALLOCATED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<1, 0, 0>> +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +5551560 states generated, 2027597 distinct states found, 1167374 states left on queue. +The depth of the complete state graph search is 10. +The average outdegree of the complete state graph is 2 (minimum is 0, the maximum 27 and the 95th percentile is 8). +Finished in 10s at (2026-08-25 14:02:58) diff --git a/tla/allocation/traces/Smoke-run1.txt b/tla/allocation/traces/Smoke-run1.txt new file mode 100644 index 0000000000..c2b9c1a8bf --- /dev/null +++ b/tla/allocation/traces/Smoke-run1.txt @@ -0,0 +1,236 @@ +TLC2 Version 2.19 of 08 August 2024 (rev: 5a47802) +Running breadth-first search Model-Checking with fp 116 and seed 6032899637897111947 with 4 workers on 10 cores with 3641MB heap and 64MB offheap memory [pid: 92939] (Mac OS X 15.7.7 aarch64, Homebrew 26.0.2 x86_64, MSBDiskFPSet, DiskStateQueue). +Parsing file /Users/mebauer/realm/tla/allocation/MCDeferredAlloc.tla +Parsing file /Users/mebauer/realm/tla/allocation/DeferredAlloc.tla +Parsing file /Users/mebauer/realm/tla/allocation/jtmp/Integers.tla +Parsing file /Users/mebauer/realm/tla/allocation/jtmp/Sequences.tla +Parsing file /Users/mebauer/realm/tla/allocation/jtmp/FiniteSets.tla +Parsing file /Users/mebauer/realm/tla/allocation/jtmp/TLC.tla +Parsing file /Users/mebauer/realm/tla/allocation/jtmp/Naturals.tla +Semantic processing of module Naturals +Semantic processing of module Integers +Semantic processing of module Sequences +Semantic processing of module FiniteSets +Semantic processing of module TLC +Semantic processing of module DeferredAlloc +Semantic processing of module MCDeferredAlloc +Starting... (2026-08-25 13:49:09) +Computing initial states... +Finished computing initial states: 1 distinct state generated at 2026-08-25 13:49:09. +Error: Deadlock reached. +Error: The behavior up to this point is: +State 1: +/\ destroyRequested = <> +/\ balC = <<"NONE", "NONE">> +/\ balD = <<"NONE", "NONE">> +/\ instOffset = <<-1, -1>> +/\ preC = <<[deps |-> {}, ballistic |-> FALSE], [deps |-> {}, ballistic |-> FALSE]>> +/\ preD = <<[deps |-> {}, ballistic |-> FALSE], [deps |-> {}, ballistic |-> FALSE]>> +/\ seqCtr = 0 +/\ cur = << >> +/\ allocatedEver = {} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = <<>> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = << >> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<>> +/\ wasDeferred = {} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"UNFIRED", "UNFIRED">> +/\ instState = <<"UNREQUESTED", "UNREQUESTED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0>> +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 2: +/\ destroyRequested = <> +/\ balC = <<"NONE", "NONE">> +/\ balD = <<"NONE", "NONE">> +/\ instOffset = <<0, -1>> +/\ preC = <<[deps |-> {}, ballistic |-> FALSE], [deps |-> {}, ballistic |-> FALSE]>> +/\ preD = <<[deps |-> {}, ballistic |-> FALSE], [deps |-> {}, ballistic |-> FALSE]>> +/\ seqCtr = 0 +/\ cur = <<[first |-> 0, size |-> 2]>> +/\ allocatedEver = {1} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = <<>> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = << >> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<>> +/\ wasDeferred = {} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"CLEAN", "UNFIRED">> +/\ instState = <<"ALLOCATED", "UNREQUESTED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0>> +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 3: +/\ destroyRequested = <> +/\ balC = <<"NONE", "PENDING">> +/\ balD = <<"NONE", "NONE">> +/\ instOffset = <<0, -1>> +/\ preC = <<[deps |-> {}, ballistic |-> FALSE], [deps |-> {}, ballistic |-> TRUE]>> +/\ preD = <<[deps |-> {}, ballistic |-> FALSE], [deps |-> {}, ballistic |-> FALSE]>> +/\ seqCtr = 0 +/\ cur = <<[first |-> 0, size |-> 2]>> +/\ allocatedEver = {1} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = <<>> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = << >> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<>> +/\ wasDeferred = {} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"CLEAN", "UNFIRED">> +/\ instState = <<"ALLOCATED", "CREATE_PENDING">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0>> +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 4: +/\ destroyRequested = <> +/\ balC = <<"NONE", "PENDING">> +/\ balD = <<"NONE", "NONE">> +/\ instOffset = <<0, -1>> +/\ preC = <<[deps |-> {}, ballistic |-> FALSE], [deps |-> {}, ballistic |-> TRUE]>> +/\ preD = <<[deps |-> {1, 2}, ballistic |-> FALSE], [deps |-> {}, ballistic |-> FALSE]>> +/\ seqCtr = 1 +/\ cur = <<[first |-> 0, size |-> 2]>> +/\ allocatedEver = {1} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = <<[inst |-> 1, isReady |-> FALSE, defNote |-> FALSE, seq |-> 1]>> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = << >> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<>> +/\ wasDeferred = {} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"CLEAN", "UNFIRED">> +/\ instState = <<"ALLOCATED", "CREATE_PENDING">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0>> +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 5: +/\ destroyRequested = <> +/\ balC = <<"NONE", "PENDING">> +/\ balD = <<"NONE", "NONE">> +/\ instOffset = <<0, -1>> +/\ preC = <<[deps |-> {}, ballistic |-> FALSE], [deps |-> {}, ballistic |-> TRUE]>> +/\ preD = <<[deps |-> {1, 2}, ballistic |-> FALSE], [deps |-> {2}, ballistic |-> FALSE]>> +/\ seqCtr = 1 +/\ cur = <<[first |-> 0, size |-> 2]>> +/\ allocatedEver = {1} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = <<[inst |-> 1, isReady |-> FALSE, defNote |-> FALSE, seq |-> 1]>> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = << >> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<>> +/\ wasDeferred = {} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"CLEAN", "UNFIRED">> +/\ instState = <<"ALLOCATED", "CREATE_PENDING_DESTROY">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0>> +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 6: +/\ destroyRequested = <> +/\ balC = <<"NONE", "FIRED">> +/\ balD = <<"NONE", "NONE">> +/\ instOffset = <<0, -1>> +/\ preC = <<[deps |-> {}, ballistic |-> FALSE], [deps |-> {}, ballistic |-> TRUE]>> +/\ preD = <<[deps |-> {1, 2}, ballistic |-> FALSE], [deps |-> {2}, ballistic |-> FALSE]>> +/\ seqCtr = 1 +/\ cur = <<[first |-> 0, size |-> 2]>> +/\ allocatedEver = {1} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = <<[inst |-> 1, isReady |-> FALSE, defNote |-> FALSE, seq |-> 1]>> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = << >> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<>> +/\ wasDeferred = {} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"CLEAN", "UNFIRED">> +/\ instState = <<"ALLOCATED", "CREATE_PENDING_DESTROY">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0>> +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +State 7: +/\ destroyRequested = <> +/\ balC = <<"NONE", "FIRED">> +/\ balD = <<"NONE", "NONE">> +/\ instOffset = <<0, -1>> +/\ preC = <<[deps |-> {}, ballistic |-> FALSE], [deps |-> {}, ballistic |-> TRUE]>> +/\ preD = <<[deps |-> {1, 2}, ballistic |-> FALSE], [deps |-> {2}, ballistic |-> FALSE]>> +/\ seqCtr = 2 +/\ cur = <<[first |-> 0, size |-> 2]>> +/\ allocatedEver = {1} +/\ futMismatch = FALSE +/\ createWaiter = <> +/\ pendingReleases = << [inst |-> 1, isReady |-> FALSE, defNote |-> FALSE, seq |-> 1], + [inst |-> 2, isReady |-> FALSE, defNote |-> FALSE, seq |-> 2] >> +/\ curFreed = {} +/\ failedVia = <<"NONE", "NONE">> +/\ missingFree = FALSE +/\ rel = <<[first |-> 0, size |-> 2]>> +/\ readyAtRebuild = FALSE +/\ pendingAllocs = <<[inst |-> 2, size |-> 2, lastSeq |-> 1]>> +/\ wasDeferred = {2} +/\ structuralAssertFailed = FALSE +/\ fut = << >> +/\ eCreated = <<"CLEAN", "UNFIRED">> +/\ instState = <<"ALLOCATED", "ALLOC_DEFERRED">> +/\ createRequested = <> +/\ unblockFailed = FALSE +/\ notifyCount = <<0, 0>> +/\ poisonReplayBad = FALSE +/\ destroyWaiter = <> + +4029 states generated, 2020 distinct states found, 910 states left on queue. +The depth of the complete state graph search is 9. +The average outdegree of the complete state graph is 2 (minimum is 0, the maximum 10 and the 95th percentile is 5). +Finished in 00s at (2026-08-25 13:49:09)